Lichen

compiler/transformer.py

232:2d23fd9e64ec
2016-11-24 Paul Boddie Only generate signatures for generated literal instantiators.
     1 """Parse tree transformation module.     2      3 Transforms Python source code into an abstract syntax tree (AST)     4 defined in the ast module.     5      6 The simplest ways to invoke this module are via parse and parseFile.     7 parse(buf) -> AST     8 parseFile(path) -> AST     9 """    10     11 # Original version written by Greg Stein (gstein@lyra.org)    12 #                         and Bill Tutt (rassilon@lima.mudlib.org)    13 # February 1997.    14 #    15 # Modifications and improvements for Python 2.0 by Jeremy Hylton and    16 # Mark Hammond    17 #    18 # Some fixes to try to have correct line number on almost all nodes    19 # (except Module, Discard and Stmt) added by Sylvain Thenault    20 #    21 # Portions of this file are:    22 # Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.    23 #    24 # This module is provided under a BSD-ish license. See    25 #   http://www.opensource.org/licenses/bsd-license.html    26 # and replace OWNER, ORGANIZATION, and YEAR as appropriate.    27     28 from compiler.ast import *    29 import parser    30 import symbol    31 import token    32     33 class WalkerError(StandardError):    34     pass    35     36 from compiler.consts import CO_VARARGS, CO_VARKEYWORDS    37 from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY    38     39 def parseFile(path):    40     f = open(path, "U")    41     # XXX The parser API tolerates files without a trailing newline,    42     # but not strings without a trailing newline.  Always add an extra    43     # newline to the file contents, since we're going through the string    44     # version of the API.    45     src = f.read() + "\n"    46     f.close()    47     return parse(src)    48     49 def parse(buf, mode="exec"):    50     if mode == "exec" or mode == "single":    51         return Transformer().parsesuite(buf)    52     elif mode == "eval":    53         return Transformer().parseexpr(buf)    54     else:    55         raise ValueError("compile() arg 3 must be"    56                          " 'exec' or 'eval' or 'single'")    57     58 def extractLineNo(ast):    59     if not isinstance(ast[1], tuple):    60         # get a terminal node    61         return ast[2]    62     for child in ast[1:]:    63         if isinstance(child, tuple):    64             lineno = extractLineNo(child)    65             if lineno is not None:    66                 return lineno    67     68 def Node(*args):    69     kind = args[0]    70     if kind in nodes:    71         try:    72             return nodes[kind](*args[1:])    73         except TypeError:    74             print nodes[kind], len(args), args    75             raise    76     else:    77         raise WalkerError, "Can't find appropriate Node type: %s" % str(args)    78         #return apply(ast.Node, args)    79     80 class Transformer:    81     """Utility object for transforming Python parse trees.    82     83     Exposes the following methods:    84         tree = transform(ast_tree)    85         tree = parsesuite(text)    86         tree = parseexpr(text)    87         tree = parsefile(fileob | filename)    88     """    89     90     def __init__(self):    91         self._dispatch = {}    92         for value, name in symbol.sym_name.items():    93             if hasattr(self, name):    94                 self._dispatch[value] = getattr(self, name)    95         self._dispatch[token.NEWLINE] = self.com_NEWLINE    96         self._atom_dispatch = {token.LPAR: self.atom_lpar,    97                                token.LSQB: self.atom_lsqb,    98                                token.LBRACE: self.atom_lbrace,    99                                token.BACKQUOTE: self.atom_backquote,   100                                token.NUMBER: self.atom_number,   101                                token.STRING: self.atom_string,   102                                token.NAME: self.atom_name,   103                                }   104         self.encoding = None   105    106     def transform(self, tree):   107         """Transform an AST into a modified parse tree."""   108         if not (isinstance(tree, tuple) or isinstance(tree, list)):   109             tree = parser.st2tuple(tree, line_info=1)   110         return self.compile_node(tree)   111    112     def parsesuite(self, text):   113         """Return a modified parse tree for the given suite text."""   114         return self.transform(parser.suite(text))   115    116     def parseexpr(self, text):   117         """Return a modified parse tree for the given expression text."""   118         return self.transform(parser.expr(text))   119    120     def parsefile(self, file):   121         """Return a modified parse tree for the contents of the given file."""   122         if type(file) == type(''):   123             file = open(file)   124         return self.parsesuite(file.read())   125    126     # --------------------------------------------------------------   127     #   128     # PRIVATE METHODS   129     #   130    131     def compile_node(self, node):   132         ### emit a line-number node?   133         n = node[0]   134    135         if n == symbol.encoding_decl:   136             self.encoding = node[2]   137             node = node[1]   138             n = node[0]   139    140         if n == symbol.single_input:   141             return self.single_input(node[1:])   142         if n == symbol.file_input:   143             return self.file_input(node[1:])   144         if n == symbol.eval_input:   145             return self.eval_input(node[1:])   146         if n == symbol.lambdef:   147             return self.lambdef(node[1:])   148         if n == symbol.funcdef:   149             return self.funcdef(node[1:])   150         if n == symbol.classdef:   151             return self.classdef(node[1:])   152    153         raise WalkerError, ('unexpected node type', n)   154    155     def single_input(self, node):   156         ### do we want to do anything about being "interactive" ?   157    158         # NEWLINE | simple_stmt | compound_stmt NEWLINE   159         n = node[0][0]   160         if n != token.NEWLINE:   161             return self.com_stmt(node[0])   162    163         return Pass()   164    165     def file_input(self, nodelist):   166         doc = self.get_docstring(nodelist, symbol.file_input)   167         if doc is not None:   168             i = 1   169         else:   170             i = 0   171         stmts = []   172         for node in nodelist[i:]:   173             if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:   174                 self.com_append_stmt(stmts, node)   175         return Module(doc, Stmt(stmts))   176    177     def eval_input(self, nodelist):   178         # from the built-in function input()   179         ### is this sufficient?   180         return Expression(self.com_node(nodelist[0]))   181    182     def decorator_name(self, nodelist):   183         listlen = len(nodelist)   184         assert listlen >= 1 and listlen % 2 == 1   185    186         item = self.atom_name(nodelist)   187         i = 1   188         while i < listlen:   189             assert nodelist[i][0] == token.DOT   190             assert nodelist[i + 1][0] == token.NAME   191             item = Getattr(item, nodelist[i + 1][1])   192             i += 2   193    194         return item   195    196     def decorator(self, nodelist):   197         # '@' dotted_name [ '(' [arglist] ')' ]   198         assert len(nodelist) in (3, 5, 6)   199         assert nodelist[0][0] == token.AT   200         assert nodelist[-1][0] == token.NEWLINE   201    202         assert nodelist[1][0] == symbol.dotted_name   203         funcname = self.decorator_name(nodelist[1][1:])   204    205         if len(nodelist) > 3:   206             assert nodelist[2][0] == token.LPAR   207             expr = self.com_call_function(funcname, nodelist[3])   208         else:   209             expr = funcname   210    211         return expr   212    213     def decorators(self, nodelist):   214         # decorators: decorator ([NEWLINE] decorator)* NEWLINE   215         items = []   216         for dec_nodelist in nodelist:   217             assert dec_nodelist[0] == symbol.decorator   218             items.append(self.decorator(dec_nodelist[1:]))   219         return Decorators(items)   220    221     def decorated(self, nodelist):   222         assert nodelist[0][0] == symbol.decorators   223         if nodelist[1][0] == symbol.funcdef:   224             n = [nodelist[0]] + list(nodelist[1][1:])   225             return self.funcdef(n)   226         elif nodelist[1][0] == symbol.classdef:   227             decorators = self.decorators(nodelist[0][1:])   228             cls = self.classdef(nodelist[1][1:])   229             cls.decorators = decorators   230             return cls   231         raise WalkerError()   232    233     def funcdef(self, nodelist):   234         #                    -6   -5    -4         -3  -2    -1   235         # funcdef: [decorators] 'def' NAME parameters ':' suite   236         # parameters: '(' [varargslist] ')'   237    238         if len(nodelist) == 6:   239             assert nodelist[0][0] == symbol.decorators   240             decorators = self.decorators(nodelist[0][1:])   241         else:   242             assert len(nodelist) == 5   243             decorators = None   244    245         lineno = nodelist[-4][2]   246         name = nodelist[-4][1]   247         args = nodelist[-3][2]   248    249         if args[0] == symbol.varargslist:   250             names, defaults, flags = self.com_arglist(args[1:])   251         else:   252             names = defaults = ()   253             flags = 0   254         doc = self.get_docstring(nodelist[-1])   255    256         # code for function   257         code = self.com_node(nodelist[-1])   258    259         if doc is not None:   260             assert isinstance(code, Stmt)   261             assert isinstance(code.nodes[0], Discard)   262             del code.nodes[0]   263         return Function(decorators, name, names, defaults, flags, doc, code,   264                      lineno=lineno)   265    266     def lambdef(self, nodelist):   267         # lambdef: 'lambda' [varargslist] ':' test   268         if nodelist[2][0] == symbol.varargslist:   269             names, defaults, flags = self.com_arglist(nodelist[2][1:])   270         else:   271             names = defaults = ()   272             flags = 0   273    274         # code for lambda   275         code = self.com_node(nodelist[-1])   276    277         return Lambda(names, defaults, flags, code, lineno=nodelist[1][2])   278     old_lambdef = lambdef   279    280     def classdef(self, nodelist):   281         # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite   282    283         name = nodelist[1][1]   284         doc = self.get_docstring(nodelist[-1])   285         if nodelist[2][0] == token.COLON:   286             bases = []   287         elif nodelist[3][0] == token.RPAR:   288             bases = []   289         else:   290             bases = self.com_bases(nodelist[3])   291    292         # code for class   293         code = self.com_node(nodelist[-1])   294    295         if doc is not None:   296             assert isinstance(code, Stmt)   297             assert isinstance(code.nodes[0], Discard)   298             del code.nodes[0]   299    300         return Class(name, bases, doc, code, lineno=nodelist[1][2])   301    302     def stmt(self, nodelist):   303         return self.com_stmt(nodelist[0])   304    305     small_stmt = stmt   306     flow_stmt = stmt   307     compound_stmt = stmt   308    309     def simple_stmt(self, nodelist):   310         # small_stmt (';' small_stmt)* [';'] NEWLINE   311         stmts = []   312         for i in range(0, len(nodelist), 2):   313             self.com_append_stmt(stmts, nodelist[i])   314         return Stmt(stmts)   315    316     def parameters(self, nodelist):   317         raise WalkerError   318    319     def varargslist(self, nodelist):   320         raise WalkerError   321    322     def fpdef(self, nodelist):   323         raise WalkerError   324    325     def fplist(self, nodelist):   326         raise WalkerError   327    328     def dotted_name(self, nodelist):   329         raise WalkerError   330    331     def comp_op(self, nodelist):   332         raise WalkerError   333    334     def trailer(self, nodelist):   335         raise WalkerError   336    337     def sliceop(self, nodelist):   338         raise WalkerError   339    340     def argument(self, nodelist):   341         raise WalkerError   342    343     # --------------------------------------------------------------   344     #   345     # STATEMENT NODES  (invoked by com_node())   346     #   347    348     def expr_stmt(self, nodelist):   349         # augassign testlist | testlist ('=' testlist)*   350         en = nodelist[-1]   351         exprNode = self.lookup_node(en)(en[1:])   352         if len(nodelist) == 1:   353             return Discard(exprNode, lineno=exprNode.lineno)   354         if nodelist[1][0] == token.EQUAL:   355             nodesl = []   356             for i in range(0, len(nodelist) - 2, 2):   357                 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))   358             return Assign(nodesl, exprNode, lineno=nodelist[1][2])   359         else:   360             lval = self.com_augassign(nodelist[0])   361             op = self.com_augassign_op(nodelist[1])   362             return AugAssign(lval, op[1], exprNode, lineno=op[2])   363         raise WalkerError, "can't get here"   364    365     def print_stmt(self, nodelist):   366         # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])   367         items = []   368         if len(nodelist) == 1:   369             start = 1   370             dest = None   371         elif nodelist[1][0] == token.RIGHTSHIFT:   372             assert len(nodelist) == 3 \   373                    or nodelist[3][0] == token.COMMA   374             dest = self.com_node(nodelist[2])   375             start = 4   376         else:   377             dest = None   378             start = 1   379         for i in range(start, len(nodelist), 2):   380             items.append(self.com_node(nodelist[i]))   381         if nodelist[-1][0] == token.COMMA:   382             return Print(items, dest, lineno=nodelist[0][2])   383         return Printnl(items, dest, lineno=nodelist[0][2])   384    385     def del_stmt(self, nodelist):   386         return self.com_assign(nodelist[1], OP_DELETE)   387    388     def pass_stmt(self, nodelist):   389         return Pass(lineno=nodelist[0][2])   390    391     def break_stmt(self, nodelist):   392         return Break(lineno=nodelist[0][2])   393    394     def continue_stmt(self, nodelist):   395         return Continue(lineno=nodelist[0][2])   396    397     def return_stmt(self, nodelist):   398         # return: [testlist]   399         if len(nodelist) < 2:   400             return Return(Const(None), lineno=nodelist[0][2])   401         return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])   402    403     def yield_stmt(self, nodelist):   404         expr = self.com_node(nodelist[0])   405         return Discard(expr, lineno=expr.lineno)   406    407     def yield_expr(self, nodelist):   408         if len(nodelist) > 1:   409             value = self.com_node(nodelist[1])   410         else:   411             value = Const(None)   412         return Yield(value, lineno=nodelist[0][2])   413    414     def raise_stmt(self, nodelist):   415         # raise: [test [',' test [',' test]]]   416         if len(nodelist) > 5:   417             expr3 = self.com_node(nodelist[5])   418         else:   419             expr3 = None   420         if len(nodelist) > 3:   421             expr2 = self.com_node(nodelist[3])   422         else:   423             expr2 = None   424         if len(nodelist) > 1:   425             expr1 = self.com_node(nodelist[1])   426         else:   427             expr1 = None   428         return Raise(expr1, expr2, expr3, lineno=nodelist[0][2])   429    430     def import_stmt(self, nodelist):   431         # import_stmt: import_name | import_from   432         assert len(nodelist) == 1   433         return self.com_node(nodelist[0])   434    435     def import_name(self, nodelist):   436         # import_name: 'import' dotted_as_names   437         return Import(self.com_dotted_as_names(nodelist[1]),   438                       lineno=nodelist[0][2])   439    440     def import_from(self, nodelist):   441         # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' |   442         #    '(' import_as_names ')' | import_as_names)   443         assert nodelist[0][1] == 'from'   444         idx = 1   445         while nodelist[idx][1] == '.':   446             idx += 1   447         level = idx - 1   448         if nodelist[idx][0] == symbol.dotted_name:   449             fromname = self.com_dotted_name(nodelist[idx])   450             idx += 1   451         else:   452             fromname = ""   453         assert nodelist[idx][1] == 'import'   454         if nodelist[idx + 1][0] == token.STAR:   455             return From(fromname, [('*', None)], level,   456                         lineno=nodelist[0][2])   457         else:   458             node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)]   459             return From(fromname, self.com_import_as_names(node), level,   460                         lineno=nodelist[0][2])   461    462     def global_stmt(self, nodelist):   463         # global: NAME (',' NAME)*   464         names = []   465         for i in range(1, len(nodelist), 2):   466             names.append(nodelist[i][1])   467         return Global(names, lineno=nodelist[0][2])   468    469     def exec_stmt(self, nodelist):   470         # exec_stmt: 'exec' expr ['in' expr [',' expr]]   471         expr1 = self.com_node(nodelist[1])   472         if len(nodelist) >= 4:   473             expr2 = self.com_node(nodelist[3])   474             if len(nodelist) >= 6:   475                 expr3 = self.com_node(nodelist[5])   476             else:   477                 expr3 = None   478         else:   479             expr2 = expr3 = None   480    481         return Exec(expr1, expr2, expr3, lineno=nodelist[0][2])   482    483     def assert_stmt(self, nodelist):   484         # 'assert': test, [',' test]   485         expr1 = self.com_node(nodelist[1])   486         if (len(nodelist) == 4):   487             expr2 = self.com_node(nodelist[3])   488         else:   489             expr2 = None   490         return Assert(expr1, expr2, lineno=nodelist[0][2])   491    492     def if_stmt(self, nodelist):   493         # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite]   494         tests = []   495         for i in range(0, len(nodelist) - 3, 4):   496             testNode = self.com_node(nodelist[i + 1])   497             suiteNode = self.com_node(nodelist[i + 3])   498             tests.append((testNode, suiteNode))   499    500         if len(nodelist) % 4 == 3:   501             elseNode = self.com_node(nodelist[-1])   502 ##      elseNode.lineno = nodelist[-1][1][2]   503         else:   504             elseNode = None   505         return If(tests, elseNode, lineno=nodelist[0][2])   506    507     def while_stmt(self, nodelist):   508         # 'while' test ':' suite ['else' ':' suite]   509    510         testNode = self.com_node(nodelist[1])   511         bodyNode = self.com_node(nodelist[3])   512    513         if len(nodelist) > 4:   514             elseNode = self.com_node(nodelist[6])   515         else:   516             elseNode = None   517    518         return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])   519    520     def for_stmt(self, nodelist):   521         # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]   522    523         assignNode = self.com_assign(nodelist[1], OP_ASSIGN)   524         listNode = self.com_node(nodelist[3])   525         bodyNode = self.com_node(nodelist[5])   526    527         if len(nodelist) > 8:   528             elseNode = self.com_node(nodelist[8])   529         else:   530             elseNode = None   531    532         return For(assignNode, listNode, bodyNode, elseNode,   533                    lineno=nodelist[0][2])   534    535     def try_stmt(self, nodelist):   536         return self.com_try_except_finally(nodelist)   537    538     def with_stmt(self, nodelist):   539         return self.com_with(nodelist)   540    541     def suite(self, nodelist):   542         # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT   543         if len(nodelist) == 1:   544             return self.com_stmt(nodelist[0])   545    546         stmts = []   547         for node in nodelist:   548             if node[0] == symbol.stmt:   549                 self.com_append_stmt(stmts, node)   550         return Stmt(stmts)   551    552     # --------------------------------------------------------------   553     #   554     # EXPRESSION NODES  (invoked by com_node())   555     #   556    557     def testlist(self, nodelist):   558         # testlist: expr (',' expr)* [',']   559         # testlist_safe: test [(',' test)+ [',']]   560         # exprlist: expr (',' expr)* [',']   561         return self.com_binary(Tuple, nodelist)   562    563     testlist_safe = testlist # XXX   564     testlist1 = testlist   565     exprlist = testlist   566    567     def testlist_comp(self, nodelist):   568         # test ( comp_for | (',' test)* [','] )   569         assert nodelist[0][0] == symbol.test   570         if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for:   571             test = self.com_node(nodelist[0])   572             return self.com_generator_expression(test, nodelist[1])   573         return self.testlist(nodelist)   574    575     def test(self, nodelist):   576         # or_test ['if' or_test 'else' test] | lambdef   577         if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:   578             return self.lambdef(nodelist[0])   579         then = self.com_node(nodelist[0])   580         if len(nodelist) > 1:   581             assert len(nodelist) == 5   582             assert nodelist[1][1] == 'if'   583             assert nodelist[3][1] == 'else'   584             test = self.com_node(nodelist[2])   585             else_ = self.com_node(nodelist[4])   586             return IfExp(test, then, else_, lineno=nodelist[1][2])   587         return then   588    589     def or_test(self, nodelist):   590         # and_test ('or' and_test)* | lambdef   591         if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:   592             return self.lambdef(nodelist[0])   593         return self.com_binary(Or, nodelist)   594     old_test = or_test   595    596     def and_test(self, nodelist):   597         # not_test ('and' not_test)*   598         return self.com_binary(And, nodelist)   599    600     def not_test(self, nodelist):   601         # 'not' not_test | comparison   602         result = self.com_node(nodelist[-1])   603         if len(nodelist) == 2:   604             return Not(result, lineno=nodelist[0][2])   605         return result   606    607     def comparison(self, nodelist):   608         # comparison: expr (comp_op expr)*   609         node = self.com_node(nodelist[0])   610         if len(nodelist) == 1:   611             return node   612    613         results = []   614         for i in range(2, len(nodelist), 2):   615             nl = nodelist[i-1]   616    617             # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='   618             #          | 'in' | 'not' 'in' | 'is' | 'is' 'not'   619             n = nl[1]   620             if n[0] == token.NAME:   621                 type = n[1]   622                 if len(nl) == 3:   623                     if type == 'not':   624                         type = 'not in'   625                     else:   626                         type = 'is not'   627             else:   628                 type = _cmp_types[n[0]]   629    630             lineno = nl[1][2]   631             results.append((type, self.com_node(nodelist[i])))   632    633         # we need a special "compare" node so that we can distinguish   634         #   3 < x < 5   from    (3 < x) < 5   635         # the two have very different semantics and results (note that the   636         # latter form is always true)   637    638         return Compare(node, results, lineno=lineno)   639    640     def expr(self, nodelist):   641         # xor_expr ('|' xor_expr)*   642         return self.com_binary(Bitor, nodelist)   643    644     def xor_expr(self, nodelist):   645         # xor_expr ('^' xor_expr)*   646         return self.com_binary(Bitxor, nodelist)   647    648     def and_expr(self, nodelist):   649         # xor_expr ('&' xor_expr)*   650         return self.com_binary(Bitand, nodelist)   651    652     def shift_expr(self, nodelist):   653         # shift_expr ('<<'|'>>' shift_expr)*   654         node = self.com_node(nodelist[0])   655         for i in range(2, len(nodelist), 2):   656             right = self.com_node(nodelist[i])   657             if nodelist[i-1][0] == token.LEFTSHIFT:   658                 node = LeftShift([node, right], lineno=nodelist[1][2])   659             elif nodelist[i-1][0] == token.RIGHTSHIFT:   660                 node = RightShift([node, right], lineno=nodelist[1][2])   661             else:   662                 raise ValueError, "unexpected token: %s" % nodelist[i-1][0]   663         return node   664    665     def arith_expr(self, nodelist):   666         node = self.com_node(nodelist[0])   667         for i in range(2, len(nodelist), 2):   668             right = self.com_node(nodelist[i])   669             if nodelist[i-1][0] == token.PLUS:   670                 node = Add([node, right], lineno=nodelist[1][2])   671             elif nodelist[i-1][0] == token.MINUS:   672                 node = Sub([node, right], lineno=nodelist[1][2])   673             else:   674                 raise ValueError, "unexpected token: %s" % nodelist[i-1][0]   675         return node   676    677     def term(self, nodelist):   678         node = self.com_node(nodelist[0])   679         for i in range(2, len(nodelist), 2):   680             right = self.com_node(nodelist[i])   681             t = nodelist[i-1][0]   682             if t == token.STAR:   683                 node = Mul([node, right])   684             elif t == token.SLASH:   685                 node = Div([node, right])   686             elif t == token.PERCENT:   687                 node = Mod([node, right])   688             elif t == token.DOUBLESLASH:   689                 node = FloorDiv([node, right])   690             else:   691                 raise ValueError, "unexpected token: %s" % t   692             node.lineno = nodelist[1][2]   693         return node   694    695     def factor(self, nodelist):   696         elt = nodelist[0]   697         t = elt[0]   698         node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])   699         # need to handle (unary op)constant here...   700         if t == token.PLUS:   701             return UnaryAdd(node, lineno=elt[2])   702         elif t == token.MINUS:   703             return UnarySub(node, lineno=elt[2])   704         elif t == token.TILDE:   705             node = Invert(node, lineno=elt[2])   706         return node   707    708     def power(self, nodelist):   709         # power: atom trailer* ('**' factor)*   710         node = self.com_node(nodelist[0])   711         for i in range(1, len(nodelist)):   712             elt = nodelist[i]   713             if elt[0] == token.DOUBLESTAR:   714                 return Power([node, self.com_node(nodelist[i+1])],   715                              lineno=elt[2])   716    717             node = self.com_apply_trailer(node, elt)   718    719         return node   720    721     def atom(self, nodelist):   722         return self._atom_dispatch[nodelist[0][0]](nodelist)   723    724     def atom_lpar(self, nodelist):   725         if nodelist[1][0] == token.RPAR:   726             return Tuple((), lineno=nodelist[0][2])   727         return self.com_node(nodelist[1])   728    729     def atom_lsqb(self, nodelist):   730         if nodelist[1][0] == token.RSQB:   731             return List((), lineno=nodelist[0][2])   732         return self.com_list_constructor(nodelist[1])   733    734     def atom_lbrace(self, nodelist):   735         if nodelist[1][0] == token.RBRACE:   736             return Dict((), lineno=nodelist[0][2])   737         return self.com_dictorsetmaker(nodelist[1])   738    739     def atom_backquote(self, nodelist):   740         return Backquote(self.com_node(nodelist[1]))   741    742     def atom_number(self, nodelist):   743         ### need to verify this matches compile.c   744         k = eval(nodelist[0][1])   745         return Const(k, lineno=nodelist[0][2])   746    747     def decode_literal(self, lit):   748         if self.encoding:   749             # this is particularly fragile & a bit of a   750             # hack... changes in compile.c:parsestr and   751             # tokenizer.c must be reflected here.   752             if self.encoding not in ['utf-8', 'iso-8859-1']:   753                 lit = unicode(lit, 'utf-8').encode(self.encoding)   754             return eval("# coding: %s\n%s" % (self.encoding, lit))   755         else:   756             return eval(lit)   757    758     def atom_string(self, nodelist):   759         k = ''   760         for node in nodelist:   761             k += self.decode_literal(node[1])   762         return Const(k, lineno=nodelist[0][2])   763    764     def atom_name(self, nodelist):   765         return Name(nodelist[0][1], lineno=nodelist[0][2])   766    767     # --------------------------------------------------------------   768     #   769     # INTERNAL PARSING UTILITIES   770     #   771    772     # The use of com_node() introduces a lot of extra stack frames,   773     # enough to cause a stack overflow compiling test.test_parser with   774     # the standard interpreter recursionlimit.  The com_node() is a   775     # convenience function that hides the dispatch details, but comes   776     # at a very high cost.  It is more efficient to dispatch directly   777     # in the callers.  In these cases, use lookup_node() and call the   778     # dispatched node directly.   779    780     def lookup_node(self, node):   781         return self._dispatch[node[0]]   782    783     def com_node(self, node):   784         # Note: compile.c has handling in com_node for del_stmt, pass_stmt,   785         #       break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,   786         #       and compound_stmt.   787         #       We'll just dispatch them.   788         return self._dispatch[node[0]](node[1:])   789    790     def com_NEWLINE(self, *args):   791         # A ';' at the end of a line can make a NEWLINE token appear   792         # here, Render it harmless. (genc discards ('discard',   793         # ('const', xxxx)) Nodes)   794         return Discard(Const(None))   795    796     def com_arglist(self, nodelist):   797         # varargslist:   798         #     (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME)   799         #   | fpdef ['=' test] (',' fpdef ['=' test])* [',']   800         # fpdef: NAME | '(' fplist ')'   801         # fplist: fpdef (',' fpdef)* [',']   802         names = []   803         defaults = []   804         flags = 0   805    806         i = 0   807         while i < len(nodelist):   808             node = nodelist[i]   809             if node[0] == token.STAR or node[0] == token.DOUBLESTAR:   810                 if node[0] == token.STAR:   811                     node = nodelist[i+1]   812                     if node[0] == token.NAME:   813                         names.append(node[1])   814                         flags = flags | CO_VARARGS   815                         i = i + 3   816    817                 if i < len(nodelist):   818                     # should be DOUBLESTAR   819                     t = nodelist[i][0]   820                     if t == token.DOUBLESTAR:   821                         node = nodelist[i+1]   822                     else:   823                         raise ValueError, "unexpected token: %s" % t   824                     names.append(node[1])   825                     flags = flags | CO_VARKEYWORDS   826    827                 break   828    829             # fpdef: NAME | '(' fplist ')'   830             names.append(self.com_fpdef(node))   831    832             i = i + 1   833             if i < len(nodelist) and nodelist[i][0] == token.EQUAL:   834                 defaults.append(self.com_node(nodelist[i + 1]))   835                 i = i + 2   836             elif len(defaults):   837                 # we have already seen an argument with default, but here   838                 # came one without   839                 raise SyntaxError, "non-default argument follows default argument"   840    841             # skip the comma   842             i = i + 1   843    844         return names, defaults, flags   845    846     def com_fpdef(self, node):   847         # fpdef: NAME | '(' fplist ')'   848         if node[1][0] == token.LPAR:   849             return self.com_fplist(node[2])   850         return node[1][1]   851    852     def com_fplist(self, node):   853         # fplist: fpdef (',' fpdef)* [',']   854         if len(node) == 2:   855             return self.com_fpdef(node[1])   856         list = []   857         for i in range(1, len(node), 2):   858             list.append(self.com_fpdef(node[i]))   859         return tuple(list)   860    861     def com_dotted_name(self, node):   862         # String together the dotted names and return the string   863         name = ""   864         for n in node:   865             if type(n) == type(()) and n[0] == 1:   866                 name = name + n[1] + '.'   867         return name[:-1]   868    869     def com_dotted_as_name(self, node):   870         assert node[0] == symbol.dotted_as_name   871         node = node[1:]   872         dot = self.com_dotted_name(node[0][1:])   873         if len(node) == 1:   874             return dot, None   875         assert node[1][1] == 'as'   876         assert node[2][0] == token.NAME   877         return dot, node[2][1]   878    879     def com_dotted_as_names(self, node):   880         assert node[0] == symbol.dotted_as_names   881         node = node[1:]   882         names = [self.com_dotted_as_name(node[0])]   883         for i in range(2, len(node), 2):   884             names.append(self.com_dotted_as_name(node[i]))   885         return names   886    887     def com_import_as_name(self, node):   888         assert node[0] == symbol.import_as_name   889         node = node[1:]   890         assert node[0][0] == token.NAME   891         if len(node) == 1:   892             return node[0][1], None   893         assert node[1][1] == 'as', node   894         assert node[2][0] == token.NAME   895         return node[0][1], node[2][1]   896    897     def com_import_as_names(self, node):   898         assert node[0] == symbol.import_as_names   899         node = node[1:]   900         names = [self.com_import_as_name(node[0])]   901         for i in range(2, len(node), 2):   902             names.append(self.com_import_as_name(node[i]))   903         return names   904    905     def com_bases(self, node):   906         bases = []   907         for i in range(1, len(node), 2):   908             bases.append(self.com_node(node[i]))   909         return bases   910    911     def com_try_except_finally(self, nodelist):   912         # ('try' ':' suite   913         #  ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite]   914         #   | 'finally' ':' suite))   915    916         if nodelist[3][0] == token.NAME:   917             # first clause is a finally clause: only try-finally   918             return TryFinally(self.com_node(nodelist[2]),   919                               self.com_node(nodelist[5]),   920                               lineno=nodelist[0][2])   921    922         #tryexcept:  [TryNode, [except_clauses], elseNode)]   923         clauses = []   924         elseNode = None   925         finallyNode = None   926         for i in range(3, len(nodelist), 3):   927             node = nodelist[i]   928             if node[0] == symbol.except_clause:   929                 # except_clause: 'except' [expr [(',' | 'as') expr]] */   930                 if len(node) > 2:   931                     expr1 = self.com_node(node[2])   932                     if len(node) > 4:   933                         expr2 = self.com_assign(node[4], OP_ASSIGN)   934                     else:   935                         expr2 = None   936                 else:   937                     expr1 = expr2 = None   938                 clauses.append((expr1, expr2, self.com_node(nodelist[i+2])))   939    940             if node[0] == token.NAME:   941                 if node[1] == 'else':   942                     elseNode = self.com_node(nodelist[i+2])   943                 elif node[1] == 'finally':   944                     finallyNode = self.com_node(nodelist[i+2])   945         try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode,   946                                lineno=nodelist[0][2])   947         if finallyNode:   948             return TryFinally(try_except, finallyNode, lineno=nodelist[0][2])   949         else:   950             return try_except   951    952     def com_with(self, nodelist):   953         # with_stmt: 'with' with_item (',' with_item)* ':' suite   954         body = self.com_node(nodelist[-1])   955         for i in range(len(nodelist) - 3, 0, -2):   956             ret = self.com_with_item(nodelist[i], body, nodelist[0][2])   957             if i == 1:   958                 return ret   959             body = ret   960    961     def com_with_item(self, nodelist, body, lineno):   962         # with_item: test ['as' expr]   963         if len(nodelist) == 4:   964             var = self.com_assign(nodelist[3], OP_ASSIGN)   965         else:   966             var = None   967         expr = self.com_node(nodelist[1])   968         return With(expr, var, body, lineno=lineno)   969    970     def com_augassign_op(self, node):   971         assert node[0] == symbol.augassign   972         return node[1]   973    974     def com_augassign(self, node):   975         """Return node suitable for lvalue of augmented assignment   976    977         Names, slices, and attributes are the only allowable nodes.   978         """   979         l = self.com_node(node)   980         if l.__class__ in (Name, Slice, Subscript, Getattr):   981             return l   982         raise SyntaxError, "can't assign to %s" % l.__class__.__name__   983    984     def com_assign(self, node, assigning):   985         # return a node suitable for use as an "lvalue"   986         # loop to avoid trivial recursion   987         while 1:   988             t = node[0]   989             if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp):   990                 if len(node) > 2:   991                     return self.com_assign_tuple(node, assigning)   992                 node = node[1]   993             elif t in _assign_types:   994                 if len(node) > 2:   995                     raise SyntaxError, "can't assign to operator"   996                 node = node[1]   997             elif t == symbol.power:   998                 if node[1][0] != symbol.atom:   999                     raise SyntaxError, "can't assign to operator"  1000                 if len(node) > 2:  1001                     primary = self.com_node(node[1])  1002                     for i in range(2, len(node)-1):  1003                         ch = node[i]  1004                         if ch[0] == token.DOUBLESTAR:  1005                             raise SyntaxError, "can't assign to operator"  1006                         primary = self.com_apply_trailer(primary, ch)  1007                     return self.com_assign_trailer(primary, node[-1],  1008                                                    assigning)  1009                 node = node[1]  1010             elif t == symbol.atom:  1011                 t = node[1][0]  1012                 if t == token.LPAR:  1013                     node = node[2]  1014                     if node[0] == token.RPAR:  1015                         raise SyntaxError, "can't assign to ()"  1016                 elif t == token.LSQB:  1017                     node = node[2]  1018                     if node[0] == token.RSQB:  1019                         raise SyntaxError, "can't assign to []"  1020                     return self.com_assign_list(node, assigning)  1021                 elif t == token.NAME:  1022                     return self.com_assign_name(node[1], assigning)  1023                 else:  1024                     raise SyntaxError, "can't assign to literal"  1025             else:  1026                 raise SyntaxError, "bad assignment (%s)" % t  1027   1028     def com_assign_tuple(self, node, assigning):  1029         assigns = []  1030         for i in range(1, len(node), 2):  1031             assigns.append(self.com_assign(node[i], assigning))  1032         return AssTuple(assigns, lineno=extractLineNo(node))  1033   1034     def com_assign_list(self, node, assigning):  1035         assigns = []  1036         for i in range(1, len(node), 2):  1037             if i + 1 < len(node):  1038                 if node[i + 1][0] == symbol.list_for:  1039                     raise SyntaxError, "can't assign to list comprehension"  1040                 assert node[i + 1][0] == token.COMMA, node[i + 1]  1041             assigns.append(self.com_assign(node[i], assigning))  1042         return AssList(assigns, lineno=extractLineNo(node))  1043   1044     def com_assign_name(self, node, assigning):  1045         return AssName(node[1], assigning, lineno=node[2])  1046   1047     def com_assign_trailer(self, primary, node, assigning):  1048         t = node[1][0]  1049         if t == token.DOT:  1050             return self.com_assign_attr(primary, node[2], assigning)  1051         if t == token.LSQB:  1052             return self.com_subscriptlist(primary, node[2], assigning)  1053         if t == token.LPAR:  1054             raise SyntaxError, "can't assign to function call"  1055         raise SyntaxError, "unknown trailer type: %s" % t  1056   1057     def com_assign_attr(self, primary, node, assigning):  1058         return AssAttr(primary, node[1], assigning, lineno=node[-1])  1059   1060     def com_binary(self, constructor, nodelist):  1061         "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."  1062         l = len(nodelist)  1063         if l == 1:  1064             n = nodelist[0]  1065             return self.lookup_node(n)(n[1:])  1066         items = []  1067         for i in range(0, l, 2):  1068             n = nodelist[i]  1069             items.append(self.lookup_node(n)(n[1:]))  1070         return constructor(items, lineno=extractLineNo(nodelist))  1071   1072     def com_stmt(self, node):  1073         result = self.lookup_node(node)(node[1:])  1074         assert result is not None  1075         if isinstance(result, Stmt):  1076             return result  1077         return Stmt([result])  1078   1079     def com_append_stmt(self, stmts, node):  1080         result = self.lookup_node(node)(node[1:])  1081         assert result is not None  1082         if isinstance(result, Stmt):  1083             stmts.extend(result.nodes)  1084         else:  1085             stmts.append(result)  1086   1087     def com_list_constructor(self, nodelist):  1088         # listmaker: test ( list_for | (',' test)* [','] )  1089         values = []  1090         for i in range(1, len(nodelist)):  1091             if nodelist[i][0] == symbol.list_for:  1092                 assert len(nodelist[i:]) == 1  1093                 return self.com_list_comprehension(values[0],  1094                                                    nodelist[i])  1095             elif nodelist[i][0] == token.COMMA:  1096                 continue  1097             values.append(self.com_node(nodelist[i]))  1098         return List(values, lineno=values[0].lineno)  1099   1100     def com_list_comprehension(self, expr, node):  1101         return self.com_comprehension(expr, None, node, 'list')  1102   1103     def com_comprehension(self, expr1, expr2, node, type):  1104         # list_iter: list_for | list_if  1105         # list_for: 'for' exprlist 'in' testlist [list_iter]  1106         # list_if: 'if' test [list_iter]  1107   1108         # XXX should raise SyntaxError for assignment  1109         # XXX(avassalotti) Set and dict comprehensions should have generator  1110         #                  semantics. In other words, they shouldn't leak  1111         #                  variables outside of the comprehension's scope.  1112   1113         lineno = node[1][2]  1114         fors = []  1115         while node:  1116             t = node[1][1]  1117             if t == 'for':  1118                 assignNode = self.com_assign(node[2], OP_ASSIGN)  1119                 compNode = self.com_node(node[4])  1120                 newfor = ListCompFor(assignNode, compNode, [])  1121                 newfor.lineno = node[1][2]  1122                 fors.append(newfor)  1123                 if len(node) == 5:  1124                     node = None  1125                 elif type == 'list':  1126                     node = self.com_list_iter(node[5])  1127                 else:  1128                     node = self.com_comp_iter(node[5])  1129             elif t == 'if':  1130                 test = self.com_node(node[2])  1131                 newif = ListCompIf(test, lineno=node[1][2])  1132                 newfor.ifs.append(newif)  1133                 if len(node) == 3:  1134                     node = None  1135                 elif type == 'list':  1136                     node = self.com_list_iter(node[3])  1137                 else:  1138                     node = self.com_comp_iter(node[3])  1139             else:  1140                 raise SyntaxError, \  1141                       ("unexpected comprehension element: %s %d"  1142                        % (node, lineno))  1143         if type == 'list':  1144             return ListComp(expr1, fors, lineno=lineno)  1145         elif type == 'set':  1146             return SetComp(expr1, fors, lineno=lineno)  1147         elif type == 'dict':  1148             return DictComp(expr1, expr2, fors, lineno=lineno)  1149         else:  1150             raise ValueError("unexpected comprehension type: " + repr(type))  1151   1152     def com_list_iter(self, node):  1153         assert node[0] == symbol.list_iter  1154         return node[1]  1155   1156     def com_comp_iter(self, node):  1157         assert node[0] == symbol.comp_iter  1158         return node[1]  1159   1160     def com_generator_expression(self, expr, node):  1161         # comp_iter: comp_for | comp_if  1162         # comp_for: 'for' exprlist 'in' test [comp_iter]  1163         # comp_if: 'if' test [comp_iter]  1164   1165         lineno = node[1][2]  1166         fors = []  1167         while node:  1168             t = node[1][1]  1169             if t == 'for':  1170                 assignNode = self.com_assign(node[2], OP_ASSIGN)  1171                 genNode = self.com_node(node[4])  1172                 newfor = GenExprFor(assignNode, genNode, [],  1173                                     lineno=node[1][2])  1174                 fors.append(newfor)  1175                 if (len(node)) == 5:  1176                     node = None  1177                 else:  1178                     node = self.com_comp_iter(node[5])  1179             elif t == 'if':  1180                 test = self.com_node(node[2])  1181                 newif = GenExprIf(test, lineno=node[1][2])  1182                 newfor.ifs.append(newif)  1183                 if len(node) == 3:  1184                     node = None  1185                 else:  1186                     node = self.com_comp_iter(node[3])  1187             else:  1188                 raise SyntaxError, \  1189                         ("unexpected generator expression element: %s %d"  1190                          % (node, lineno))  1191         fors[0].is_outmost = True  1192         return GenExpr(GenExprInner(expr, fors), lineno=lineno)  1193   1194     def com_dictorsetmaker(self, nodelist):  1195         # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |  1196         #                   (test (comp_for | (',' test)* [','])) )  1197         assert nodelist[0] == symbol.dictorsetmaker  1198         nodelist = nodelist[1:]  1199         if len(nodelist) == 1 or nodelist[1][0] == token.COMMA:  1200             # set literal  1201             items = []  1202             for i in range(0, len(nodelist), 2):  1203                 items.append(self.com_node(nodelist[i]))  1204             return Set(items, lineno=items[0].lineno)  1205         elif nodelist[1][0] == symbol.comp_for:  1206             # set comprehension  1207             expr = self.com_node(nodelist[0])  1208             return self.com_comprehension(expr, None, nodelist[1], 'set')  1209         elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for:  1210             # dict comprehension  1211             assert nodelist[1][0] == token.COLON  1212             key = self.com_node(nodelist[0])  1213             value = self.com_node(nodelist[2])  1214             return self.com_comprehension(key, value, nodelist[3], 'dict')  1215         else:  1216             # dict literal  1217             items = []  1218             for i in range(0, len(nodelist), 4):  1219                 items.append((self.com_node(nodelist[i]),  1220                               self.com_node(nodelist[i+2])))  1221             return Dict(items, lineno=items[0][0].lineno)  1222   1223     def com_apply_trailer(self, primaryNode, nodelist):  1224         t = nodelist[1][0]  1225         if t == token.LPAR:  1226             return self.com_call_function(primaryNode, nodelist[2])  1227         if t == token.DOT:  1228             return self.com_select_member(primaryNode, nodelist[2])  1229         if t == token.LSQB:  1230             return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY)  1231   1232         raise SyntaxError, 'unknown node type: %s' % t  1233   1234     def com_select_member(self, primaryNode, nodelist):  1235         if nodelist[0] != token.NAME:  1236             raise SyntaxError, "member must be a name"  1237         return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])  1238   1239     def com_call_function(self, primaryNode, nodelist):  1240         if nodelist[0] == token.RPAR:  1241             return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))  1242         args = []  1243         kw = 0  1244         star_node = dstar_node = None  1245         len_nodelist = len(nodelist)  1246         i = 1  1247         while i < len_nodelist:  1248             node = nodelist[i]  1249   1250             if node[0]==token.STAR:  1251                 if star_node is not None:  1252                     raise SyntaxError, 'already have the varargs indentifier'  1253                 star_node = self.com_node(nodelist[i+1])  1254                 i = i + 3  1255                 continue  1256             elif node[0]==token.DOUBLESTAR:  1257                 if dstar_node is not None:  1258                     raise SyntaxError, 'already have the kwargs indentifier'  1259                 dstar_node = self.com_node(nodelist[i+1])  1260                 i = i + 3  1261                 continue  1262   1263             # positional or named parameters  1264             kw, result = self.com_argument(node, kw, star_node)  1265   1266             if len_nodelist != 2 and isinstance(result, GenExpr) \  1267                and len(node) == 3 and node[2][0] == symbol.comp_for:  1268                 # allow f(x for x in y), but reject f(x for x in y, 1)  1269                 # should use f((x for x in y), 1) instead of f(x for x in y, 1)  1270                 raise SyntaxError, 'generator expression needs parenthesis'  1271   1272             args.append(result)  1273             i = i + 2  1274   1275         return CallFunc(primaryNode, args, star_node, dstar_node,  1276                         lineno=extractLineNo(nodelist))  1277   1278     def com_argument(self, nodelist, kw, star_node):  1279         if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for:  1280             test = self.com_node(nodelist[1])  1281             return 0, self.com_generator_expression(test, nodelist[2])  1282         if len(nodelist) == 2:  1283             if kw:  1284                 raise SyntaxError, "non-keyword arg after keyword arg"  1285             if star_node:  1286                 raise SyntaxError, "only named arguments may follow *expression"  1287             return 0, self.com_node(nodelist[1])  1288         result = self.com_node(nodelist[3])  1289         n = nodelist[1]  1290         while len(n) == 2 and n[0] != token.NAME:  1291             n = n[1]  1292         if n[0] != token.NAME:  1293             raise SyntaxError, "keyword can't be an expression (%s)"%n[0]  1294         node = Keyword(n[1], result, lineno=n[2])  1295         return 1, node  1296   1297     def com_subscriptlist(self, primary, nodelist, assigning):  1298         # slicing:      simple_slicing | extended_slicing  1299         # simple_slicing:   primary "[" short_slice "]"  1300         # extended_slicing: primary "[" slice_list "]"  1301         # slice_list:   slice_item ("," slice_item)* [","]  1302   1303         # backwards compat slice for '[i:j]'  1304         if len(nodelist) == 2:  1305             sub = nodelist[1]  1306             if (sub[1][0] == token.COLON or \  1307                             (len(sub) > 2 and sub[2][0] == token.COLON)) and \  1308                             sub[-1][0] != symbol.sliceop:  1309                 return self.com_slice(primary, sub, assigning)  1310   1311         subscripts = []  1312         for i in range(1, len(nodelist), 2):  1313             subscripts.append(self.com_subscript(nodelist[i]))  1314         return Subscript(primary, assigning, subscripts,  1315                          lineno=extractLineNo(nodelist))  1316   1317     def com_subscript(self, node):  1318         # slice_item: expression | proper_slice | ellipsis  1319         ch = node[1]  1320         t = ch[0]  1321         if t == token.DOT and node[2][0] == token.DOT:  1322             return Ellipsis()  1323         if t == token.COLON or len(node) > 2:  1324             return self.com_sliceobj(node)  1325         return self.com_node(ch)  1326   1327     def com_sliceobj(self, node):  1328         # proper_slice: short_slice | long_slice  1329         # short_slice:  [lower_bound] ":" [upper_bound]  1330         # long_slice:   short_slice ":" [stride]  1331         # lower_bound:  expression  1332         # upper_bound:  expression  1333         # stride:       expression  1334         #  1335         # Note: a stride may be further slicing...  1336   1337         items = []  1338   1339         if node[1][0] == token.COLON:  1340             items.append(Const(None))  1341             i = 2  1342         else:  1343             items.append(self.com_node(node[1]))  1344             # i == 2 is a COLON  1345             i = 3  1346   1347         if i < len(node) and node[i][0] == symbol.test:  1348             items.append(self.com_node(node[i]))  1349             i = i + 1  1350         else:  1351             items.append(Const(None))  1352   1353         # a short_slice has been built. look for long_slice now by looking  1354         # for strides...  1355         for j in range(i, len(node)):  1356             ch = node[j]  1357             if len(ch) == 2:  1358                 items.append(Const(None))  1359             else:  1360                 items.append(self.com_node(ch[2]))  1361         return Sliceobj(items, lineno=extractLineNo(node))  1362   1363     def com_slice(self, primary, node, assigning):  1364         # short_slice:  [lower_bound] ":" [upper_bound]  1365         lower = upper = None  1366         if len(node) == 3:  1367             if node[1][0] == token.COLON:  1368                 upper = self.com_node(node[2])  1369             else:  1370                 lower = self.com_node(node[1])  1371         elif len(node) == 4:  1372             lower = self.com_node(node[1])  1373             upper = self.com_node(node[3])  1374         return Slice(primary, assigning, lower, upper,  1375                      lineno=extractLineNo(node))  1376   1377     def get_docstring(self, node, n=None):  1378         if n is None:  1379             n = node[0]  1380             node = node[1:]  1381         if n == symbol.suite:  1382             if len(node) == 1:  1383                 return self.get_docstring(node[0])  1384             for sub in node:  1385                 if sub[0] == symbol.stmt:  1386                     return self.get_docstring(sub)  1387             return None  1388         if n == symbol.file_input:  1389             for sub in node:  1390                 if sub[0] == symbol.stmt:  1391                     return self.get_docstring(sub)  1392             return None  1393         if n == symbol.atom:  1394             if node[0][0] == token.STRING:  1395                 s = ''  1396                 for t in node:  1397                     s = s + eval(t[1])  1398                 return s  1399             return None  1400         if n == symbol.stmt or n == symbol.simple_stmt \  1401            or n == symbol.small_stmt:  1402             return self.get_docstring(node[0])  1403         if n in _doc_nodes and len(node) == 1:  1404             return self.get_docstring(node[0])  1405         return None  1406   1407   1408 _doc_nodes = [  1409     symbol.expr_stmt,  1410     symbol.testlist,  1411     symbol.testlist_safe,  1412     symbol.test,  1413     symbol.or_test,  1414     symbol.and_test,  1415     symbol.not_test,  1416     symbol.comparison,  1417     symbol.expr,  1418     symbol.xor_expr,  1419     symbol.and_expr,  1420     symbol.shift_expr,  1421     symbol.arith_expr,  1422     symbol.term,  1423     symbol.factor,  1424     symbol.power,  1425     ]  1426   1427 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='  1428 #             | 'in' | 'not' 'in' | 'is' | 'is' 'not'  1429 _cmp_types = {  1430     token.LESS : '<',  1431     token.GREATER : '>',  1432     token.EQEQUAL : '==',  1433     token.EQUAL : '==',  1434     token.LESSEQUAL : '<=',  1435     token.GREATEREQUAL : '>=',  1436     token.NOTEQUAL : '!=',  1437     }  1438   1439 _legal_node_types = [  1440     symbol.funcdef,  1441     symbol.classdef,  1442     symbol.stmt,  1443     symbol.small_stmt,  1444     symbol.flow_stmt,  1445     symbol.simple_stmt,  1446     symbol.compound_stmt,  1447     symbol.expr_stmt,  1448     symbol.print_stmt,  1449     symbol.del_stmt,  1450     symbol.pass_stmt,  1451     symbol.break_stmt,  1452     symbol.continue_stmt,  1453     symbol.return_stmt,  1454     symbol.raise_stmt,  1455     symbol.import_stmt,  1456     symbol.global_stmt,  1457     symbol.exec_stmt,  1458     symbol.assert_stmt,  1459     symbol.if_stmt,  1460     symbol.while_stmt,  1461     symbol.for_stmt,  1462     symbol.try_stmt,  1463     symbol.with_stmt,  1464     symbol.suite,  1465     symbol.testlist,  1466     symbol.testlist_safe,  1467     symbol.test,  1468     symbol.and_test,  1469     symbol.not_test,  1470     symbol.comparison,  1471     symbol.exprlist,  1472     symbol.expr,  1473     symbol.xor_expr,  1474     symbol.and_expr,  1475     symbol.shift_expr,  1476     symbol.arith_expr,  1477     symbol.term,  1478     symbol.factor,  1479     symbol.power,  1480     symbol.atom,  1481     symbol.yield_stmt,  1482     symbol.yield_expr,  1483     ]  1484   1485 _assign_types = [  1486     symbol.test,  1487     symbol.or_test,  1488     symbol.and_test,  1489     symbol.not_test,  1490     symbol.comparison,  1491     symbol.expr,  1492     symbol.xor_expr,  1493     symbol.and_expr,  1494     symbol.shift_expr,  1495     symbol.arith_expr,  1496     symbol.term,  1497     symbol.factor,  1498     ]  1499   1500 _names = {}  1501 for k, v in symbol.sym_name.items():  1502     _names[k] = v  1503 for k, v in token.tok_name.items():  1504     _names[k] = v  1505   1506 def debug_tree(tree):  1507     l = []  1508     for elt in tree:  1509         if isinstance(elt, int):  1510             l.append(_names.get(elt, elt))  1511         elif isinstance(elt, str):  1512             l.append(elt)  1513         else:  1514             l.append(debug_tree(elt))  1515     return l