1 """Parse tree transformation module. 2 3 Transforms Python source code into an abstract syntax tree (AST) 4 defined in the ast module. 5 6 The simplest ways to invoke this module are via parse and parseFile. 7 parse(buf) -> AST 8 parseFile(path) -> AST 9 """ 10 11 # Original version written by Greg Stein (gstein@lyra.org) 12 # and Bill Tutt (rassilon@lima.mudlib.org) 13 # February 1997. 14 # 15 # Modifications and improvements for Python 2.0 by Jeremy Hylton and 16 # Mark Hammond 17 # 18 # Some fixes to try to have correct line number on almost all nodes 19 # (except Module, Discard and Stmt) added by Sylvain Thenault 20 # 21 # Portions of this file are: 22 # Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. 23 # 24 # This module is provided under a BSD-ish license. See 25 # http://www.opensource.org/licenses/bsd-license.html 26 # and replace OWNER, ORGANIZATION, and YEAR as appropriate. 27 28 from compiler.ast import * 29 from pyparser.pygram import syms as symbol, sym_name, tokens as token, tok_name 30 import pyparser.pyparse as parser 31 32 class WalkerError(StandardError): 33 pass 34 35 from compiler.consts import CO_VARARGS, CO_VARKEYWORDS 36 from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY 37 38 def parseFile(path): 39 f = open(path, "U") 40 # XXX The parser API tolerates files without a trailing newline, 41 # but not strings without a trailing newline. Always add an extra 42 # newline to the file contents, since we're going through the string 43 # version of the API. 44 src = f.read() + "\n" 45 f.close() 46 return parse(src) 47 48 def parse(buf, mode="exec"): 49 if mode == "exec" or mode == "single": 50 return Transformer().parsesuite(buf) 51 elif mode == "eval": 52 return Transformer().parseexpr(buf) 53 else: 54 raise ValueError("compile() arg 3 must be" 55 " 'exec' or 'eval' or 'single'") 56 57 def extractLineNo(ast): 58 if not isinstance(ast[1], tuple): 59 # get a terminal node 60 return ast[2] 61 for child in ast[1:]: 62 if isinstance(child, tuple): 63 lineno = extractLineNo(child) 64 if lineno is not None: 65 return lineno 66 67 def Node(*args): 68 kind = args[0] 69 if kind in nodes: 70 try: 71 return nodes[kind](*args[1:]) 72 except TypeError: 73 print nodes[kind], len(args), args 74 raise 75 else: 76 raise WalkerError, "Can't find appropriate Node type: %s" % str(args) 77 #return apply(ast.Node, args) 78 79 class Transformer: 80 """Utility object for transforming Python parse trees. 81 82 Exposes the following methods: 83 tree = transform(ast_tree) 84 tree = parsesuite(text) 85 tree = parseexpr(text) 86 tree = parsefile(fileob | filename) 87 """ 88 89 def __init__(self): 90 self._dispatch = {} 91 for value, name in sym_name.items(): 92 if hasattr(self, name): 93 self._dispatch[value] = getattr(self, name) 94 self._dispatch[token["NEWLINE"]] = self.com_NEWLINE 95 self._atom_dispatch = {token["LPAR"]: self.atom_lpar, 96 token["LSQB"]: self.atom_lsqb, 97 token["LBRACE"]: self.atom_lbrace, 98 token["BACKQUOTE"]: self.atom_backquote, 99 token["NUMBER"]: self.atom_number, 100 token["STRING"]: self.atom_string, 101 token["NAME"]: self.atom_name, 102 } 103 self.encoding = None 104 105 def transform(self, tree): 106 """Transform an AST into a modified parse tree.""" 107 if not (isinstance(tree, tuple) or isinstance(tree, list)): 108 tree = parser.st2tuple(tree, line_info=1) 109 return self.compile_node(tree) 110 111 def parsesuite(self, text): 112 """Return a modified parse tree for the given suite text.""" 113 return self.transform(parser.suite(text)) 114 115 def parseexpr(self, text): 116 """Return a modified parse tree for the given expression text.""" 117 return self.transform(parser.expr(text)) 118 119 def parsefile(self, file): 120 """Return a modified parse tree for the contents of the given file.""" 121 if type(file) == type(''): 122 file = open(file) 123 return self.parsesuite(file.read()) 124 125 # -------------------------------------------------------------- 126 # 127 # PRIVATE METHODS 128 # 129 130 def compile_node(self, node): 131 ### emit a line-number node? 132 n = node[0] 133 134 if n == symbol["encoding_decl"]: 135 self.encoding = node[2] 136 node = node[1] 137 n = node[0] 138 139 if n == symbol["single_input"]: 140 return self.single_input(node[1:]) 141 if n == symbol["file_input"]: 142 return self.file_input(node[1:]) 143 if n == symbol["eval_input"]: 144 return self.eval_input(node[1:]) 145 if n == symbol["lambdef"]: 146 return self.lambdef(node[1:]) 147 if n == symbol["funcdef"]: 148 return self.funcdef(node[1:]) 149 if n == symbol["classdef"]: 150 return self.classdef(node[1:]) 151 152 raise WalkerError, ('unexpected node type', n) 153 154 def single_input(self, node): 155 ### do we want to do anything about being "interactive" ? 156 157 # NEWLINE | simple_stmt | compound_stmt NEWLINE 158 n = node[0][0] 159 if n != token["NEWLINE"]: 160 return self.com_stmt(node[0]) 161 162 return Pass() 163 164 def file_input(self, nodelist): 165 doc = self.get_docstring(nodelist, symbol["file_input"]) 166 if doc is not None: 167 i = 1 168 else: 169 i = 0 170 stmts = [] 171 for node in nodelist[i:]: 172 if node[0] != token["ENDMARKER"] and node[0] != token["NEWLINE"]: 173 self.com_append_stmt(stmts, node) 174 return Module(doc, Stmt(stmts)) 175 176 def eval_input(self, nodelist): 177 # from the built-in function input() 178 ### is this sufficient? 179 return Expression(self.com_node(nodelist[0])) 180 181 def funcdef(self, nodelist): 182 # -5 -4 -3 -2 -1 183 # funcdef: 'def' NAME parameters ':' suite 184 # parameters: '(' [varargslist] ')' 185 186 assert len(nodelist) == 5 187 decorators = None 188 189 lineno = nodelist[-4][2] 190 name = nodelist[-4][1] 191 args = nodelist[-3][2] 192 193 if args[0] == symbol["varargslist"]: 194 names, defaults, flags = self.com_arglist(args[1:]) 195 else: 196 names = defaults = () 197 flags = 0 198 doc = self.get_docstring(nodelist[-1]) 199 200 # code for function 201 code = self.com_node(nodelist[-1]) 202 203 if doc is not None: 204 assert isinstance(code, Stmt) 205 assert isinstance(code.nodes[0], Discard) 206 del code.nodes[0] 207 return Function(decorators, name, names, defaults, flags, doc, code, 208 lineno=lineno) 209 210 def lambdef(self, nodelist): 211 # lambdef: 'lambda' [varargslist] ':' test 212 if nodelist[2][0] == symbol["varargslist"]: 213 names, defaults, flags = self.com_arglist(nodelist[2][1:]) 214 else: 215 names = defaults = () 216 flags = 0 217 218 # code for lambda 219 code = self.com_node(nodelist[-1]) 220 221 return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) 222 old_lambdef = lambdef 223 224 def classdef(self, nodelist): 225 # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 226 227 name = nodelist[1][1] 228 doc = self.get_docstring(nodelist[-1]) 229 if nodelist[2][0] == token["COLON"]: 230 bases = [] 231 elif nodelist[3][0] == token["RPAR"]: 232 bases = [] 233 else: 234 bases = self.com_bases(nodelist[3]) 235 236 # code for class 237 code = self.com_node(nodelist[-1]) 238 239 if doc is not None: 240 assert isinstance(code, Stmt) 241 assert isinstance(code.nodes[0], Discard) 242 del code.nodes[0] 243 244 return Class(name, bases, doc, code, lineno=nodelist[1][2]) 245 246 def stmt(self, nodelist): 247 return self.com_stmt(nodelist[0]) 248 249 small_stmt = stmt 250 flow_stmt = stmt 251 compound_stmt = stmt 252 253 def simple_stmt(self, nodelist): 254 # small_stmt (';' small_stmt)* [';'] NEWLINE 255 stmts = [] 256 for i in range(0, len(nodelist), 2): 257 self.com_append_stmt(stmts, nodelist[i]) 258 return Stmt(stmts) 259 260 def parameters(self, nodelist): 261 raise WalkerError 262 263 def varargslist(self, nodelist): 264 raise WalkerError 265 266 def fpdef(self, nodelist): 267 raise WalkerError 268 269 def fplist(self, nodelist): 270 raise WalkerError 271 272 def dotted_name(self, nodelist): 273 raise WalkerError 274 275 def comp_op(self, nodelist): 276 raise WalkerError 277 278 def trailer(self, nodelist): 279 raise WalkerError 280 281 def sliceop(self, nodelist): 282 raise WalkerError 283 284 def argument(self, nodelist): 285 raise WalkerError 286 287 # -------------------------------------------------------------- 288 # 289 # STATEMENT NODES (invoked by com_node()) 290 # 291 292 def expr_stmt(self, nodelist): 293 # augassign testlist | testlist ('=' testlist)* 294 en = nodelist[-1] 295 exprNode = self.lookup_node(en)(en[1:]) 296 if len(nodelist) == 1: 297 return Discard(exprNode, lineno=exprNode.lineno) 298 if nodelist[1][0] == token["EQUAL"]: 299 nodesl = [] 300 for i in range(0, len(nodelist) - 2, 2): 301 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) 302 return Assign(nodesl, exprNode, lineno=nodelist[1][2]) 303 else: 304 lval = self.com_augassign(nodelist[0]) 305 op = self.com_augassign_op(nodelist[1]) 306 return AugAssign(lval, op[1], exprNode, lineno=op[2]) 307 raise WalkerError, "can't get here" 308 309 def print_stmt(self, nodelist): 310 # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) 311 items = [] 312 if len(nodelist) == 1: 313 start = 1 314 dest = None 315 elif nodelist[1][0] == token["RIGHTSHIFT"]: 316 assert len(nodelist) == 3 \ 317 or nodelist[3][0] == token["COMMA"] 318 dest = self.com_node(nodelist[2]) 319 start = 4 320 else: 321 dest = None 322 start = 1 323 for i in range(start, len(nodelist), 2): 324 items.append(self.com_node(nodelist[i])) 325 if nodelist[-1][0] == token["COMMA"]: 326 return Print(items, dest, lineno=nodelist[0][2]) 327 return Printnl(items, dest, lineno=nodelist[0][2]) 328 329 def del_stmt(self, nodelist): 330 return self.com_assign(nodelist[1], OP_DELETE) 331 332 def pass_stmt(self, nodelist): 333 return Pass(lineno=nodelist[0][2]) 334 335 def break_stmt(self, nodelist): 336 return Break(lineno=nodelist[0][2]) 337 338 def continue_stmt(self, nodelist): 339 return Continue(lineno=nodelist[0][2]) 340 341 def return_stmt(self, nodelist): 342 # return: [testlist] 343 if len(nodelist) < 2: 344 return Return(Const(None), lineno=nodelist[0][2]) 345 return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) 346 347 def raise_stmt(self, nodelist): 348 # raise: [test [',' test [',' test]]] 349 if len(nodelist) > 5: 350 expr3 = self.com_node(nodelist[5]) 351 else: 352 expr3 = None 353 if len(nodelist) > 3: 354 expr2 = self.com_node(nodelist[3]) 355 else: 356 expr2 = None 357 if len(nodelist) > 1: 358 expr1 = self.com_node(nodelist[1]) 359 else: 360 expr1 = None 361 return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) 362 363 def import_stmt(self, nodelist): 364 # import_stmt: import_name | import_from 365 assert len(nodelist) == 1 366 return self.com_node(nodelist[0]) 367 368 def import_name(self, nodelist): 369 # import_name: 'import' dotted_as_names 370 return Import(self.com_dotted_as_names(nodelist[1]), 371 lineno=nodelist[0][2]) 372 373 def import_from(self, nodelist): 374 # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | 375 # '(' import_as_names ')' | import_as_names) 376 assert nodelist[0][1] == 'from' 377 idx = 1 378 while nodelist[idx][1] == '.': 379 idx += 1 380 level = idx - 1 381 if nodelist[idx][0] == symbol["dotted_name"]: 382 fromname = self.com_dotted_name(nodelist[idx]) 383 idx += 1 384 else: 385 fromname = "" 386 assert nodelist[idx][1] == 'import' 387 if nodelist[idx + 1][0] == token["STAR"]: 388 return From(fromname, [('*', None)], level, 389 lineno=nodelist[0][2]) 390 else: 391 node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token["LPAR"])] 392 return From(fromname, self.com_import_as_names(node), level, 393 lineno=nodelist[0][2]) 394 395 def global_stmt(self, nodelist): 396 # global: NAME (',' NAME)* 397 names = [] 398 for i in range(1, len(nodelist), 2): 399 names.append(nodelist[i][1]) 400 return Global(names, lineno=nodelist[0][2]) 401 402 def exec_stmt(self, nodelist): 403 # exec_stmt: 'exec' expr ['in' expr [',' expr]] 404 expr1 = self.com_node(nodelist[1]) 405 if len(nodelist) >= 4: 406 expr2 = self.com_node(nodelist[3]) 407 if len(nodelist) >= 6: 408 expr3 = self.com_node(nodelist[5]) 409 else: 410 expr3 = None 411 else: 412 expr2 = expr3 = None 413 414 return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) 415 416 def assert_stmt(self, nodelist): 417 # 'assert': test, [',' test] 418 expr1 = self.com_node(nodelist[1]) 419 if (len(nodelist) == 4): 420 expr2 = self.com_node(nodelist[3]) 421 else: 422 expr2 = None 423 return Assert(expr1, expr2, lineno=nodelist[0][2]) 424 425 def if_stmt(self, nodelist): 426 # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 427 tests = [] 428 for i in range(0, len(nodelist) - 3, 4): 429 testNode = self.com_node(nodelist[i + 1]) 430 suiteNode = self.com_node(nodelist[i + 3]) 431 tests.append((testNode, suiteNode)) 432 433 if len(nodelist) % 4 == 3: 434 elseNode = self.com_node(nodelist[-1]) 435 ## elseNode.lineno = nodelist[-1][1][2] 436 else: 437 elseNode = None 438 return If(tests, elseNode, lineno=nodelist[0][2]) 439 440 def while_stmt(self, nodelist): 441 # 'while' test ':' suite ['else' ':' suite] 442 443 testNode = self.com_node(nodelist[1]) 444 bodyNode = self.com_node(nodelist[3]) 445 446 if len(nodelist) > 4: 447 elseNode = self.com_node(nodelist[6]) 448 else: 449 elseNode = None 450 451 return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) 452 453 def for_stmt(self, nodelist): 454 # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] 455 456 assignNode = self.com_assign(nodelist[1], OP_ASSIGN) 457 listNode = self.com_node(nodelist[3]) 458 bodyNode = self.com_node(nodelist[5]) 459 460 if len(nodelist) > 8: 461 elseNode = self.com_node(nodelist[8]) 462 else: 463 elseNode = None 464 465 return For(assignNode, listNode, bodyNode, elseNode, 466 lineno=nodelist[0][2]) 467 468 def try_stmt(self, nodelist): 469 return self.com_try_except_finally(nodelist) 470 471 def with_stmt(self, nodelist): 472 return self.com_with(nodelist) 473 474 def suite(self, nodelist): 475 # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT 476 if len(nodelist) == 1: 477 return self.com_stmt(nodelist[0]) 478 479 stmts = [] 480 for node in nodelist: 481 if node[0] == symbol["stmt"]: 482 self.com_append_stmt(stmts, node) 483 return Stmt(stmts) 484 485 # -------------------------------------------------------------- 486 # 487 # EXPRESSION NODES (invoked by com_node()) 488 # 489 490 def testlist(self, nodelist): 491 # testlist: expr (',' expr)* [','] 492 # testlist_safe: test [(',' test)+ [',']] 493 # exprlist: expr (',' expr)* [','] 494 return self.com_binary(Tuple, nodelist) 495 496 testlist_safe = testlist # XXX 497 testlist1 = testlist 498 exprlist = testlist 499 500 def testlist_comp(self, nodelist): 501 # test ( (',' test)* [','] ) 502 assert nodelist[0][0] == symbol["test"] 503 return self.testlist(nodelist) 504 505 def test(self, nodelist): 506 # or_test ['if' or_test 'else' test] | lambdef 507 if len(nodelist) == 1 and nodelist[0][0] == symbol["lambdef"]: 508 return self.lambdef(nodelist[0]) 509 then = self.com_node(nodelist[0]) 510 if len(nodelist) > 1: 511 assert len(nodelist) == 5 512 assert nodelist[1][1] == 'if' 513 assert nodelist[3][1] == 'else' 514 test = self.com_node(nodelist[2]) 515 else_ = self.com_node(nodelist[4]) 516 return IfExp(test, then, else_, lineno=nodelist[1][2]) 517 return then 518 519 def or_test(self, nodelist): 520 # and_test ('or' and_test)* | lambdef 521 if len(nodelist) == 1 and nodelist[0][0] == symbol["lambdef"]: 522 return self.lambdef(nodelist[0]) 523 return self.com_binary(Or, nodelist) 524 old_test = or_test 525 526 def and_test(self, nodelist): 527 # not_test ('and' not_test)* 528 return self.com_binary(And, nodelist) 529 530 def not_test(self, nodelist): 531 # 'not' not_test | comparison 532 result = self.com_node(nodelist[-1]) 533 if len(nodelist) == 2: 534 return Not(result, lineno=nodelist[0][2]) 535 return result 536 537 def comparison(self, nodelist): 538 # comparison: expr (comp_op expr)* 539 node = self.com_node(nodelist[0]) 540 if len(nodelist) == 1: 541 return node 542 543 results = [] 544 for i in range(2, len(nodelist), 2): 545 nl = nodelist[i-1] 546 547 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 548 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 549 n = nl[1] 550 if n[0] == token["NAME"]: 551 type = n[1] 552 if len(nl) == 3: 553 if type == 'not': 554 type = 'not in' 555 else: 556 type = 'is not' 557 else: 558 type = _cmp_types[n[0]] 559 560 lineno = nl[1][2] 561 results.append((type, self.com_node(nodelist[i]))) 562 563 # we need a special "compare" node so that we can distinguish 564 # 3 < x < 5 from (3 < x) < 5 565 # the two have very different semantics and results (note that the 566 # latter form is always true) 567 568 return Compare(node, results, lineno=lineno) 569 570 def expr(self, nodelist): 571 # xor_expr ('|' xor_expr)* 572 return self.com_binary(Bitor, nodelist) 573 574 def xor_expr(self, nodelist): 575 # xor_expr ('^' xor_expr)* 576 return self.com_binary(Bitxor, nodelist) 577 578 def and_expr(self, nodelist): 579 # xor_expr ('&' xor_expr)* 580 return self.com_binary(Bitand, nodelist) 581 582 def shift_expr(self, nodelist): 583 # shift_expr ('<<'|'>>' shift_expr)* 584 node = self.com_node(nodelist[0]) 585 for i in range(2, len(nodelist), 2): 586 right = self.com_node(nodelist[i]) 587 if nodelist[i-1][0] == token["LEFTSHIFT"]: 588 node = LeftShift([node, right], lineno=nodelist[1][2]) 589 elif nodelist[i-1][0] == token["RIGHTSHIFT"]: 590 node = RightShift([node, right], lineno=nodelist[1][2]) 591 else: 592 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 593 return node 594 595 def arith_expr(self, nodelist): 596 node = self.com_node(nodelist[0]) 597 for i in range(2, len(nodelist), 2): 598 right = self.com_node(nodelist[i]) 599 if nodelist[i-1][0] == token["PLUS"]: 600 node = Add([node, right], lineno=nodelist[1][2]) 601 elif nodelist[i-1][0] == token["MINUS"]: 602 node = Sub([node, right], lineno=nodelist[1][2]) 603 else: 604 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 605 return node 606 607 def term(self, nodelist): 608 node = self.com_node(nodelist[0]) 609 for i in range(2, len(nodelist), 2): 610 right = self.com_node(nodelist[i]) 611 t = nodelist[i-1][0] 612 if t == token["STAR"]: 613 node = Mul([node, right]) 614 elif t == token["SLASH"]: 615 node = Div([node, right]) 616 elif t == token["PERCENT"]: 617 node = Mod([node, right]) 618 elif t == token["DOUBLESLASH"]: 619 node = FloorDiv([node, right]) 620 else: 621 raise ValueError, "unexpected token: %s" % t 622 node.lineno = nodelist[1][2] 623 return node 624 625 def factor(self, nodelist): 626 elt = nodelist[0] 627 t = elt[0] 628 node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) 629 # need to handle (unary op)constant here... 630 if t == token["PLUS"]: 631 return UnaryAdd(node, lineno=elt[2]) 632 elif t == token["MINUS"]: 633 return UnarySub(node, lineno=elt[2]) 634 elif t == token["TILDE"]: 635 node = Invert(node, lineno=elt[2]) 636 return node 637 638 def power(self, nodelist): 639 # power: atom trailer* ('**' factor)* 640 node = self.com_node(nodelist[0]) 641 for i in range(1, len(nodelist)): 642 elt = nodelist[i] 643 if elt[0] == token["DOUBLESTAR"]: 644 return Power([node, self.com_node(nodelist[i+1])], 645 lineno=elt[2]) 646 647 node = self.com_apply_trailer(node, elt) 648 649 return node 650 651 def atom(self, nodelist): 652 return self._atom_dispatch[nodelist[0][0]](nodelist) 653 654 def atom_lpar(self, nodelist): 655 if nodelist[1][0] == token["RPAR"]: 656 return Tuple((), lineno=nodelist[0][2]) 657 return self.com_node(nodelist[1]) 658 659 def atom_lsqb(self, nodelist): 660 if nodelist[1][0] == token["RSQB"]: 661 return List((), lineno=nodelist[0][2]) 662 return self.com_list_constructor(nodelist[1]) 663 664 def atom_lbrace(self, nodelist): 665 if nodelist[1][0] == token["RBRACE"]: 666 return Dict((), lineno=nodelist[0][2]) 667 return self.com_dictorsetmaker(nodelist[1]) 668 669 def atom_backquote(self, nodelist): 670 return Backquote(self.com_node(nodelist[1])) 671 672 def atom_number(self, nodelist): 673 ### need to verify this matches compile.c 674 k = eval(nodelist[0][1]) 675 return Const(k, nodelist[0][1], lineno=nodelist[0][2]) 676 677 def decode_literal(self, lit): 678 if self.encoding: 679 # this is particularly fragile & a bit of a 680 # hack... changes in compile.c:parsestr and 681 # tokenizer.c must be reflected here. 682 if self.encoding not in ['utf-8', 'iso-8859-1']: 683 lit = unicode(lit, 'utf-8').encode(self.encoding) 684 return eval("# coding: %s\n%s" % (self.encoding, lit)) 685 else: 686 return eval(lit) 687 688 def atom_string(self, nodelist): 689 k = '' 690 for node in nodelist: 691 k += self.decode_literal(node[1]) 692 return Const(k, node[1], lineno=nodelist[0][2]) 693 694 def atom_name(self, nodelist): 695 return Name(nodelist[0][1], lineno=nodelist[0][2]) 696 697 # -------------------------------------------------------------- 698 # 699 # INTERNAL PARSING UTILITIES 700 # 701 702 # The use of com_node() introduces a lot of extra stack frames, 703 # enough to cause a stack overflow compiling test.test_parser with 704 # the standard interpreter recursionlimit. The com_node() is a 705 # convenience function that hides the dispatch details, but comes 706 # at a very high cost. It is more efficient to dispatch directly 707 # in the callers. In these cases, use lookup_node() and call the 708 # dispatched node directly. 709 710 def lookup_node(self, node): 711 return self._dispatch[node[0]] 712 713 def com_node(self, node): 714 # Note: compile.c has handling in com_node for del_stmt, pass_stmt, 715 # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, 716 # and compound_stmt. 717 # We'll just dispatch them. 718 return self._dispatch[node[0]](node[1:]) 719 720 def com_NEWLINE(self, *args): 721 # A ';' at the end of a line can make a NEWLINE token appear 722 # here, Render it harmless. (genc discards ('discard', 723 # ('const', xxxx)) Nodes) 724 return Discard(Const(None)) 725 726 def com_arglist(self, nodelist): 727 # varargslist: 728 # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) 729 # | fpdef ['=' test] (',' fpdef ['=' test])* [','] 730 # fpdef: NAME | '(' fplist ')' 731 # fplist: fpdef (',' fpdef)* [','] 732 names = [] 733 defaults = [] 734 flags = 0 735 736 i = 0 737 while i < len(nodelist): 738 node = nodelist[i] 739 if node[0] == token["STAR"] or node[0] == token["DOUBLESTAR"]: 740 if node[0] == token["STAR"]: 741 node = nodelist[i+1] 742 if node[0] == token["NAME"]: 743 names.append(node[1]) 744 flags = flags | CO_VARARGS 745 i = i + 3 746 747 if i < len(nodelist): 748 # should be DOUBLESTAR 749 t = nodelist[i][0] 750 if t == token["DOUBLESTAR"]: 751 node = nodelist[i+1] 752 else: 753 raise ValueError, "unexpected token: %s" % t 754 names.append(node[1]) 755 flags = flags | CO_VARKEYWORDS 756 757 break 758 759 # fpdef: NAME | '(' fplist ')' 760 names.append(self.com_fpdef(node)) 761 762 i = i + 1 763 if i < len(nodelist) and nodelist[i][0] == token["EQUAL"]: 764 defaults.append(self.com_node(nodelist[i + 1])) 765 i = i + 2 766 elif len(defaults): 767 # we have already seen an argument with default, but here 768 # came one without 769 raise SyntaxError, "non-default argument follows default argument" 770 771 # skip the comma 772 i = i + 1 773 774 return names, defaults, flags 775 776 def com_fpdef(self, node): 777 # fpdef: NAME | '(' fplist ')' 778 if node[1][0] == token["LPAR"]: 779 return self.com_fplist(node[2]) 780 return node[1][1] 781 782 def com_fplist(self, node): 783 # fplist: fpdef (',' fpdef)* [','] 784 if len(node) == 2: 785 return self.com_fpdef(node[1]) 786 list = [] 787 for i in range(1, len(node), 2): 788 list.append(self.com_fpdef(node[i])) 789 return tuple(list) 790 791 def com_dotted_name(self, node): 792 # String together the dotted names and return the string 793 name = "" 794 for n in node: 795 if type(n) == type(()) and n[0] == 1: 796 name = name + n[1] + '.' 797 return name[:-1] 798 799 def com_dotted_as_name(self, node): 800 assert node[0] == symbol["dotted_as_name"] 801 node = node[1:] 802 dot = self.com_dotted_name(node[0][1:]) 803 if len(node) == 1: 804 return dot, None 805 assert node[1][1] == 'as' 806 assert node[2][0] == token["NAME"] 807 return dot, node[2][1] 808 809 def com_dotted_as_names(self, node): 810 assert node[0] == symbol["dotted_as_names"] 811 node = node[1:] 812 names = [self.com_dotted_as_name(node[0])] 813 for i in range(2, len(node), 2): 814 names.append(self.com_dotted_as_name(node[i])) 815 return names 816 817 def com_import_as_name(self, node): 818 assert node[0] == symbol["import_as_name"] 819 node = node[1:] 820 assert node[0][0] == token["NAME"] 821 if len(node) == 1: 822 return node[0][1], None 823 assert node[1][1] == 'as', node 824 assert node[2][0] == token["NAME"] 825 return node[0][1], node[2][1] 826 827 def com_import_as_names(self, node): 828 assert node[0] == symbol["import_as_names"] 829 node = node[1:] 830 names = [self.com_import_as_name(node[0])] 831 for i in range(2, len(node), 2): 832 names.append(self.com_import_as_name(node[i])) 833 return names 834 835 def com_bases(self, node): 836 bases = [] 837 for i in range(1, len(node), 2): 838 bases.append(self.com_node(node[i])) 839 return bases 840 841 def com_try_except_finally(self, nodelist): 842 # ('try' ':' suite 843 # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] 844 # | 'finally' ':' suite)) 845 846 if nodelist[3][0] == token["NAME"]: 847 # first clause is a finally clause: only try-finally 848 return TryFinally(self.com_node(nodelist[2]), 849 self.com_node(nodelist[5]), 850 lineno=nodelist[0][2]) 851 852 #tryexcept: [TryNode, [except_clauses], elseNode)] 853 clauses = [] 854 elseNode = None 855 finallyNode = None 856 for i in range(3, len(nodelist), 3): 857 node = nodelist[i] 858 if node[0] == symbol["except_clause"]: 859 # except_clause: 'except' [expr [(',' | 'as') expr]] */ 860 if len(node) > 2: 861 expr1 = self.com_node(node[2]) 862 if len(node) > 4: 863 expr2 = self.com_assign(node[4], OP_ASSIGN) 864 else: 865 expr2 = None 866 else: 867 expr1 = expr2 = None 868 clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) 869 870 if node[0] == token["NAME"]: 871 if node[1] == 'else': 872 elseNode = self.com_node(nodelist[i+2]) 873 elif node[1] == 'finally': 874 finallyNode = self.com_node(nodelist[i+2]) 875 try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, 876 lineno=nodelist[0][2]) 877 if finallyNode: 878 return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) 879 else: 880 return try_except 881 882 def com_with(self, nodelist): 883 # with_stmt: 'with' with_item (',' with_item)* ':' suite 884 body = self.com_node(nodelist[-1]) 885 for i in range(len(nodelist) - 3, 0, -2): 886 ret = self.com_with_item(nodelist[i], body, nodelist[0][2]) 887 if i == 1: 888 return ret 889 body = ret 890 891 def com_with_item(self, nodelist, body, lineno): 892 # with_item: test ['as' expr] 893 if len(nodelist) == 4: 894 var = self.com_assign(nodelist[3], OP_ASSIGN) 895 else: 896 var = None 897 expr = self.com_node(nodelist[1]) 898 return With(expr, var, body, lineno=lineno) 899 900 def com_augassign_op(self, node): 901 assert node[0] == symbol["augassign"] 902 return node[1] 903 904 def com_augassign(self, node): 905 """Return node suitable for lvalue of augmented assignment 906 907 Names, slices, and attributes are the only allowable nodes. 908 """ 909 l = self.com_node(node) 910 if l.__class__ in (Name, Slice, Subscript, Getattr): 911 return l 912 raise SyntaxError, "can't assign to %s" % l.__class__.__name__ 913 914 def com_assign(self, node, assigning): 915 # return a node suitable for use as an "lvalue" 916 # loop to avoid trivial recursion 917 while 1: 918 t = node[0] 919 if t in (symbol["exprlist"], symbol["testlist"], symbol["testlist_safe"], symbol["testlist_comp"]): 920 if len(node) > 2: 921 return self.com_assign_tuple(node, assigning) 922 node = node[1] 923 elif t in _assign_types: 924 if len(node) > 2: 925 raise SyntaxError, "can't assign to operator" 926 node = node[1] 927 elif t == symbol["power"]: 928 if node[1][0] != symbol["atom"]: 929 raise SyntaxError, "can't assign to operator" 930 if len(node) > 2: 931 primary = self.com_node(node[1]) 932 for i in range(2, len(node)-1): 933 ch = node[i] 934 if ch[0] == token["DOUBLESTAR"]: 935 raise SyntaxError, "can't assign to operator" 936 primary = self.com_apply_trailer(primary, ch) 937 return self.com_assign_trailer(primary, node[-1], 938 assigning) 939 node = node[1] 940 elif t == symbol["atom"]: 941 t = node[1][0] 942 if t == token["LPAR"]: 943 node = node[2] 944 if node[0] == token["RPAR"]: 945 raise SyntaxError, "can't assign to ()" 946 elif t == token["LSQB"]: 947 node = node[2] 948 if node[0] == token["RSQB"]: 949 raise SyntaxError, "can't assign to []" 950 return self.com_assign_list(node, assigning) 951 elif t == token["NAME"]: 952 return self.com_assign_name(node[1], assigning) 953 else: 954 raise SyntaxError, "can't assign to literal" 955 else: 956 raise SyntaxError, "bad assignment (%s)" % t 957 958 def com_assign_tuple(self, node, assigning): 959 assigns = [] 960 for i in range(1, len(node), 2): 961 assigns.append(self.com_assign(node[i], assigning)) 962 return AssTuple(assigns, lineno=extractLineNo(node)) 963 964 def com_assign_list(self, node, assigning): 965 assigns = [] 966 for i in range(1, len(node), 2): 967 if i + 1 < len(node): 968 if node[i + 1][0] == symbol["list_for"]: 969 raise SyntaxError, "can't assign to list comprehension" 970 assert node[i + 1][0] == token["COMMA"], node[i + 1] 971 assigns.append(self.com_assign(node[i], assigning)) 972 return AssList(assigns, lineno=extractLineNo(node)) 973 974 def com_assign_name(self, node, assigning): 975 return AssName(node[1], assigning, lineno=node[2]) 976 977 def com_assign_trailer(self, primary, node, assigning): 978 t = node[1][0] 979 if t == token["DOT"]: 980 return self.com_assign_attr(primary, node[2], assigning) 981 if t == token["LSQB"]: 982 return self.com_subscriptlist(primary, node[2], assigning) 983 if t == token["LPAR"]: 984 raise SyntaxError, "can't assign to function call" 985 raise SyntaxError, "unknown trailer type: %s" % t 986 987 def com_assign_attr(self, primary, node, assigning): 988 return AssAttr(primary, node[1], assigning, lineno=node[-1]) 989 990 def com_binary(self, constructor, nodelist): 991 "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." 992 l = len(nodelist) 993 if l == 1: 994 n = nodelist[0] 995 return self.lookup_node(n)(n[1:]) 996 items = [] 997 for i in range(0, l, 2): 998 n = nodelist[i] 999 items.append(self.lookup_node(n)(n[1:])) 1000 return constructor(items, lineno=extractLineNo(nodelist)) 1001 1002 def com_stmt(self, node): 1003 result = self.lookup_node(node)(node[1:]) 1004 assert result is not None 1005 if isinstance(result, Stmt): 1006 return result 1007 return Stmt([result]) 1008 1009 def com_append_stmt(self, stmts, node): 1010 result = self.lookup_node(node)(node[1:]) 1011 assert result is not None 1012 if isinstance(result, Stmt): 1013 stmts.extend(result.nodes) 1014 else: 1015 stmts.append(result) 1016 1017 def com_list_constructor(self, nodelist): 1018 # listmaker: test ( (',' test)* [','] ) 1019 values = [] 1020 for i in range(1, len(nodelist)): 1021 if nodelist[i][0] == token["COMMA"]: 1022 continue 1023 values.append(self.com_node(nodelist[i])) 1024 return List(values, lineno=values[0].lineno) 1025 1026 def com_dictorsetmaker(self, nodelist): 1027 # dictorsetmaker: ( (test ':' test ( (',' test ':' test)* [','])) | 1028 # (test ( (',' test)* [','])) ) 1029 assert nodelist[0] == symbol["dictorsetmaker"] 1030 nodelist = nodelist[1:] 1031 if len(nodelist) == 1 or nodelist[1][0] == token["COMMA"]: 1032 # set literal 1033 items = [] 1034 for i in range(0, len(nodelist), 2): 1035 items.append(self.com_node(nodelist[i])) 1036 return Set(items, lineno=items[0].lineno) 1037 else: 1038 # dict literal 1039 items = [] 1040 for i in range(0, len(nodelist), 4): 1041 items.append((self.com_node(nodelist[i]), 1042 self.com_node(nodelist[i+2]))) 1043 return Dict(items, lineno=items[0][0].lineno) 1044 1045 def com_apply_trailer(self, primaryNode, nodelist): 1046 t = nodelist[1][0] 1047 if t == token["LPAR"]: 1048 return self.com_call_function(primaryNode, nodelist[2]) 1049 if t == token["DOT"]: 1050 return self.com_select_member(primaryNode, nodelist[2]) 1051 if t == token["LSQB"]: 1052 return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) 1053 1054 raise SyntaxError, 'unknown node type: %s' % t 1055 1056 def com_select_member(self, primaryNode, nodelist): 1057 if nodelist[0] != token["NAME"]: 1058 raise SyntaxError, "member must be a name" 1059 return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) 1060 1061 def com_call_function(self, primaryNode, nodelist): 1062 if nodelist[0] == token["RPAR"]: 1063 return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) 1064 args = [] 1065 kw = 0 1066 star_node = dstar_node = None 1067 len_nodelist = len(nodelist) 1068 i = 1 1069 while i < len_nodelist: 1070 node = nodelist[i] 1071 1072 if node[0]==token["STAR"]: 1073 if star_node is not None: 1074 raise SyntaxError, 'already have the varargs indentifier' 1075 star_node = self.com_node(nodelist[i+1]) 1076 i = i + 3 1077 continue 1078 elif node[0]==token["DOUBLESTAR"]: 1079 if dstar_node is not None: 1080 raise SyntaxError, 'already have the kwargs indentifier' 1081 dstar_node = self.com_node(nodelist[i+1]) 1082 i = i + 3 1083 continue 1084 1085 # positional or named parameters 1086 kw, result = self.com_argument(node, kw, star_node) 1087 1088 args.append(result) 1089 i = i + 2 1090 1091 return CallFunc(primaryNode, args, star_node, dstar_node, 1092 lineno=extractLineNo(nodelist)) 1093 1094 def com_argument(self, nodelist, kw, star_node): 1095 if len(nodelist) == 2: 1096 if kw: 1097 raise SyntaxError, "non-keyword arg after keyword arg" 1098 if star_node: 1099 raise SyntaxError, "only named arguments may follow *expression" 1100 return 0, self.com_node(nodelist[1]) 1101 result = self.com_node(nodelist[3]) 1102 n = nodelist[1] 1103 while len(n) == 2 and n[0] != token["NAME"]: 1104 n = n[1] 1105 if n[0] != token["NAME"]: 1106 raise SyntaxError, "keyword can't be an expression (%s)"%n[0] 1107 node = Keyword(n[1], result, lineno=n[2]) 1108 return 1, node 1109 1110 def com_subscriptlist(self, primary, nodelist, assigning): 1111 # slicing: simple_slicing | extended_slicing 1112 # simple_slicing: primary "[" short_slice "]" 1113 # extended_slicing: primary "[" slice_list "]" 1114 # slice_list: slice_item ("," slice_item)* [","] 1115 1116 # backwards compat slice for '[i:j]' 1117 if len(nodelist) == 2: 1118 sub = nodelist[1] 1119 if (sub[1][0] == token["COLON"] or \ 1120 (len(sub) > 2 and sub[2][0] == token["COLON"])) and \ 1121 sub[-1][0] != symbol["sliceop"]: 1122 return self.com_slice(primary, sub, assigning) 1123 1124 subscripts = [] 1125 for i in range(1, len(nodelist), 2): 1126 subscripts.append(self.com_subscript(nodelist[i])) 1127 return Subscript(primary, assigning, subscripts, 1128 lineno=extractLineNo(nodelist)) 1129 1130 def com_subscript(self, node): 1131 # slice_item: expression | proper_slice | ellipsis 1132 ch = node[1] 1133 t = ch[0] 1134 if t == token["DOT"] and node[2][0] == token["DOT"]: 1135 return Ellipsis() 1136 if t == token["COLON"] or len(node) > 2: 1137 return self.com_sliceobj(node) 1138 return self.com_node(ch) 1139 1140 def com_sliceobj(self, node): 1141 # proper_slice: short_slice | long_slice 1142 # short_slice: [lower_bound] ":" [upper_bound] 1143 # long_slice: short_slice ":" [stride] 1144 # lower_bound: expression 1145 # upper_bound: expression 1146 # stride: expression 1147 # 1148 # Note: a stride may be further slicing... 1149 1150 items = [] 1151 1152 if node[1][0] == token["COLON"]: 1153 items.append(Const(None)) 1154 i = 2 1155 else: 1156 items.append(self.com_node(node[1])) 1157 # i == 2 is a COLON 1158 i = 3 1159 1160 if i < len(node) and node[i][0] == symbol["test"]: 1161 items.append(self.com_node(node[i])) 1162 i = i + 1 1163 else: 1164 items.append(Const(None)) 1165 1166 # a short_slice has been built. look for long_slice now by looking 1167 # for strides... 1168 for j in range(i, len(node)): 1169 ch = node[j] 1170 if len(ch) == 2: 1171 items.append(Const(None)) 1172 else: 1173 items.append(self.com_node(ch[2])) 1174 return Sliceobj(items, lineno=extractLineNo(node)) 1175 1176 def com_slice(self, primary, node, assigning): 1177 # short_slice: [lower_bound] ":" [upper_bound] 1178 lower = upper = None 1179 if len(node) == 3: 1180 if node[1][0] == token["COLON"]: 1181 upper = self.com_node(node[2]) 1182 else: 1183 lower = self.com_node(node[1]) 1184 elif len(node) == 4: 1185 lower = self.com_node(node[1]) 1186 upper = self.com_node(node[3]) 1187 return Slice(primary, assigning, lower, upper, 1188 lineno=extractLineNo(node)) 1189 1190 def get_docstring(self, node, n=None): 1191 if n is None: 1192 n = node[0] 1193 node = node[1:] 1194 if n == symbol["suite"]: 1195 if len(node) == 1: 1196 return self.get_docstring(node[0]) 1197 for sub in node: 1198 if sub[0] == symbol["stmt"]: 1199 return self.get_docstring(sub) 1200 return None 1201 if n == symbol["file_input"]: 1202 for sub in node: 1203 if sub[0] == symbol["stmt"]: 1204 return self.get_docstring(sub) 1205 return None 1206 if n == symbol["atom"]: 1207 if node[0][0] == token["STRING"]: 1208 s = '' 1209 for t in node: 1210 s = s + eval(t[1]) 1211 return s 1212 return None 1213 if n == symbol["stmt"] or n == symbol["simple_stmt"] \ 1214 or n == symbol["small_stmt"]: 1215 return self.get_docstring(node[0]) 1216 if n in _doc_nodes and len(node) == 1: 1217 return self.get_docstring(node[0]) 1218 return None 1219 1220 1221 _doc_nodes = [ 1222 symbol["expr_stmt"], 1223 symbol["testlist"], 1224 symbol["testlist_safe"], 1225 symbol["test"], 1226 symbol["or_test"], 1227 symbol["and_test"], 1228 symbol["not_test"], 1229 symbol["comparison"], 1230 symbol["expr"], 1231 symbol["xor_expr"], 1232 symbol["and_expr"], 1233 symbol["shift_expr"], 1234 symbol["arith_expr"], 1235 symbol["term"], 1236 symbol["factor"], 1237 symbol["power"], 1238 ] 1239 1240 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 1241 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 1242 _cmp_types = { 1243 token["LESS"] : '<', 1244 token["GREATER"] : '>', 1245 token["EQEQUAL"] : '==', 1246 token["EQUAL"] : '==', 1247 token["LESSEQUAL"] : '<=', 1248 token["GREATEREQUAL"] : '>=', 1249 token["NOTEQUAL"] : '!=', 1250 } 1251 1252 _legal_node_types = [ 1253 symbol["funcdef"], 1254 symbol["classdef"], 1255 symbol["stmt"], 1256 symbol["small_stmt"], 1257 symbol["flow_stmt"], 1258 symbol["simple_stmt"], 1259 symbol["compound_stmt"], 1260 symbol["expr_stmt"], 1261 symbol["print_stmt"], 1262 symbol["del_stmt"], 1263 symbol["pass_stmt"], 1264 symbol["break_stmt"], 1265 symbol["continue_stmt"], 1266 symbol["return_stmt"], 1267 symbol["raise_stmt"], 1268 symbol["import_stmt"], 1269 symbol["global_stmt"], 1270 symbol["exec_stmt"], 1271 symbol["assert_stmt"], 1272 symbol["if_stmt"], 1273 symbol["while_stmt"], 1274 symbol["for_stmt"], 1275 symbol["try_stmt"], 1276 symbol["with_stmt"], 1277 symbol["suite"], 1278 symbol["testlist"], 1279 symbol["testlist_safe"], 1280 symbol["test"], 1281 symbol["and_test"], 1282 symbol["not_test"], 1283 symbol["comparison"], 1284 symbol["exprlist"], 1285 symbol["expr"], 1286 symbol["xor_expr"], 1287 symbol["and_expr"], 1288 symbol["shift_expr"], 1289 symbol["arith_expr"], 1290 symbol["term"], 1291 symbol["factor"], 1292 symbol["power"], 1293 symbol["atom"], 1294 ] 1295 1296 _assign_types = [ 1297 symbol["test"], 1298 symbol["or_test"], 1299 symbol["and_test"], 1300 symbol["not_test"], 1301 symbol["comparison"], 1302 symbol["expr"], 1303 symbol["xor_expr"], 1304 symbol["and_expr"], 1305 symbol["shift_expr"], 1306 symbol["arith_expr"], 1307 symbol["term"], 1308 symbol["factor"], 1309 ] 1310 1311 _names = {} 1312 for k, v in sym_name.items(): 1313 _names[k] = v 1314 for k, v in tok_name.items(): 1315 _names[k] = v 1316 1317 def debug_tree(tree): 1318 l = [] 1319 for elt in tree: 1320 if isinstance(elt, int): 1321 l.append(_names.get(elt, elt)) 1322 elif isinstance(elt, str): 1323 l.append(elt) 1324 else: 1325 l.append(debug_tree(elt)) 1326 return l