1 """Parse tree transformation module. 2 3 Transforms Python source code into an abstract syntax tree (AST) 4 defined in the ast module. 5 6 The simplest ways to invoke this module are via parse and parseFile. 7 parse(buf) -> AST 8 parseFile(path) -> AST 9 """ 10 11 # Original version written by Greg Stein (gstein@lyra.org) 12 # and Bill Tutt (rassilon@lima.mudlib.org) 13 # February 1997. 14 # 15 # Modifications and improvements for Python 2.0 by Jeremy Hylton and 16 # Mark Hammond 17 # 18 # Some fixes to try to have correct line number on almost all nodes 19 # (except Module, Discard and Stmt) added by Sylvain Thenault 20 # 21 # Portions of this file are: 22 # Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. 23 # 24 # This module is provided under a BSD-ish license. See 25 # http://www.opensource.org/licenses/bsd-license.html 26 # and replace OWNER, ORGANIZATION, and YEAR as appropriate. 27 28 from compiler.ast import * 29 from pyparser.pygram import syms as symbol, sym_name, tokens as token, tok_name 30 import pyparser.pyparse as parser 31 32 class WalkerError(StandardError): 33 pass 34 35 from compiler.consts import CO_VARARGS, CO_VARKEYWORDS 36 from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY 37 38 def parseFile(path): 39 f = open(path, "U") 40 # XXX The parser API tolerates files without a trailing newline, 41 # but not strings without a trailing newline. Always add an extra 42 # newline to the file contents, since we're going through the string 43 # version of the API. 44 src = f.read() + "\n" 45 f.close() 46 return parse(src) 47 48 def parse(buf, mode="exec"): 49 if mode == "exec" or mode == "single": 50 return Transformer().parsesuite(buf) 51 elif mode == "eval": 52 return Transformer().parseexpr(buf) 53 else: 54 raise ValueError("compile() arg 3 must be" 55 " 'exec' or 'eval' or 'single'") 56 57 def extractLineNo(ast): 58 if not isinstance(ast[1], tuple): 59 # get a terminal node 60 return ast[2] 61 for child in ast[1:]: 62 if isinstance(child, tuple): 63 lineno = extractLineNo(child) 64 if lineno is not None: 65 return lineno 66 67 def Node(*args): 68 kind = args[0] 69 if kind in nodes: 70 try: 71 return nodes[kind](*args[1:]) 72 except TypeError: 73 print nodes[kind], len(args), args 74 raise 75 else: 76 raise WalkerError, "Can't find appropriate Node type: %s" % str(args) 77 #return apply(ast.Node, args) 78 79 class Transformer: 80 """Utility object for transforming Python parse trees. 81 82 Exposes the following methods: 83 tree = transform(ast_tree) 84 tree = parsesuite(text) 85 tree = parseexpr(text) 86 tree = parsefile(fileob | filename) 87 """ 88 89 def __init__(self): 90 self._dispatch = {} 91 for value, name in sym_name.items(): 92 if hasattr(self, name): 93 self._dispatch[value] = getattr(self, name) 94 self._dispatch[token["NEWLINE"]] = self.com_NEWLINE 95 self._atom_dispatch = {token["LPAR"]: self.atom_lpar, 96 token["LSQB"]: self.atom_lsqb, 97 token["LBRACE"]: self.atom_lbrace, 98 token["BACKQUOTE"]: self.atom_backquote, 99 token["NUMBER"]: self.atom_number, 100 token["STRING"]: self.atom_string, 101 token["NAME"]: self.atom_name, 102 } 103 self.encoding = None 104 105 def transform(self, tree): 106 """Transform an AST into a modified parse tree.""" 107 if not (isinstance(tree, tuple) or isinstance(tree, list)): 108 tree = parser.st2tuple(tree, line_info=1) 109 return self.compile_node(tree) 110 111 def parsesuite(self, text): 112 """Return a modified parse tree for the given suite text.""" 113 return self.transform(parser.suite(text)) 114 115 def parseexpr(self, text): 116 """Return a modified parse tree for the given expression text.""" 117 return self.transform(parser.expr(text)) 118 119 def parsefile(self, file): 120 """Return a modified parse tree for the contents of the given file.""" 121 if type(file) == type(''): 122 file = open(file) 123 return self.parsesuite(file.read()) 124 125 # -------------------------------------------------------------- 126 # 127 # PRIVATE METHODS 128 # 129 130 def compile_node(self, node): 131 ### emit a line-number node? 132 n = node[0] 133 134 if n == symbol["encoding_decl"]: 135 self.encoding = node[2] 136 node = node[1] 137 n = node[0] 138 139 if n == symbol["single_input"]: 140 return self.single_input(node[1:]) 141 if n == symbol["file_input"]: 142 return self.file_input(node[1:]) 143 if n == symbol["eval_input"]: 144 return self.eval_input(node[1:]) 145 if n == symbol["lambdef"]: 146 return self.lambdef(node[1:]) 147 if n == symbol["funcdef"]: 148 return self.funcdef(node[1:]) 149 if n == symbol["classdef"]: 150 return self.classdef(node[1:]) 151 152 raise WalkerError, ('unexpected node type', n) 153 154 def single_input(self, node): 155 ### do we want to do anything about being "interactive" ? 156 157 # NEWLINE | simple_stmt | compound_stmt NEWLINE 158 n = node[0][0] 159 if n != token["NEWLINE"]: 160 return self.com_stmt(node[0]) 161 162 return Pass() 163 164 def file_input(self, nodelist): 165 doc = self.get_docstring(nodelist, symbol["file_input"]) 166 if doc is not None: 167 i = 1 168 else: 169 i = 0 170 stmts = [] 171 for node in nodelist[i:]: 172 if node[0] != token["ENDMARKER"] and node[0] != token["NEWLINE"]: 173 self.com_append_stmt(stmts, node) 174 return Module(doc, Stmt(stmts)) 175 176 def eval_input(self, nodelist): 177 # from the built-in function input() 178 ### is this sufficient? 179 return Expression(self.com_node(nodelist[0])) 180 181 def decorator_name(self, nodelist): 182 listlen = len(nodelist) 183 assert listlen >= 1 and listlen % 2 == 1 184 185 item = self.atom_name(nodelist) 186 i = 1 187 while i < listlen: 188 assert nodelist[i][0] == token["DOT"] 189 assert nodelist[i + 1][0] == token["NAME"] 190 item = Getattr(item, nodelist[i + 1][1]) 191 i += 2 192 193 return item 194 195 def decorator(self, nodelist): 196 # '@' dotted_name [ '(' [arglist] ')' ] 197 assert len(nodelist) in (3, 5, 6) 198 assert nodelist[0][0] == token["AT"] 199 assert nodelist[-1][0] == token["NEWLINE"] 200 201 assert nodelist[1][0] == symbol["dotted_name"] 202 funcname = self.decorator_name(nodelist[1][1:]) 203 204 if len(nodelist) > 3: 205 assert nodelist[2][0] == token["LPAR"] 206 expr = self.com_call_function(funcname, nodelist[3]) 207 else: 208 expr = funcname 209 210 return expr 211 212 def decorators(self, nodelist): 213 # decorators: decorator ([NEWLINE] decorator)* NEWLINE 214 items = [] 215 for dec_nodelist in nodelist: 216 assert dec_nodelist[0] == symbol["decorator"] 217 items.append(self.decorator(dec_nodelist[1:])) 218 return Decorators(items) 219 220 def decorated(self, nodelist): 221 assert nodelist[0][0] == symbol["decorators"] 222 if nodelist[1][0] == symbol["funcdef"]: 223 n = [nodelist[0]] + list(nodelist[1][1:]) 224 return self.funcdef(n) 225 elif nodelist[1][0] == symbol["classdef"]: 226 decorators = self.decorators(nodelist[0][1:]) 227 cls = self.classdef(nodelist[1][1:]) 228 cls.decorators = decorators 229 return cls 230 raise WalkerError() 231 232 def funcdef(self, nodelist): 233 # -6 -5 -4 -3 -2 -1 234 # funcdef: [decorators] 'def' NAME parameters ':' suite 235 # parameters: '(' [varargslist] ')' 236 237 if len(nodelist) == 6: 238 assert nodelist[0][0] == symbol["decorators"] 239 decorators = self.decorators(nodelist[0][1:]) 240 else: 241 assert len(nodelist) == 5 242 decorators = None 243 244 lineno = nodelist[-4][2] 245 name = nodelist[-4][1] 246 args = nodelist[-3][2] 247 248 if args[0] == symbol["varargslist"]: 249 names, defaults, flags = self.com_arglist(args[1:]) 250 else: 251 names = defaults = () 252 flags = 0 253 doc = self.get_docstring(nodelist[-1]) 254 255 # code for function 256 code = self.com_node(nodelist[-1]) 257 258 if doc is not None: 259 assert isinstance(code, Stmt) 260 assert isinstance(code.nodes[0], Discard) 261 del code.nodes[0] 262 return Function(decorators, name, names, defaults, flags, doc, code, 263 lineno=lineno) 264 265 def lambdef(self, nodelist): 266 # lambdef: 'lambda' [varargslist] ':' test 267 if nodelist[2][0] == symbol["varargslist"]: 268 names, defaults, flags = self.com_arglist(nodelist[2][1:]) 269 else: 270 names = defaults = () 271 flags = 0 272 273 # code for lambda 274 code = self.com_node(nodelist[-1]) 275 276 return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) 277 old_lambdef = lambdef 278 279 def classdef(self, nodelist): 280 # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 281 282 name = nodelist[1][1] 283 doc = self.get_docstring(nodelist[-1]) 284 if nodelist[2][0] == token["COLON"]: 285 bases = [] 286 elif nodelist[3][0] == token["RPAR"]: 287 bases = [] 288 else: 289 bases = self.com_bases(nodelist[3]) 290 291 # code for class 292 code = self.com_node(nodelist[-1]) 293 294 if doc is not None: 295 assert isinstance(code, Stmt) 296 assert isinstance(code.nodes[0], Discard) 297 del code.nodes[0] 298 299 return Class(name, bases, doc, code, lineno=nodelist[1][2]) 300 301 def stmt(self, nodelist): 302 return self.com_stmt(nodelist[0]) 303 304 small_stmt = stmt 305 flow_stmt = stmt 306 compound_stmt = stmt 307 308 def simple_stmt(self, nodelist): 309 # small_stmt (';' small_stmt)* [';'] NEWLINE 310 stmts = [] 311 for i in range(0, len(nodelist), 2): 312 self.com_append_stmt(stmts, nodelist[i]) 313 return Stmt(stmts) 314 315 def parameters(self, nodelist): 316 raise WalkerError 317 318 def varargslist(self, nodelist): 319 raise WalkerError 320 321 def fpdef(self, nodelist): 322 raise WalkerError 323 324 def fplist(self, nodelist): 325 raise WalkerError 326 327 def dotted_name(self, nodelist): 328 raise WalkerError 329 330 def comp_op(self, nodelist): 331 raise WalkerError 332 333 def trailer(self, nodelist): 334 raise WalkerError 335 336 def sliceop(self, nodelist): 337 raise WalkerError 338 339 def argument(self, nodelist): 340 raise WalkerError 341 342 # -------------------------------------------------------------- 343 # 344 # STATEMENT NODES (invoked by com_node()) 345 # 346 347 def expr_stmt(self, nodelist): 348 # augassign testlist | testlist ('=' testlist)* 349 en = nodelist[-1] 350 exprNode = self.lookup_node(en)(en[1:]) 351 if len(nodelist) == 1: 352 return Discard(exprNode, lineno=exprNode.lineno) 353 if nodelist[1][0] == token["EQUAL"]: 354 nodesl = [] 355 for i in range(0, len(nodelist) - 2, 2): 356 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) 357 return Assign(nodesl, exprNode, lineno=nodelist[1][2]) 358 else: 359 lval = self.com_augassign(nodelist[0]) 360 op = self.com_augassign_op(nodelist[1]) 361 return AugAssign(lval, op[1], exprNode, lineno=op[2]) 362 raise WalkerError, "can't get here" 363 364 def print_stmt(self, nodelist): 365 # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) 366 items = [] 367 if len(nodelist) == 1: 368 start = 1 369 dest = None 370 elif nodelist[1][0] == token["RIGHTSHIFT"]: 371 assert len(nodelist) == 3 \ 372 or nodelist[3][0] == token["COMMA"] 373 dest = self.com_node(nodelist[2]) 374 start = 4 375 else: 376 dest = None 377 start = 1 378 for i in range(start, len(nodelist), 2): 379 items.append(self.com_node(nodelist[i])) 380 if nodelist[-1][0] == token["COMMA"]: 381 return Print(items, dest, lineno=nodelist[0][2]) 382 return Printnl(items, dest, lineno=nodelist[0][2]) 383 384 def del_stmt(self, nodelist): 385 return self.com_assign(nodelist[1], OP_DELETE) 386 387 def pass_stmt(self, nodelist): 388 return Pass(lineno=nodelist[0][2]) 389 390 def break_stmt(self, nodelist): 391 return Break(lineno=nodelist[0][2]) 392 393 def continue_stmt(self, nodelist): 394 return Continue(lineno=nodelist[0][2]) 395 396 def return_stmt(self, nodelist): 397 # return: [testlist] 398 if len(nodelist) < 2: 399 return Return(Const(None), lineno=nodelist[0][2]) 400 return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) 401 402 def yield_stmt(self, nodelist): 403 expr = self.com_node(nodelist[0]) 404 return Discard(expr, lineno=expr.lineno) 405 406 def yield_expr(self, nodelist): 407 if len(nodelist) > 1: 408 value = self.com_node(nodelist[1]) 409 else: 410 value = Const(None) 411 return Yield(value, lineno=nodelist[0][2]) 412 413 def raise_stmt(self, nodelist): 414 # raise: [test [',' test [',' test]]] 415 if len(nodelist) > 5: 416 expr3 = self.com_node(nodelist[5]) 417 else: 418 expr3 = None 419 if len(nodelist) > 3: 420 expr2 = self.com_node(nodelist[3]) 421 else: 422 expr2 = None 423 if len(nodelist) > 1: 424 expr1 = self.com_node(nodelist[1]) 425 else: 426 expr1 = None 427 return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) 428 429 def import_stmt(self, nodelist): 430 # import_stmt: import_name | import_from 431 assert len(nodelist) == 1 432 return self.com_node(nodelist[0]) 433 434 def import_name(self, nodelist): 435 # import_name: 'import' dotted_as_names 436 return Import(self.com_dotted_as_names(nodelist[1]), 437 lineno=nodelist[0][2]) 438 439 def import_from(self, nodelist): 440 # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | 441 # '(' import_as_names ')' | import_as_names) 442 assert nodelist[0][1] == 'from' 443 idx = 1 444 while nodelist[idx][1] == '.': 445 idx += 1 446 level = idx - 1 447 if nodelist[idx][0] == symbol["dotted_name"]: 448 fromname = self.com_dotted_name(nodelist[idx]) 449 idx += 1 450 else: 451 fromname = "" 452 assert nodelist[idx][1] == 'import' 453 if nodelist[idx + 1][0] == token["STAR"]: 454 return From(fromname, [('*', None)], level, 455 lineno=nodelist[0][2]) 456 else: 457 node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token["LPAR"])] 458 return From(fromname, self.com_import_as_names(node), level, 459 lineno=nodelist[0][2]) 460 461 def global_stmt(self, nodelist): 462 # global: NAME (',' NAME)* 463 names = [] 464 for i in range(1, len(nodelist), 2): 465 names.append(nodelist[i][1]) 466 return Global(names, lineno=nodelist[0][2]) 467 468 def exec_stmt(self, nodelist): 469 # exec_stmt: 'exec' expr ['in' expr [',' expr]] 470 expr1 = self.com_node(nodelist[1]) 471 if len(nodelist) >= 4: 472 expr2 = self.com_node(nodelist[3]) 473 if len(nodelist) >= 6: 474 expr3 = self.com_node(nodelist[5]) 475 else: 476 expr3 = None 477 else: 478 expr2 = expr3 = None 479 480 return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) 481 482 def assert_stmt(self, nodelist): 483 # 'assert': test, [',' test] 484 expr1 = self.com_node(nodelist[1]) 485 if (len(nodelist) == 4): 486 expr2 = self.com_node(nodelist[3]) 487 else: 488 expr2 = None 489 return Assert(expr1, expr2, lineno=nodelist[0][2]) 490 491 def if_stmt(self, nodelist): 492 # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 493 tests = [] 494 for i in range(0, len(nodelist) - 3, 4): 495 testNode = self.com_node(nodelist[i + 1]) 496 suiteNode = self.com_node(nodelist[i + 3]) 497 tests.append((testNode, suiteNode)) 498 499 if len(nodelist) % 4 == 3: 500 elseNode = self.com_node(nodelist[-1]) 501 ## elseNode.lineno = nodelist[-1][1][2] 502 else: 503 elseNode = None 504 return If(tests, elseNode, lineno=nodelist[0][2]) 505 506 def while_stmt(self, nodelist): 507 # 'while' test ':' suite ['else' ':' suite] 508 509 testNode = self.com_node(nodelist[1]) 510 bodyNode = self.com_node(nodelist[3]) 511 512 if len(nodelist) > 4: 513 elseNode = self.com_node(nodelist[6]) 514 else: 515 elseNode = None 516 517 return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) 518 519 def for_stmt(self, nodelist): 520 # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] 521 522 assignNode = self.com_assign(nodelist[1], OP_ASSIGN) 523 listNode = self.com_node(nodelist[3]) 524 bodyNode = self.com_node(nodelist[5]) 525 526 if len(nodelist) > 8: 527 elseNode = self.com_node(nodelist[8]) 528 else: 529 elseNode = None 530 531 return For(assignNode, listNode, bodyNode, elseNode, 532 lineno=nodelist[0][2]) 533 534 def try_stmt(self, nodelist): 535 return self.com_try_except_finally(nodelist) 536 537 def with_stmt(self, nodelist): 538 return self.com_with(nodelist) 539 540 def suite(self, nodelist): 541 # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT 542 if len(nodelist) == 1: 543 return self.com_stmt(nodelist[0]) 544 545 stmts = [] 546 for node in nodelist: 547 if node[0] == symbol["stmt"]: 548 self.com_append_stmt(stmts, node) 549 return Stmt(stmts) 550 551 # -------------------------------------------------------------- 552 # 553 # EXPRESSION NODES (invoked by com_node()) 554 # 555 556 def testlist(self, nodelist): 557 # testlist: expr (',' expr)* [','] 558 # testlist_safe: test [(',' test)+ [',']] 559 # exprlist: expr (',' expr)* [','] 560 return self.com_binary(Tuple, nodelist) 561 562 testlist_safe = testlist # XXX 563 testlist1 = testlist 564 exprlist = testlist 565 566 def testlist_comp(self, nodelist): 567 # test ( comp_for | (',' test)* [','] ) 568 assert nodelist[0][0] == symbol["test"] 569 if len(nodelist) == 2 and nodelist[1][0] == symbol["comp_for"]: 570 test = self.com_node(nodelist[0]) 571 return self.com_generator_expression(test, nodelist[1]) 572 return self.testlist(nodelist) 573 574 def test(self, nodelist): 575 # or_test ['if' or_test 'else' test] | lambdef 576 if len(nodelist) == 1 and nodelist[0][0] == symbol["lambdef"]: 577 return self.lambdef(nodelist[0]) 578 then = self.com_node(nodelist[0]) 579 if len(nodelist) > 1: 580 assert len(nodelist) == 5 581 assert nodelist[1][1] == 'if' 582 assert nodelist[3][1] == 'else' 583 test = self.com_node(nodelist[2]) 584 else_ = self.com_node(nodelist[4]) 585 return IfExp(test, then, else_, lineno=nodelist[1][2]) 586 return then 587 588 def or_test(self, nodelist): 589 # and_test ('or' and_test)* | lambdef 590 if len(nodelist) == 1 and nodelist[0][0] == symbol["lambdef"]: 591 return self.lambdef(nodelist[0]) 592 return self.com_binary(Or, nodelist) 593 old_test = or_test 594 595 def and_test(self, nodelist): 596 # not_test ('and' not_test)* 597 return self.com_binary(And, nodelist) 598 599 def not_test(self, nodelist): 600 # 'not' not_test | comparison 601 result = self.com_node(nodelist[-1]) 602 if len(nodelist) == 2: 603 return Not(result, lineno=nodelist[0][2]) 604 return result 605 606 def comparison(self, nodelist): 607 # comparison: expr (comp_op expr)* 608 node = self.com_node(nodelist[0]) 609 if len(nodelist) == 1: 610 return node 611 612 results = [] 613 for i in range(2, len(nodelist), 2): 614 nl = nodelist[i-1] 615 616 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 617 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 618 n = nl[1] 619 if n[0] == token["NAME"]: 620 type = n[1] 621 if len(nl) == 3: 622 if type == 'not': 623 type = 'not in' 624 else: 625 type = 'is not' 626 else: 627 type = _cmp_types[n[0]] 628 629 lineno = nl[1][2] 630 results.append((type, self.com_node(nodelist[i]))) 631 632 # we need a special "compare" node so that we can distinguish 633 # 3 < x < 5 from (3 < x) < 5 634 # the two have very different semantics and results (note that the 635 # latter form is always true) 636 637 return Compare(node, results, lineno=lineno) 638 639 def expr(self, nodelist): 640 # xor_expr ('|' xor_expr)* 641 return self.com_binary(Bitor, nodelist) 642 643 def xor_expr(self, nodelist): 644 # xor_expr ('^' xor_expr)* 645 return self.com_binary(Bitxor, nodelist) 646 647 def and_expr(self, nodelist): 648 # xor_expr ('&' xor_expr)* 649 return self.com_binary(Bitand, nodelist) 650 651 def shift_expr(self, nodelist): 652 # shift_expr ('<<'|'>>' shift_expr)* 653 node = self.com_node(nodelist[0]) 654 for i in range(2, len(nodelist), 2): 655 right = self.com_node(nodelist[i]) 656 if nodelist[i-1][0] == token["LEFTSHIFT"]: 657 node = LeftShift([node, right], lineno=nodelist[1][2]) 658 elif nodelist[i-1][0] == token["RIGHTSHIFT"]: 659 node = RightShift([node, right], lineno=nodelist[1][2]) 660 else: 661 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 662 return node 663 664 def arith_expr(self, nodelist): 665 node = self.com_node(nodelist[0]) 666 for i in range(2, len(nodelist), 2): 667 right = self.com_node(nodelist[i]) 668 if nodelist[i-1][0] == token["PLUS"]: 669 node = Add([node, right], lineno=nodelist[1][2]) 670 elif nodelist[i-1][0] == token["MINUS"]: 671 node = Sub([node, right], lineno=nodelist[1][2]) 672 else: 673 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 674 return node 675 676 def term(self, nodelist): 677 node = self.com_node(nodelist[0]) 678 for i in range(2, len(nodelist), 2): 679 right = self.com_node(nodelist[i]) 680 t = nodelist[i-1][0] 681 if t == token["STAR"]: 682 node = Mul([node, right]) 683 elif t == token["SLASH"]: 684 node = Div([node, right]) 685 elif t == token["PERCENT"]: 686 node = Mod([node, right]) 687 elif t == token["DOUBLESLASH"]: 688 node = FloorDiv([node, right]) 689 else: 690 raise ValueError, "unexpected token: %s" % t 691 node.lineno = nodelist[1][2] 692 return node 693 694 def factor(self, nodelist): 695 elt = nodelist[0] 696 t = elt[0] 697 node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) 698 # need to handle (unary op)constant here... 699 if t == token["PLUS"]: 700 return UnaryAdd(node, lineno=elt[2]) 701 elif t == token["MINUS"]: 702 return UnarySub(node, lineno=elt[2]) 703 elif t == token["TILDE"]: 704 node = Invert(node, lineno=elt[2]) 705 return node 706 707 def power(self, nodelist): 708 # power: atom trailer* ('**' factor)* 709 node = self.com_node(nodelist[0]) 710 for i in range(1, len(nodelist)): 711 elt = nodelist[i] 712 if elt[0] == token["DOUBLESTAR"]: 713 return Power([node, self.com_node(nodelist[i+1])], 714 lineno=elt[2]) 715 716 node = self.com_apply_trailer(node, elt) 717 718 return node 719 720 def atom(self, nodelist): 721 return self._atom_dispatch[nodelist[0][0]](nodelist) 722 723 def atom_lpar(self, nodelist): 724 if nodelist[1][0] == token["RPAR"]: 725 return Tuple((), lineno=nodelist[0][2]) 726 return self.com_node(nodelist[1]) 727 728 def atom_lsqb(self, nodelist): 729 if nodelist[1][0] == token["RSQB"]: 730 return List((), lineno=nodelist[0][2]) 731 return self.com_list_constructor(nodelist[1]) 732 733 def atom_lbrace(self, nodelist): 734 if nodelist[1][0] == token["RBRACE"]: 735 return Dict((), lineno=nodelist[0][2]) 736 return self.com_dictorsetmaker(nodelist[1]) 737 738 def atom_backquote(self, nodelist): 739 return Backquote(self.com_node(nodelist[1])) 740 741 def atom_number(self, nodelist): 742 ### need to verify this matches compile.c 743 k = eval(nodelist[0][1]) 744 return Const(k, nodelist[0][1], lineno=nodelist[0][2]) 745 746 def decode_literal(self, lit): 747 if self.encoding: 748 # this is particularly fragile & a bit of a 749 # hack... changes in compile.c:parsestr and 750 # tokenizer.c must be reflected here. 751 if self.encoding not in ['utf-8', 'iso-8859-1']: 752 lit = unicode(lit, 'utf-8').encode(self.encoding) 753 return eval("# coding: %s\n%s" % (self.encoding, lit)) 754 else: 755 return eval(lit) 756 757 def atom_string(self, nodelist): 758 k = '' 759 for node in nodelist: 760 k += self.decode_literal(node[1]) 761 return Const(k, node[1], lineno=nodelist[0][2]) 762 763 def atom_name(self, nodelist): 764 return Name(nodelist[0][1], lineno=nodelist[0][2]) 765 766 # -------------------------------------------------------------- 767 # 768 # INTERNAL PARSING UTILITIES 769 # 770 771 # The use of com_node() introduces a lot of extra stack frames, 772 # enough to cause a stack overflow compiling test.test_parser with 773 # the standard interpreter recursionlimit. The com_node() is a 774 # convenience function that hides the dispatch details, but comes 775 # at a very high cost. It is more efficient to dispatch directly 776 # in the callers. In these cases, use lookup_node() and call the 777 # dispatched node directly. 778 779 def lookup_node(self, node): 780 return self._dispatch[node[0]] 781 782 def com_node(self, node): 783 # Note: compile.c has handling in com_node for del_stmt, pass_stmt, 784 # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, 785 # and compound_stmt. 786 # We'll just dispatch them. 787 return self._dispatch[node[0]](node[1:]) 788 789 def com_NEWLINE(self, *args): 790 # A ';' at the end of a line can make a NEWLINE token appear 791 # here, Render it harmless. (genc discards ('discard', 792 # ('const', xxxx)) Nodes) 793 return Discard(Const(None)) 794 795 def com_arglist(self, nodelist): 796 # varargslist: 797 # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) 798 # | fpdef ['=' test] (',' fpdef ['=' test])* [','] 799 # fpdef: NAME | '(' fplist ')' 800 # fplist: fpdef (',' fpdef)* [','] 801 names = [] 802 defaults = [] 803 flags = 0 804 805 i = 0 806 while i < len(nodelist): 807 node = nodelist[i] 808 if node[0] == token["STAR"] or node[0] == token["DOUBLESTAR"]: 809 if node[0] == token["STAR"]: 810 node = nodelist[i+1] 811 if node[0] == token["NAME"]: 812 names.append(node[1]) 813 flags = flags | CO_VARARGS 814 i = i + 3 815 816 if i < len(nodelist): 817 # should be DOUBLESTAR 818 t = nodelist[i][0] 819 if t == token["DOUBLESTAR"]: 820 node = nodelist[i+1] 821 else: 822 raise ValueError, "unexpected token: %s" % t 823 names.append(node[1]) 824 flags = flags | CO_VARKEYWORDS 825 826 break 827 828 # fpdef: NAME | '(' fplist ')' 829 names.append(self.com_fpdef(node)) 830 831 i = i + 1 832 if i < len(nodelist) and nodelist[i][0] == token["EQUAL"]: 833 defaults.append(self.com_node(nodelist[i + 1])) 834 i = i + 2 835 elif len(defaults): 836 # we have already seen an argument with default, but here 837 # came one without 838 raise SyntaxError, "non-default argument follows default argument" 839 840 # skip the comma 841 i = i + 1 842 843 return names, defaults, flags 844 845 def com_fpdef(self, node): 846 # fpdef: NAME | '(' fplist ')' 847 if node[1][0] == token["LPAR"]: 848 return self.com_fplist(node[2]) 849 return node[1][1] 850 851 def com_fplist(self, node): 852 # fplist: fpdef (',' fpdef)* [','] 853 if len(node) == 2: 854 return self.com_fpdef(node[1]) 855 list = [] 856 for i in range(1, len(node), 2): 857 list.append(self.com_fpdef(node[i])) 858 return tuple(list) 859 860 def com_dotted_name(self, node): 861 # String together the dotted names and return the string 862 name = "" 863 for n in node: 864 if type(n) == type(()) and n[0] == 1: 865 name = name + n[1] + '.' 866 return name[:-1] 867 868 def com_dotted_as_name(self, node): 869 assert node[0] == symbol["dotted_as_name"] 870 node = node[1:] 871 dot = self.com_dotted_name(node[0][1:]) 872 if len(node) == 1: 873 return dot, None 874 assert node[1][1] == 'as' 875 assert node[2][0] == token["NAME"] 876 return dot, node[2][1] 877 878 def com_dotted_as_names(self, node): 879 assert node[0] == symbol["dotted_as_names"] 880 node = node[1:] 881 names = [self.com_dotted_as_name(node[0])] 882 for i in range(2, len(node), 2): 883 names.append(self.com_dotted_as_name(node[i])) 884 return names 885 886 def com_import_as_name(self, node): 887 assert node[0] == symbol["import_as_name"] 888 node = node[1:] 889 assert node[0][0] == token["NAME"] 890 if len(node) == 1: 891 return node[0][1], None 892 assert node[1][1] == 'as', node 893 assert node[2][0] == token["NAME"] 894 return node[0][1], node[2][1] 895 896 def com_import_as_names(self, node): 897 assert node[0] == symbol["import_as_names"] 898 node = node[1:] 899 names = [self.com_import_as_name(node[0])] 900 for i in range(2, len(node), 2): 901 names.append(self.com_import_as_name(node[i])) 902 return names 903 904 def com_bases(self, node): 905 bases = [] 906 for i in range(1, len(node), 2): 907 bases.append(self.com_node(node[i])) 908 return bases 909 910 def com_try_except_finally(self, nodelist): 911 # ('try' ':' suite 912 # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] 913 # | 'finally' ':' suite)) 914 915 if nodelist[3][0] == token["NAME"]: 916 # first clause is a finally clause: only try-finally 917 return TryFinally(self.com_node(nodelist[2]), 918 self.com_node(nodelist[5]), 919 lineno=nodelist[0][2]) 920 921 #tryexcept: [TryNode, [except_clauses], elseNode)] 922 clauses = [] 923 elseNode = None 924 finallyNode = None 925 for i in range(3, len(nodelist), 3): 926 node = nodelist[i] 927 if node[0] == symbol["except_clause"]: 928 # except_clause: 'except' [expr [(',' | 'as') expr]] */ 929 if len(node) > 2: 930 expr1 = self.com_node(node[2]) 931 if len(node) > 4: 932 expr2 = self.com_assign(node[4], OP_ASSIGN) 933 else: 934 expr2 = None 935 else: 936 expr1 = expr2 = None 937 clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) 938 939 if node[0] == token["NAME"]: 940 if node[1] == 'else': 941 elseNode = self.com_node(nodelist[i+2]) 942 elif node[1] == 'finally': 943 finallyNode = self.com_node(nodelist[i+2]) 944 try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, 945 lineno=nodelist[0][2]) 946 if finallyNode: 947 return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) 948 else: 949 return try_except 950 951 def com_with(self, nodelist): 952 # with_stmt: 'with' with_item (',' with_item)* ':' suite 953 body = self.com_node(nodelist[-1]) 954 for i in range(len(nodelist) - 3, 0, -2): 955 ret = self.com_with_item(nodelist[i], body, nodelist[0][2]) 956 if i == 1: 957 return ret 958 body = ret 959 960 def com_with_item(self, nodelist, body, lineno): 961 # with_item: test ['as' expr] 962 if len(nodelist) == 4: 963 var = self.com_assign(nodelist[3], OP_ASSIGN) 964 else: 965 var = None 966 expr = self.com_node(nodelist[1]) 967 return With(expr, var, body, lineno=lineno) 968 969 def com_augassign_op(self, node): 970 assert node[0] == symbol["augassign"] 971 return node[1] 972 973 def com_augassign(self, node): 974 """Return node suitable for lvalue of augmented assignment 975 976 Names, slices, and attributes are the only allowable nodes. 977 """ 978 l = self.com_node(node) 979 if l.__class__ in (Name, Slice, Subscript, Getattr): 980 return l 981 raise SyntaxError, "can't assign to %s" % l.__class__.__name__ 982 983 def com_assign(self, node, assigning): 984 # return a node suitable for use as an "lvalue" 985 # loop to avoid trivial recursion 986 while 1: 987 t = node[0] 988 if t in (symbol["exprlist"], symbol["testlist"], symbol["testlist_safe"], symbol["testlist_comp"]): 989 if len(node) > 2: 990 return self.com_assign_tuple(node, assigning) 991 node = node[1] 992 elif t in _assign_types: 993 if len(node) > 2: 994 raise SyntaxError, "can't assign to operator" 995 node = node[1] 996 elif t == symbol["power"]: 997 if node[1][0] != symbol["atom"]: 998 raise SyntaxError, "can't assign to operator" 999 if len(node) > 2: 1000 primary = self.com_node(node[1]) 1001 for i in range(2, len(node)-1): 1002 ch = node[i] 1003 if ch[0] == token["DOUBLESTAR"]: 1004 raise SyntaxError, "can't assign to operator" 1005 primary = self.com_apply_trailer(primary, ch) 1006 return self.com_assign_trailer(primary, node[-1], 1007 assigning) 1008 node = node[1] 1009 elif t == symbol["atom"]: 1010 t = node[1][0] 1011 if t == token["LPAR"]: 1012 node = node[2] 1013 if node[0] == token["RPAR"]: 1014 raise SyntaxError, "can't assign to ()" 1015 elif t == token["LSQB"]: 1016 node = node[2] 1017 if node[0] == token["RSQB"]: 1018 raise SyntaxError, "can't assign to []" 1019 return self.com_assign_list(node, assigning) 1020 elif t == token["NAME"]: 1021 return self.com_assign_name(node[1], assigning) 1022 else: 1023 raise SyntaxError, "can't assign to literal" 1024 else: 1025 raise SyntaxError, "bad assignment (%s)" % t 1026 1027 def com_assign_tuple(self, node, assigning): 1028 assigns = [] 1029 for i in range(1, len(node), 2): 1030 assigns.append(self.com_assign(node[i], assigning)) 1031 return AssTuple(assigns, lineno=extractLineNo(node)) 1032 1033 def com_assign_list(self, node, assigning): 1034 assigns = [] 1035 for i in range(1, len(node), 2): 1036 if i + 1 < len(node): 1037 if node[i + 1][0] == symbol["list_for"]: 1038 raise SyntaxError, "can't assign to list comprehension" 1039 assert node[i + 1][0] == token["COMMA"], node[i + 1] 1040 assigns.append(self.com_assign(node[i], assigning)) 1041 return AssList(assigns, lineno=extractLineNo(node)) 1042 1043 def com_assign_name(self, node, assigning): 1044 return AssName(node[1], assigning, lineno=node[2]) 1045 1046 def com_assign_trailer(self, primary, node, assigning): 1047 t = node[1][0] 1048 if t == token["DOT"]: 1049 return self.com_assign_attr(primary, node[2], assigning) 1050 if t == token["LSQB"]: 1051 return self.com_subscriptlist(primary, node[2], assigning) 1052 if t == token["LPAR"]: 1053 raise SyntaxError, "can't assign to function call" 1054 raise SyntaxError, "unknown trailer type: %s" % t 1055 1056 def com_assign_attr(self, primary, node, assigning): 1057 return AssAttr(primary, node[1], assigning, lineno=node[-1]) 1058 1059 def com_binary(self, constructor, nodelist): 1060 "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." 1061 l = len(nodelist) 1062 if l == 1: 1063 n = nodelist[0] 1064 return self.lookup_node(n)(n[1:]) 1065 items = [] 1066 for i in range(0, l, 2): 1067 n = nodelist[i] 1068 items.append(self.lookup_node(n)(n[1:])) 1069 return constructor(items, lineno=extractLineNo(nodelist)) 1070 1071 def com_stmt(self, node): 1072 result = self.lookup_node(node)(node[1:]) 1073 assert result is not None 1074 if isinstance(result, Stmt): 1075 return result 1076 return Stmt([result]) 1077 1078 def com_append_stmt(self, stmts, node): 1079 result = self.lookup_node(node)(node[1:]) 1080 assert result is not None 1081 if isinstance(result, Stmt): 1082 stmts.extend(result.nodes) 1083 else: 1084 stmts.append(result) 1085 1086 def com_list_constructor(self, nodelist): 1087 # listmaker: test ( list_for | (',' test)* [','] ) 1088 values = [] 1089 for i in range(1, len(nodelist)): 1090 if nodelist[i][0] == symbol["list_for"]: 1091 assert len(nodelist[i:]) == 1 1092 return self.com_list_comprehension(values[0], 1093 nodelist[i]) 1094 elif nodelist[i][0] == token["COMMA"]: 1095 continue 1096 values.append(self.com_node(nodelist[i])) 1097 return List(values, lineno=values[0].lineno) 1098 1099 def com_list_comprehension(self, expr, node): 1100 return self.com_comprehension(expr, None, node, 'list') 1101 1102 def com_comprehension(self, expr1, expr2, node, type): 1103 # list_iter: list_for | list_if 1104 # list_for: 'for' exprlist 'in' testlist [list_iter] 1105 # list_if: 'if' test [list_iter] 1106 1107 # XXX should raise SyntaxError for assignment 1108 # XXX(avassalotti) Set and dict comprehensions should have generator 1109 # semantics. In other words, they shouldn't leak 1110 # variables outside of the comprehension's scope. 1111 1112 lineno = node[1][2] 1113 fors = [] 1114 while node: 1115 t = node[1][1] 1116 if t == 'for': 1117 assignNode = self.com_assign(node[2], OP_ASSIGN) 1118 compNode = self.com_node(node[4]) 1119 newfor = ListCompFor(assignNode, compNode, []) 1120 newfor.lineno = node[1][2] 1121 fors.append(newfor) 1122 if len(node) == 5: 1123 node = None 1124 elif type == 'list': 1125 node = self.com_list_iter(node[5]) 1126 else: 1127 node = self.com_comp_iter(node[5]) 1128 elif t == 'if': 1129 test = self.com_node(node[2]) 1130 newif = ListCompIf(test, lineno=node[1][2]) 1131 newfor.ifs.append(newif) 1132 if len(node) == 3: 1133 node = None 1134 elif type == 'list': 1135 node = self.com_list_iter(node[3]) 1136 else: 1137 node = self.com_comp_iter(node[3]) 1138 else: 1139 raise SyntaxError, \ 1140 ("unexpected comprehension element: %s %d" 1141 % (node, lineno)) 1142 if type == 'list': 1143 return ListComp(expr1, fors, lineno=lineno) 1144 elif type == 'set': 1145 return SetComp(expr1, fors, lineno=lineno) 1146 elif type == 'dict': 1147 return DictComp(expr1, expr2, fors, lineno=lineno) 1148 else: 1149 raise ValueError("unexpected comprehension type: " + repr(type)) 1150 1151 def com_list_iter(self, node): 1152 assert node[0] == symbol["list_iter"] 1153 return node[1] 1154 1155 def com_comp_iter(self, node): 1156 assert node[0] == symbol["comp_iter"] 1157 return node[1] 1158 1159 def com_generator_expression(self, expr, node): 1160 # comp_iter: comp_for | comp_if 1161 # comp_for: 'for' exprlist 'in' test [comp_iter] 1162 # comp_if: 'if' test [comp_iter] 1163 1164 lineno = node[1][2] 1165 fors = [] 1166 while node: 1167 t = node[1][1] 1168 if t == 'for': 1169 assignNode = self.com_assign(node[2], OP_ASSIGN) 1170 genNode = self.com_node(node[4]) 1171 newfor = GenExprFor(assignNode, genNode, [], 1172 lineno=node[1][2]) 1173 fors.append(newfor) 1174 if (len(node)) == 5: 1175 node = None 1176 else: 1177 node = self.com_comp_iter(node[5]) 1178 elif t == 'if': 1179 test = self.com_node(node[2]) 1180 newif = GenExprIf(test, lineno=node[1][2]) 1181 newfor.ifs.append(newif) 1182 if len(node) == 3: 1183 node = None 1184 else: 1185 node = self.com_comp_iter(node[3]) 1186 else: 1187 raise SyntaxError, \ 1188 ("unexpected generator expression element: %s %d" 1189 % (node, lineno)) 1190 fors[0].is_outmost = True 1191 return GenExpr(GenExprInner(expr, fors), lineno=lineno) 1192 1193 def com_dictorsetmaker(self, nodelist): 1194 # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 1195 # (test (comp_for | (',' test)* [','])) ) 1196 assert nodelist[0] == symbol["dictorsetmaker"] 1197 nodelist = nodelist[1:] 1198 if len(nodelist) == 1 or nodelist[1][0] == token["COMMA"]: 1199 # set literal 1200 items = [] 1201 for i in range(0, len(nodelist), 2): 1202 items.append(self.com_node(nodelist[i])) 1203 return Set(items, lineno=items[0].lineno) 1204 elif nodelist[1][0] == symbol["comp_for"]: 1205 # set comprehension 1206 expr = self.com_node(nodelist[0]) 1207 return self.com_comprehension(expr, None, nodelist[1], 'set') 1208 elif len(nodelist) > 3 and nodelist[3][0] == symbol["comp_for"]: 1209 # dict comprehension 1210 assert nodelist[1][0] == token["COLON"] 1211 key = self.com_node(nodelist[0]) 1212 value = self.com_node(nodelist[2]) 1213 return self.com_comprehension(key, value, nodelist[3], 'dict') 1214 else: 1215 # dict literal 1216 items = [] 1217 for i in range(0, len(nodelist), 4): 1218 items.append((self.com_node(nodelist[i]), 1219 self.com_node(nodelist[i+2]))) 1220 return Dict(items, lineno=items[0][0].lineno) 1221 1222 def com_apply_trailer(self, primaryNode, nodelist): 1223 t = nodelist[1][0] 1224 if t == token["LPAR"]: 1225 return self.com_call_function(primaryNode, nodelist[2]) 1226 if t == token["DOT"]: 1227 return self.com_select_member(primaryNode, nodelist[2]) 1228 if t == token["LSQB"]: 1229 return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) 1230 1231 raise SyntaxError, 'unknown node type: %s' % t 1232 1233 def com_select_member(self, primaryNode, nodelist): 1234 if nodelist[0] != token["NAME"]: 1235 raise SyntaxError, "member must be a name" 1236 return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) 1237 1238 def com_call_function(self, primaryNode, nodelist): 1239 if nodelist[0] == token["RPAR"]: 1240 return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) 1241 args = [] 1242 kw = 0 1243 star_node = dstar_node = None 1244 len_nodelist = len(nodelist) 1245 i = 1 1246 while i < len_nodelist: 1247 node = nodelist[i] 1248 1249 if node[0]==token["STAR"]: 1250 if star_node is not None: 1251 raise SyntaxError, 'already have the varargs indentifier' 1252 star_node = self.com_node(nodelist[i+1]) 1253 i = i + 3 1254 continue 1255 elif node[0]==token["DOUBLESTAR"]: 1256 if dstar_node is not None: 1257 raise SyntaxError, 'already have the kwargs indentifier' 1258 dstar_node = self.com_node(nodelist[i+1]) 1259 i = i + 3 1260 continue 1261 1262 # positional or named parameters 1263 kw, result = self.com_argument(node, kw, star_node) 1264 1265 if len_nodelist != 2 and isinstance(result, GenExpr) \ 1266 and len(node) == 3 and node[2][0] == symbol["comp_for"]: 1267 # allow f(x for x in y), but reject f(x for x in y, 1) 1268 # should use f((x for x in y), 1) instead of f(x for x in y, 1) 1269 raise SyntaxError, 'generator expression needs parenthesis' 1270 1271 args.append(result) 1272 i = i + 2 1273 1274 return CallFunc(primaryNode, args, star_node, dstar_node, 1275 lineno=extractLineNo(nodelist)) 1276 1277 def com_argument(self, nodelist, kw, star_node): 1278 if len(nodelist) == 3 and nodelist[2][0] == symbol["comp_for"]: 1279 test = self.com_node(nodelist[1]) 1280 return 0, self.com_generator_expression(test, nodelist[2]) 1281 if len(nodelist) == 2: 1282 if kw: 1283 raise SyntaxError, "non-keyword arg after keyword arg" 1284 if star_node: 1285 raise SyntaxError, "only named arguments may follow *expression" 1286 return 0, self.com_node(nodelist[1]) 1287 result = self.com_node(nodelist[3]) 1288 n = nodelist[1] 1289 while len(n) == 2 and n[0] != token["NAME"]: 1290 n = n[1] 1291 if n[0] != token["NAME"]: 1292 raise SyntaxError, "keyword can't be an expression (%s)"%n[0] 1293 node = Keyword(n[1], result, lineno=n[2]) 1294 return 1, node 1295 1296 def com_subscriptlist(self, primary, nodelist, assigning): 1297 # slicing: simple_slicing | extended_slicing 1298 # simple_slicing: primary "[" short_slice "]" 1299 # extended_slicing: primary "[" slice_list "]" 1300 # slice_list: slice_item ("," slice_item)* [","] 1301 1302 # backwards compat slice for '[i:j]' 1303 if len(nodelist) == 2: 1304 sub = nodelist[1] 1305 if (sub[1][0] == token["COLON"] or \ 1306 (len(sub) > 2 and sub[2][0] == token["COLON"])) and \ 1307 sub[-1][0] != symbol["sliceop"]: 1308 return self.com_slice(primary, sub, assigning) 1309 1310 subscripts = [] 1311 for i in range(1, len(nodelist), 2): 1312 subscripts.append(self.com_subscript(nodelist[i])) 1313 return Subscript(primary, assigning, subscripts, 1314 lineno=extractLineNo(nodelist)) 1315 1316 def com_subscript(self, node): 1317 # slice_item: expression | proper_slice | ellipsis 1318 ch = node[1] 1319 t = ch[0] 1320 if t == token["DOT"] and node[2][0] == token["DOT"]: 1321 return Ellipsis() 1322 if t == token["COLON"] or len(node) > 2: 1323 return self.com_sliceobj(node) 1324 return self.com_node(ch) 1325 1326 def com_sliceobj(self, node): 1327 # proper_slice: short_slice | long_slice 1328 # short_slice: [lower_bound] ":" [upper_bound] 1329 # long_slice: short_slice ":" [stride] 1330 # lower_bound: expression 1331 # upper_bound: expression 1332 # stride: expression 1333 # 1334 # Note: a stride may be further slicing... 1335 1336 items = [] 1337 1338 if node[1][0] == token["COLON"]: 1339 items.append(Const(None)) 1340 i = 2 1341 else: 1342 items.append(self.com_node(node[1])) 1343 # i == 2 is a COLON 1344 i = 3 1345 1346 if i < len(node) and node[i][0] == symbol["test"]: 1347 items.append(self.com_node(node[i])) 1348 i = i + 1 1349 else: 1350 items.append(Const(None)) 1351 1352 # a short_slice has been built. look for long_slice now by looking 1353 # for strides... 1354 for j in range(i, len(node)): 1355 ch = node[j] 1356 if len(ch) == 2: 1357 items.append(Const(None)) 1358 else: 1359 items.append(self.com_node(ch[2])) 1360 return Sliceobj(items, lineno=extractLineNo(node)) 1361 1362 def com_slice(self, primary, node, assigning): 1363 # short_slice: [lower_bound] ":" [upper_bound] 1364 lower = upper = None 1365 if len(node) == 3: 1366 if node[1][0] == token["COLON"]: 1367 upper = self.com_node(node[2]) 1368 else: 1369 lower = self.com_node(node[1]) 1370 elif len(node) == 4: 1371 lower = self.com_node(node[1]) 1372 upper = self.com_node(node[3]) 1373 return Slice(primary, assigning, lower, upper, 1374 lineno=extractLineNo(node)) 1375 1376 def get_docstring(self, node, n=None): 1377 if n is None: 1378 n = node[0] 1379 node = node[1:] 1380 if n == symbol["suite"]: 1381 if len(node) == 1: 1382 return self.get_docstring(node[0]) 1383 for sub in node: 1384 if sub[0] == symbol["stmt"]: 1385 return self.get_docstring(sub) 1386 return None 1387 if n == symbol["file_input"]: 1388 for sub in node: 1389 if sub[0] == symbol["stmt"]: 1390 return self.get_docstring(sub) 1391 return None 1392 if n == symbol["atom"]: 1393 if node[0][0] == token["STRING"]: 1394 s = '' 1395 for t in node: 1396 s = s + eval(t[1]) 1397 return s 1398 return None 1399 if n == symbol["stmt"] or n == symbol["simple_stmt"] \ 1400 or n == symbol["small_stmt"]: 1401 return self.get_docstring(node[0]) 1402 if n in _doc_nodes and len(node) == 1: 1403 return self.get_docstring(node[0]) 1404 return None 1405 1406 1407 _doc_nodes = [ 1408 symbol["expr_stmt"], 1409 symbol["testlist"], 1410 symbol["testlist_safe"], 1411 symbol["test"], 1412 symbol["or_test"], 1413 symbol["and_test"], 1414 symbol["not_test"], 1415 symbol["comparison"], 1416 symbol["expr"], 1417 symbol["xor_expr"], 1418 symbol["and_expr"], 1419 symbol["shift_expr"], 1420 symbol["arith_expr"], 1421 symbol["term"], 1422 symbol["factor"], 1423 symbol["power"], 1424 ] 1425 1426 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 1427 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 1428 _cmp_types = { 1429 token["LESS"] : '<', 1430 token["GREATER"] : '>', 1431 token["EQEQUAL"] : '==', 1432 token["EQUAL"] : '==', 1433 token["LESSEQUAL"] : '<=', 1434 token["GREATEREQUAL"] : '>=', 1435 token["NOTEQUAL"] : '!=', 1436 } 1437 1438 _legal_node_types = [ 1439 symbol["funcdef"], 1440 symbol["classdef"], 1441 symbol["stmt"], 1442 symbol["small_stmt"], 1443 symbol["flow_stmt"], 1444 symbol["simple_stmt"], 1445 symbol["compound_stmt"], 1446 symbol["expr_stmt"], 1447 symbol["print_stmt"], 1448 symbol["del_stmt"], 1449 symbol["pass_stmt"], 1450 symbol["break_stmt"], 1451 symbol["continue_stmt"], 1452 symbol["return_stmt"], 1453 symbol["raise_stmt"], 1454 symbol["import_stmt"], 1455 symbol["global_stmt"], 1456 symbol["exec_stmt"], 1457 symbol["assert_stmt"], 1458 symbol["if_stmt"], 1459 symbol["while_stmt"], 1460 symbol["for_stmt"], 1461 symbol["try_stmt"], 1462 symbol["with_stmt"], 1463 symbol["suite"], 1464 symbol["testlist"], 1465 symbol["testlist_safe"], 1466 symbol["test"], 1467 symbol["and_test"], 1468 symbol["not_test"], 1469 symbol["comparison"], 1470 symbol["exprlist"], 1471 symbol["expr"], 1472 symbol["xor_expr"], 1473 symbol["and_expr"], 1474 symbol["shift_expr"], 1475 symbol["arith_expr"], 1476 symbol["term"], 1477 symbol["factor"], 1478 symbol["power"], 1479 symbol["atom"], 1480 symbol["yield_stmt"], 1481 symbol["yield_expr"], 1482 ] 1483 1484 _assign_types = [ 1485 symbol["test"], 1486 symbol["or_test"], 1487 symbol["and_test"], 1488 symbol["not_test"], 1489 symbol["comparison"], 1490 symbol["expr"], 1491 symbol["xor_expr"], 1492 symbol["and_expr"], 1493 symbol["shift_expr"], 1494 symbol["arith_expr"], 1495 symbol["term"], 1496 symbol["factor"], 1497 ] 1498 1499 _names = {} 1500 for k, v in sym_name.items(): 1501 _names[k] = v 1502 for k, v in tok_name.items(): 1503 _names[k] = v 1504 1505 def debug_tree(tree): 1506 l = [] 1507 for elt in tree: 1508 if isinstance(elt, int): 1509 l.append(_names.get(elt, elt)) 1510 elif isinstance(elt, str): 1511 l.append(elt) 1512 else: 1513 l.append(debug_tree(elt)) 1514 return l