1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/compiler/transformer.py Tue Aug 30 16:51:10 2016 +0200
1.3 @@ -0,0 +1,1515 @@
1.4 +"""Parse tree transformation module.
1.5 +
1.6 +Transforms Python source code into an abstract syntax tree (AST)
1.7 +defined in the ast module.
1.8 +
1.9 +The simplest ways to invoke this module are via parse and parseFile.
1.10 +parse(buf) -> AST
1.11 +parseFile(path) -> AST
1.12 +"""
1.13 +
1.14 +# Original version written by Greg Stein (gstein@lyra.org)
1.15 +# and Bill Tutt (rassilon@lima.mudlib.org)
1.16 +# February 1997.
1.17 +#
1.18 +# Modifications and improvements for Python 2.0 by Jeremy Hylton and
1.19 +# Mark Hammond
1.20 +#
1.21 +# Some fixes to try to have correct line number on almost all nodes
1.22 +# (except Module, Discard and Stmt) added by Sylvain Thenault
1.23 +#
1.24 +# Portions of this file are:
1.25 +# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
1.26 +#
1.27 +# This module is provided under a BSD-ish license. See
1.28 +# http://www.opensource.org/licenses/bsd-license.html
1.29 +# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
1.30 +
1.31 +from compiler.ast import *
1.32 +import parser
1.33 +import symbol
1.34 +import token
1.35 +
1.36 +class WalkerError(StandardError):
1.37 + pass
1.38 +
1.39 +from compiler.consts import CO_VARARGS, CO_VARKEYWORDS
1.40 +from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY
1.41 +
1.42 +def parseFile(path):
1.43 + f = open(path, "U")
1.44 + # XXX The parser API tolerates files without a trailing newline,
1.45 + # but not strings without a trailing newline. Always add an extra
1.46 + # newline to the file contents, since we're going through the string
1.47 + # version of the API.
1.48 + src = f.read() + "\n"
1.49 + f.close()
1.50 + return parse(src)
1.51 +
1.52 +def parse(buf, mode="exec"):
1.53 + if mode == "exec" or mode == "single":
1.54 + return Transformer().parsesuite(buf)
1.55 + elif mode == "eval":
1.56 + return Transformer().parseexpr(buf)
1.57 + else:
1.58 + raise ValueError("compile() arg 3 must be"
1.59 + " 'exec' or 'eval' or 'single'")
1.60 +
1.61 +def extractLineNo(ast):
1.62 + if not isinstance(ast[1], tuple):
1.63 + # get a terminal node
1.64 + return ast[2]
1.65 + for child in ast[1:]:
1.66 + if isinstance(child, tuple):
1.67 + lineno = extractLineNo(child)
1.68 + if lineno is not None:
1.69 + return lineno
1.70 +
1.71 +def Node(*args):
1.72 + kind = args[0]
1.73 + if kind in nodes:
1.74 + try:
1.75 + return nodes[kind](*args[1:])
1.76 + except TypeError:
1.77 + print nodes[kind], len(args), args
1.78 + raise
1.79 + else:
1.80 + raise WalkerError, "Can't find appropriate Node type: %s" % str(args)
1.81 + #return apply(ast.Node, args)
1.82 +
1.83 +class Transformer:
1.84 + """Utility object for transforming Python parse trees.
1.85 +
1.86 + Exposes the following methods:
1.87 + tree = transform(ast_tree)
1.88 + tree = parsesuite(text)
1.89 + tree = parseexpr(text)
1.90 + tree = parsefile(fileob | filename)
1.91 + """
1.92 +
1.93 + def __init__(self):
1.94 + self._dispatch = {}
1.95 + for value, name in symbol.sym_name.items():
1.96 + if hasattr(self, name):
1.97 + self._dispatch[value] = getattr(self, name)
1.98 + self._dispatch[token.NEWLINE] = self.com_NEWLINE
1.99 + self._atom_dispatch = {token.LPAR: self.atom_lpar,
1.100 + token.LSQB: self.atom_lsqb,
1.101 + token.LBRACE: self.atom_lbrace,
1.102 + token.BACKQUOTE: self.atom_backquote,
1.103 + token.NUMBER: self.atom_number,
1.104 + token.STRING: self.atom_string,
1.105 + token.NAME: self.atom_name,
1.106 + }
1.107 + self.encoding = None
1.108 +
1.109 + def transform(self, tree):
1.110 + """Transform an AST into a modified parse tree."""
1.111 + if not (isinstance(tree, tuple) or isinstance(tree, list)):
1.112 + tree = parser.st2tuple(tree, line_info=1)
1.113 + return self.compile_node(tree)
1.114 +
1.115 + def parsesuite(self, text):
1.116 + """Return a modified parse tree for the given suite text."""
1.117 + return self.transform(parser.suite(text))
1.118 +
1.119 + def parseexpr(self, text):
1.120 + """Return a modified parse tree for the given expression text."""
1.121 + return self.transform(parser.expr(text))
1.122 +
1.123 + def parsefile(self, file):
1.124 + """Return a modified parse tree for the contents of the given file."""
1.125 + if type(file) == type(''):
1.126 + file = open(file)
1.127 + return self.parsesuite(file.read())
1.128 +
1.129 + # --------------------------------------------------------------
1.130 + #
1.131 + # PRIVATE METHODS
1.132 + #
1.133 +
1.134 + def compile_node(self, node):
1.135 + ### emit a line-number node?
1.136 + n = node[0]
1.137 +
1.138 + if n == symbol.encoding_decl:
1.139 + self.encoding = node[2]
1.140 + node = node[1]
1.141 + n = node[0]
1.142 +
1.143 + if n == symbol.single_input:
1.144 + return self.single_input(node[1:])
1.145 + if n == symbol.file_input:
1.146 + return self.file_input(node[1:])
1.147 + if n == symbol.eval_input:
1.148 + return self.eval_input(node[1:])
1.149 + if n == symbol.lambdef:
1.150 + return self.lambdef(node[1:])
1.151 + if n == symbol.funcdef:
1.152 + return self.funcdef(node[1:])
1.153 + if n == symbol.classdef:
1.154 + return self.classdef(node[1:])
1.155 +
1.156 + raise WalkerError, ('unexpected node type', n)
1.157 +
1.158 + def single_input(self, node):
1.159 + ### do we want to do anything about being "interactive" ?
1.160 +
1.161 + # NEWLINE | simple_stmt | compound_stmt NEWLINE
1.162 + n = node[0][0]
1.163 + if n != token.NEWLINE:
1.164 + return self.com_stmt(node[0])
1.165 +
1.166 + return Pass()
1.167 +
1.168 + def file_input(self, nodelist):
1.169 + doc = self.get_docstring(nodelist, symbol.file_input)
1.170 + if doc is not None:
1.171 + i = 1
1.172 + else:
1.173 + i = 0
1.174 + stmts = []
1.175 + for node in nodelist[i:]:
1.176 + if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
1.177 + self.com_append_stmt(stmts, node)
1.178 + return Module(doc, Stmt(stmts))
1.179 +
1.180 + def eval_input(self, nodelist):
1.181 + # from the built-in function input()
1.182 + ### is this sufficient?
1.183 + return Expression(self.com_node(nodelist[0]))
1.184 +
1.185 + def decorator_name(self, nodelist):
1.186 + listlen = len(nodelist)
1.187 + assert listlen >= 1 and listlen % 2 == 1
1.188 +
1.189 + item = self.atom_name(nodelist)
1.190 + i = 1
1.191 + while i < listlen:
1.192 + assert nodelist[i][0] == token.DOT
1.193 + assert nodelist[i + 1][0] == token.NAME
1.194 + item = Getattr(item, nodelist[i + 1][1])
1.195 + i += 2
1.196 +
1.197 + return item
1.198 +
1.199 + def decorator(self, nodelist):
1.200 + # '@' dotted_name [ '(' [arglist] ')' ]
1.201 + assert len(nodelist) in (3, 5, 6)
1.202 + assert nodelist[0][0] == token.AT
1.203 + assert nodelist[-1][0] == token.NEWLINE
1.204 +
1.205 + assert nodelist[1][0] == symbol.dotted_name
1.206 + funcname = self.decorator_name(nodelist[1][1:])
1.207 +
1.208 + if len(nodelist) > 3:
1.209 + assert nodelist[2][0] == token.LPAR
1.210 + expr = self.com_call_function(funcname, nodelist[3])
1.211 + else:
1.212 + expr = funcname
1.213 +
1.214 + return expr
1.215 +
1.216 + def decorators(self, nodelist):
1.217 + # decorators: decorator ([NEWLINE] decorator)* NEWLINE
1.218 + items = []
1.219 + for dec_nodelist in nodelist:
1.220 + assert dec_nodelist[0] == symbol.decorator
1.221 + items.append(self.decorator(dec_nodelist[1:]))
1.222 + return Decorators(items)
1.223 +
1.224 + def decorated(self, nodelist):
1.225 + assert nodelist[0][0] == symbol.decorators
1.226 + if nodelist[1][0] == symbol.funcdef:
1.227 + n = [nodelist[0]] + list(nodelist[1][1:])
1.228 + return self.funcdef(n)
1.229 + elif nodelist[1][0] == symbol.classdef:
1.230 + decorators = self.decorators(nodelist[0][1:])
1.231 + cls = self.classdef(nodelist[1][1:])
1.232 + cls.decorators = decorators
1.233 + return cls
1.234 + raise WalkerError()
1.235 +
1.236 + def funcdef(self, nodelist):
1.237 + # -6 -5 -4 -3 -2 -1
1.238 + # funcdef: [decorators] 'def' NAME parameters ':' suite
1.239 + # parameters: '(' [varargslist] ')'
1.240 +
1.241 + if len(nodelist) == 6:
1.242 + assert nodelist[0][0] == symbol.decorators
1.243 + decorators = self.decorators(nodelist[0][1:])
1.244 + else:
1.245 + assert len(nodelist) == 5
1.246 + decorators = None
1.247 +
1.248 + lineno = nodelist[-4][2]
1.249 + name = nodelist[-4][1]
1.250 + args = nodelist[-3][2]
1.251 +
1.252 + if args[0] == symbol.varargslist:
1.253 + names, defaults, flags = self.com_arglist(args[1:])
1.254 + else:
1.255 + names = defaults = ()
1.256 + flags = 0
1.257 + doc = self.get_docstring(nodelist[-1])
1.258 +
1.259 + # code for function
1.260 + code = self.com_node(nodelist[-1])
1.261 +
1.262 + if doc is not None:
1.263 + assert isinstance(code, Stmt)
1.264 + assert isinstance(code.nodes[0], Discard)
1.265 + del code.nodes[0]
1.266 + return Function(decorators, name, names, defaults, flags, doc, code,
1.267 + lineno=lineno)
1.268 +
1.269 + def lambdef(self, nodelist):
1.270 + # lambdef: 'lambda' [varargslist] ':' test
1.271 + if nodelist[2][0] == symbol.varargslist:
1.272 + names, defaults, flags = self.com_arglist(nodelist[2][1:])
1.273 + else:
1.274 + names = defaults = ()
1.275 + flags = 0
1.276 +
1.277 + # code for lambda
1.278 + code = self.com_node(nodelist[-1])
1.279 +
1.280 + return Lambda(names, defaults, flags, code, lineno=nodelist[1][2])
1.281 + old_lambdef = lambdef
1.282 +
1.283 + def classdef(self, nodelist):
1.284 + # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
1.285 +
1.286 + name = nodelist[1][1]
1.287 + doc = self.get_docstring(nodelist[-1])
1.288 + if nodelist[2][0] == token.COLON:
1.289 + bases = []
1.290 + elif nodelist[3][0] == token.RPAR:
1.291 + bases = []
1.292 + else:
1.293 + bases = self.com_bases(nodelist[3])
1.294 +
1.295 + # code for class
1.296 + code = self.com_node(nodelist[-1])
1.297 +
1.298 + if doc is not None:
1.299 + assert isinstance(code, Stmt)
1.300 + assert isinstance(code.nodes[0], Discard)
1.301 + del code.nodes[0]
1.302 +
1.303 + return Class(name, bases, doc, code, lineno=nodelist[1][2])
1.304 +
1.305 + def stmt(self, nodelist):
1.306 + return self.com_stmt(nodelist[0])
1.307 +
1.308 + small_stmt = stmt
1.309 + flow_stmt = stmt
1.310 + compound_stmt = stmt
1.311 +
1.312 + def simple_stmt(self, nodelist):
1.313 + # small_stmt (';' small_stmt)* [';'] NEWLINE
1.314 + stmts = []
1.315 + for i in range(0, len(nodelist), 2):
1.316 + self.com_append_stmt(stmts, nodelist[i])
1.317 + return Stmt(stmts)
1.318 +
1.319 + def parameters(self, nodelist):
1.320 + raise WalkerError
1.321 +
1.322 + def varargslist(self, nodelist):
1.323 + raise WalkerError
1.324 +
1.325 + def fpdef(self, nodelist):
1.326 + raise WalkerError
1.327 +
1.328 + def fplist(self, nodelist):
1.329 + raise WalkerError
1.330 +
1.331 + def dotted_name(self, nodelist):
1.332 + raise WalkerError
1.333 +
1.334 + def comp_op(self, nodelist):
1.335 + raise WalkerError
1.336 +
1.337 + def trailer(self, nodelist):
1.338 + raise WalkerError
1.339 +
1.340 + def sliceop(self, nodelist):
1.341 + raise WalkerError
1.342 +
1.343 + def argument(self, nodelist):
1.344 + raise WalkerError
1.345 +
1.346 + # --------------------------------------------------------------
1.347 + #
1.348 + # STATEMENT NODES (invoked by com_node())
1.349 + #
1.350 +
1.351 + def expr_stmt(self, nodelist):
1.352 + # augassign testlist | testlist ('=' testlist)*
1.353 + en = nodelist[-1]
1.354 + exprNode = self.lookup_node(en)(en[1:])
1.355 + if len(nodelist) == 1:
1.356 + return Discard(exprNode, lineno=exprNode.lineno)
1.357 + if nodelist[1][0] == token.EQUAL:
1.358 + nodesl = []
1.359 + for i in range(0, len(nodelist) - 2, 2):
1.360 + nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
1.361 + return Assign(nodesl, exprNode, lineno=nodelist[1][2])
1.362 + else:
1.363 + lval = self.com_augassign(nodelist[0])
1.364 + op = self.com_augassign_op(nodelist[1])
1.365 + return AugAssign(lval, op[1], exprNode, lineno=op[2])
1.366 + raise WalkerError, "can't get here"
1.367 +
1.368 + def print_stmt(self, nodelist):
1.369 + # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
1.370 + items = []
1.371 + if len(nodelist) == 1:
1.372 + start = 1
1.373 + dest = None
1.374 + elif nodelist[1][0] == token.RIGHTSHIFT:
1.375 + assert len(nodelist) == 3 \
1.376 + or nodelist[3][0] == token.COMMA
1.377 + dest = self.com_node(nodelist[2])
1.378 + start = 4
1.379 + else:
1.380 + dest = None
1.381 + start = 1
1.382 + for i in range(start, len(nodelist), 2):
1.383 + items.append(self.com_node(nodelist[i]))
1.384 + if nodelist[-1][0] == token.COMMA:
1.385 + return Print(items, dest, lineno=nodelist[0][2])
1.386 + return Printnl(items, dest, lineno=nodelist[0][2])
1.387 +
1.388 + def del_stmt(self, nodelist):
1.389 + return self.com_assign(nodelist[1], OP_DELETE)
1.390 +
1.391 + def pass_stmt(self, nodelist):
1.392 + return Pass(lineno=nodelist[0][2])
1.393 +
1.394 + def break_stmt(self, nodelist):
1.395 + return Break(lineno=nodelist[0][2])
1.396 +
1.397 + def continue_stmt(self, nodelist):
1.398 + return Continue(lineno=nodelist[0][2])
1.399 +
1.400 + def return_stmt(self, nodelist):
1.401 + # return: [testlist]
1.402 + if len(nodelist) < 2:
1.403 + return Return(Const(None), lineno=nodelist[0][2])
1.404 + return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
1.405 +
1.406 + def yield_stmt(self, nodelist):
1.407 + expr = self.com_node(nodelist[0])
1.408 + return Discard(expr, lineno=expr.lineno)
1.409 +
1.410 + def yield_expr(self, nodelist):
1.411 + if len(nodelist) > 1:
1.412 + value = self.com_node(nodelist[1])
1.413 + else:
1.414 + value = Const(None)
1.415 + return Yield(value, lineno=nodelist[0][2])
1.416 +
1.417 + def raise_stmt(self, nodelist):
1.418 + # raise: [test [',' test [',' test]]]
1.419 + if len(nodelist) > 5:
1.420 + expr3 = self.com_node(nodelist[5])
1.421 + else:
1.422 + expr3 = None
1.423 + if len(nodelist) > 3:
1.424 + expr2 = self.com_node(nodelist[3])
1.425 + else:
1.426 + expr2 = None
1.427 + if len(nodelist) > 1:
1.428 + expr1 = self.com_node(nodelist[1])
1.429 + else:
1.430 + expr1 = None
1.431 + return Raise(expr1, expr2, expr3, lineno=nodelist[0][2])
1.432 +
1.433 + def import_stmt(self, nodelist):
1.434 + # import_stmt: import_name | import_from
1.435 + assert len(nodelist) == 1
1.436 + return self.com_node(nodelist[0])
1.437 +
1.438 + def import_name(self, nodelist):
1.439 + # import_name: 'import' dotted_as_names
1.440 + return Import(self.com_dotted_as_names(nodelist[1]),
1.441 + lineno=nodelist[0][2])
1.442 +
1.443 + def import_from(self, nodelist):
1.444 + # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' |
1.445 + # '(' import_as_names ')' | import_as_names)
1.446 + assert nodelist[0][1] == 'from'
1.447 + idx = 1
1.448 + while nodelist[idx][1] == '.':
1.449 + idx += 1
1.450 + level = idx - 1
1.451 + if nodelist[idx][0] == symbol.dotted_name:
1.452 + fromname = self.com_dotted_name(nodelist[idx])
1.453 + idx += 1
1.454 + else:
1.455 + fromname = ""
1.456 + assert nodelist[idx][1] == 'import'
1.457 + if nodelist[idx + 1][0] == token.STAR:
1.458 + return From(fromname, [('*', None)], level,
1.459 + lineno=nodelist[0][2])
1.460 + else:
1.461 + node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)]
1.462 + return From(fromname, self.com_import_as_names(node), level,
1.463 + lineno=nodelist[0][2])
1.464 +
1.465 + def global_stmt(self, nodelist):
1.466 + # global: NAME (',' NAME)*
1.467 + names = []
1.468 + for i in range(1, len(nodelist), 2):
1.469 + names.append(nodelist[i][1])
1.470 + return Global(names, lineno=nodelist[0][2])
1.471 +
1.472 + def exec_stmt(self, nodelist):
1.473 + # exec_stmt: 'exec' expr ['in' expr [',' expr]]
1.474 + expr1 = self.com_node(nodelist[1])
1.475 + if len(nodelist) >= 4:
1.476 + expr2 = self.com_node(nodelist[3])
1.477 + if len(nodelist) >= 6:
1.478 + expr3 = self.com_node(nodelist[5])
1.479 + else:
1.480 + expr3 = None
1.481 + else:
1.482 + expr2 = expr3 = None
1.483 +
1.484 + return Exec(expr1, expr2, expr3, lineno=nodelist[0][2])
1.485 +
1.486 + def assert_stmt(self, nodelist):
1.487 + # 'assert': test, [',' test]
1.488 + expr1 = self.com_node(nodelist[1])
1.489 + if (len(nodelist) == 4):
1.490 + expr2 = self.com_node(nodelist[3])
1.491 + else:
1.492 + expr2 = None
1.493 + return Assert(expr1, expr2, lineno=nodelist[0][2])
1.494 +
1.495 + def if_stmt(self, nodelist):
1.496 + # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
1.497 + tests = []
1.498 + for i in range(0, len(nodelist) - 3, 4):
1.499 + testNode = self.com_node(nodelist[i + 1])
1.500 + suiteNode = self.com_node(nodelist[i + 3])
1.501 + tests.append((testNode, suiteNode))
1.502 +
1.503 + if len(nodelist) % 4 == 3:
1.504 + elseNode = self.com_node(nodelist[-1])
1.505 +## elseNode.lineno = nodelist[-1][1][2]
1.506 + else:
1.507 + elseNode = None
1.508 + return If(tests, elseNode, lineno=nodelist[0][2])
1.509 +
1.510 + def while_stmt(self, nodelist):
1.511 + # 'while' test ':' suite ['else' ':' suite]
1.512 +
1.513 + testNode = self.com_node(nodelist[1])
1.514 + bodyNode = self.com_node(nodelist[3])
1.515 +
1.516 + if len(nodelist) > 4:
1.517 + elseNode = self.com_node(nodelist[6])
1.518 + else:
1.519 + elseNode = None
1.520 +
1.521 + return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])
1.522 +
1.523 + def for_stmt(self, nodelist):
1.524 + # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
1.525 +
1.526 + assignNode = self.com_assign(nodelist[1], OP_ASSIGN)
1.527 + listNode = self.com_node(nodelist[3])
1.528 + bodyNode = self.com_node(nodelist[5])
1.529 +
1.530 + if len(nodelist) > 8:
1.531 + elseNode = self.com_node(nodelist[8])
1.532 + else:
1.533 + elseNode = None
1.534 +
1.535 + return For(assignNode, listNode, bodyNode, elseNode,
1.536 + lineno=nodelist[0][2])
1.537 +
1.538 + def try_stmt(self, nodelist):
1.539 + return self.com_try_except_finally(nodelist)
1.540 +
1.541 + def with_stmt(self, nodelist):
1.542 + return self.com_with(nodelist)
1.543 +
1.544 + def suite(self, nodelist):
1.545 + # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
1.546 + if len(nodelist) == 1:
1.547 + return self.com_stmt(nodelist[0])
1.548 +
1.549 + stmts = []
1.550 + for node in nodelist:
1.551 + if node[0] == symbol.stmt:
1.552 + self.com_append_stmt(stmts, node)
1.553 + return Stmt(stmts)
1.554 +
1.555 + # --------------------------------------------------------------
1.556 + #
1.557 + # EXPRESSION NODES (invoked by com_node())
1.558 + #
1.559 +
1.560 + def testlist(self, nodelist):
1.561 + # testlist: expr (',' expr)* [',']
1.562 + # testlist_safe: test [(',' test)+ [',']]
1.563 + # exprlist: expr (',' expr)* [',']
1.564 + return self.com_binary(Tuple, nodelist)
1.565 +
1.566 + testlist_safe = testlist # XXX
1.567 + testlist1 = testlist
1.568 + exprlist = testlist
1.569 +
1.570 + def testlist_comp(self, nodelist):
1.571 + # test ( comp_for | (',' test)* [','] )
1.572 + assert nodelist[0][0] == symbol.test
1.573 + if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for:
1.574 + test = self.com_node(nodelist[0])
1.575 + return self.com_generator_expression(test, nodelist[1])
1.576 + return self.testlist(nodelist)
1.577 +
1.578 + def test(self, nodelist):
1.579 + # or_test ['if' or_test 'else' test] | lambdef
1.580 + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
1.581 + return self.lambdef(nodelist[0])
1.582 + then = self.com_node(nodelist[0])
1.583 + if len(nodelist) > 1:
1.584 + assert len(nodelist) == 5
1.585 + assert nodelist[1][1] == 'if'
1.586 + assert nodelist[3][1] == 'else'
1.587 + test = self.com_node(nodelist[2])
1.588 + else_ = self.com_node(nodelist[4])
1.589 + return IfExp(test, then, else_, lineno=nodelist[1][2])
1.590 + return then
1.591 +
1.592 + def or_test(self, nodelist):
1.593 + # and_test ('or' and_test)* | lambdef
1.594 + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
1.595 + return self.lambdef(nodelist[0])
1.596 + return self.com_binary(Or, nodelist)
1.597 + old_test = or_test
1.598 +
1.599 + def and_test(self, nodelist):
1.600 + # not_test ('and' not_test)*
1.601 + return self.com_binary(And, nodelist)
1.602 +
1.603 + def not_test(self, nodelist):
1.604 + # 'not' not_test | comparison
1.605 + result = self.com_node(nodelist[-1])
1.606 + if len(nodelist) == 2:
1.607 + return Not(result, lineno=nodelist[0][2])
1.608 + return result
1.609 +
1.610 + def comparison(self, nodelist):
1.611 + # comparison: expr (comp_op expr)*
1.612 + node = self.com_node(nodelist[0])
1.613 + if len(nodelist) == 1:
1.614 + return node
1.615 +
1.616 + results = []
1.617 + for i in range(2, len(nodelist), 2):
1.618 + nl = nodelist[i-1]
1.619 +
1.620 + # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
1.621 + # | 'in' | 'not' 'in' | 'is' | 'is' 'not'
1.622 + n = nl[1]
1.623 + if n[0] == token.NAME:
1.624 + type = n[1]
1.625 + if len(nl) == 3:
1.626 + if type == 'not':
1.627 + type = 'not in'
1.628 + else:
1.629 + type = 'is not'
1.630 + else:
1.631 + type = _cmp_types[n[0]]
1.632 +
1.633 + lineno = nl[1][2]
1.634 + results.append((type, self.com_node(nodelist[i])))
1.635 +
1.636 + # we need a special "compare" node so that we can distinguish
1.637 + # 3 < x < 5 from (3 < x) < 5
1.638 + # the two have very different semantics and results (note that the
1.639 + # latter form is always true)
1.640 +
1.641 + return Compare(node, results, lineno=lineno)
1.642 +
1.643 + def expr(self, nodelist):
1.644 + # xor_expr ('|' xor_expr)*
1.645 + return self.com_binary(Bitor, nodelist)
1.646 +
1.647 + def xor_expr(self, nodelist):
1.648 + # xor_expr ('^' xor_expr)*
1.649 + return self.com_binary(Bitxor, nodelist)
1.650 +
1.651 + def and_expr(self, nodelist):
1.652 + # xor_expr ('&' xor_expr)*
1.653 + return self.com_binary(Bitand, nodelist)
1.654 +
1.655 + def shift_expr(self, nodelist):
1.656 + # shift_expr ('<<'|'>>' shift_expr)*
1.657 + node = self.com_node(nodelist[0])
1.658 + for i in range(2, len(nodelist), 2):
1.659 + right = self.com_node(nodelist[i])
1.660 + if nodelist[i-1][0] == token.LEFTSHIFT:
1.661 + node = LeftShift([node, right], lineno=nodelist[1][2])
1.662 + elif nodelist[i-1][0] == token.RIGHTSHIFT:
1.663 + node = RightShift([node, right], lineno=nodelist[1][2])
1.664 + else:
1.665 + raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
1.666 + return node
1.667 +
1.668 + def arith_expr(self, nodelist):
1.669 + node = self.com_node(nodelist[0])
1.670 + for i in range(2, len(nodelist), 2):
1.671 + right = self.com_node(nodelist[i])
1.672 + if nodelist[i-1][0] == token.PLUS:
1.673 + node = Add([node, right], lineno=nodelist[1][2])
1.674 + elif nodelist[i-1][0] == token.MINUS:
1.675 + node = Sub([node, right], lineno=nodelist[1][2])
1.676 + else:
1.677 + raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
1.678 + return node
1.679 +
1.680 + def term(self, nodelist):
1.681 + node = self.com_node(nodelist[0])
1.682 + for i in range(2, len(nodelist), 2):
1.683 + right = self.com_node(nodelist[i])
1.684 + t = nodelist[i-1][0]
1.685 + if t == token.STAR:
1.686 + node = Mul([node, right])
1.687 + elif t == token.SLASH:
1.688 + node = Div([node, right])
1.689 + elif t == token.PERCENT:
1.690 + node = Mod([node, right])
1.691 + elif t == token.DOUBLESLASH:
1.692 + node = FloorDiv([node, right])
1.693 + else:
1.694 + raise ValueError, "unexpected token: %s" % t
1.695 + node.lineno = nodelist[1][2]
1.696 + return node
1.697 +
1.698 + def factor(self, nodelist):
1.699 + elt = nodelist[0]
1.700 + t = elt[0]
1.701 + node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
1.702 + # need to handle (unary op)constant here...
1.703 + if t == token.PLUS:
1.704 + return UnaryAdd(node, lineno=elt[2])
1.705 + elif t == token.MINUS:
1.706 + return UnarySub(node, lineno=elt[2])
1.707 + elif t == token.TILDE:
1.708 + node = Invert(node, lineno=elt[2])
1.709 + return node
1.710 +
1.711 + def power(self, nodelist):
1.712 + # power: atom trailer* ('**' factor)*
1.713 + node = self.com_node(nodelist[0])
1.714 + for i in range(1, len(nodelist)):
1.715 + elt = nodelist[i]
1.716 + if elt[0] == token.DOUBLESTAR:
1.717 + return Power([node, self.com_node(nodelist[i+1])],
1.718 + lineno=elt[2])
1.719 +
1.720 + node = self.com_apply_trailer(node, elt)
1.721 +
1.722 + return node
1.723 +
1.724 + def atom(self, nodelist):
1.725 + return self._atom_dispatch[nodelist[0][0]](nodelist)
1.726 +
1.727 + def atom_lpar(self, nodelist):
1.728 + if nodelist[1][0] == token.RPAR:
1.729 + return Tuple((), lineno=nodelist[0][2])
1.730 + return self.com_node(nodelist[1])
1.731 +
1.732 + def atom_lsqb(self, nodelist):
1.733 + if nodelist[1][0] == token.RSQB:
1.734 + return List((), lineno=nodelist[0][2])
1.735 + return self.com_list_constructor(nodelist[1])
1.736 +
1.737 + def atom_lbrace(self, nodelist):
1.738 + if nodelist[1][0] == token.RBRACE:
1.739 + return Dict((), lineno=nodelist[0][2])
1.740 + return self.com_dictorsetmaker(nodelist[1])
1.741 +
1.742 + def atom_backquote(self, nodelist):
1.743 + return Backquote(self.com_node(nodelist[1]))
1.744 +
1.745 + def atom_number(self, nodelist):
1.746 + ### need to verify this matches compile.c
1.747 + k = eval(nodelist[0][1])
1.748 + return Const(k, lineno=nodelist[0][2])
1.749 +
1.750 + def decode_literal(self, lit):
1.751 + if self.encoding:
1.752 + # this is particularly fragile & a bit of a
1.753 + # hack... changes in compile.c:parsestr and
1.754 + # tokenizer.c must be reflected here.
1.755 + if self.encoding not in ['utf-8', 'iso-8859-1']:
1.756 + lit = unicode(lit, 'utf-8').encode(self.encoding)
1.757 + return eval("# coding: %s\n%s" % (self.encoding, lit))
1.758 + else:
1.759 + return eval(lit)
1.760 +
1.761 + def atom_string(self, nodelist):
1.762 + k = ''
1.763 + for node in nodelist:
1.764 + k += self.decode_literal(node[1])
1.765 + return Const(k, lineno=nodelist[0][2])
1.766 +
1.767 + def atom_name(self, nodelist):
1.768 + return Name(nodelist[0][1], lineno=nodelist[0][2])
1.769 +
1.770 + # --------------------------------------------------------------
1.771 + #
1.772 + # INTERNAL PARSING UTILITIES
1.773 + #
1.774 +
1.775 + # The use of com_node() introduces a lot of extra stack frames,
1.776 + # enough to cause a stack overflow compiling test.test_parser with
1.777 + # the standard interpreter recursionlimit. The com_node() is a
1.778 + # convenience function that hides the dispatch details, but comes
1.779 + # at a very high cost. It is more efficient to dispatch directly
1.780 + # in the callers. In these cases, use lookup_node() and call the
1.781 + # dispatched node directly.
1.782 +
1.783 + def lookup_node(self, node):
1.784 + return self._dispatch[node[0]]
1.785 +
1.786 + def com_node(self, node):
1.787 + # Note: compile.c has handling in com_node for del_stmt, pass_stmt,
1.788 + # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
1.789 + # and compound_stmt.
1.790 + # We'll just dispatch them.
1.791 + return self._dispatch[node[0]](node[1:])
1.792 +
1.793 + def com_NEWLINE(self, *args):
1.794 + # A ';' at the end of a line can make a NEWLINE token appear
1.795 + # here, Render it harmless. (genc discards ('discard',
1.796 + # ('const', xxxx)) Nodes)
1.797 + return Discard(Const(None))
1.798 +
1.799 + def com_arglist(self, nodelist):
1.800 + # varargslist:
1.801 + # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME)
1.802 + # | fpdef ['=' test] (',' fpdef ['=' test])* [',']
1.803 + # fpdef: NAME | '(' fplist ')'
1.804 + # fplist: fpdef (',' fpdef)* [',']
1.805 + names = []
1.806 + defaults = []
1.807 + flags = 0
1.808 +
1.809 + i = 0
1.810 + while i < len(nodelist):
1.811 + node = nodelist[i]
1.812 + if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
1.813 + if node[0] == token.STAR:
1.814 + node = nodelist[i+1]
1.815 + if node[0] == token.NAME:
1.816 + names.append(node[1])
1.817 + flags = flags | CO_VARARGS
1.818 + i = i + 3
1.819 +
1.820 + if i < len(nodelist):
1.821 + # should be DOUBLESTAR
1.822 + t = nodelist[i][0]
1.823 + if t == token.DOUBLESTAR:
1.824 + node = nodelist[i+1]
1.825 + else:
1.826 + raise ValueError, "unexpected token: %s" % t
1.827 + names.append(node[1])
1.828 + flags = flags | CO_VARKEYWORDS
1.829 +
1.830 + break
1.831 +
1.832 + # fpdef: NAME | '(' fplist ')'
1.833 + names.append(self.com_fpdef(node))
1.834 +
1.835 + i = i + 1
1.836 + if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
1.837 + defaults.append(self.com_node(nodelist[i + 1]))
1.838 + i = i + 2
1.839 + elif len(defaults):
1.840 + # we have already seen an argument with default, but here
1.841 + # came one without
1.842 + raise SyntaxError, "non-default argument follows default argument"
1.843 +
1.844 + # skip the comma
1.845 + i = i + 1
1.846 +
1.847 + return names, defaults, flags
1.848 +
1.849 + def com_fpdef(self, node):
1.850 + # fpdef: NAME | '(' fplist ')'
1.851 + if node[1][0] == token.LPAR:
1.852 + return self.com_fplist(node[2])
1.853 + return node[1][1]
1.854 +
1.855 + def com_fplist(self, node):
1.856 + # fplist: fpdef (',' fpdef)* [',']
1.857 + if len(node) == 2:
1.858 + return self.com_fpdef(node[1])
1.859 + list = []
1.860 + for i in range(1, len(node), 2):
1.861 + list.append(self.com_fpdef(node[i]))
1.862 + return tuple(list)
1.863 +
1.864 + def com_dotted_name(self, node):
1.865 + # String together the dotted names and return the string
1.866 + name = ""
1.867 + for n in node:
1.868 + if type(n) == type(()) and n[0] == 1:
1.869 + name = name + n[1] + '.'
1.870 + return name[:-1]
1.871 +
1.872 + def com_dotted_as_name(self, node):
1.873 + assert node[0] == symbol.dotted_as_name
1.874 + node = node[1:]
1.875 + dot = self.com_dotted_name(node[0][1:])
1.876 + if len(node) == 1:
1.877 + return dot, None
1.878 + assert node[1][1] == 'as'
1.879 + assert node[2][0] == token.NAME
1.880 + return dot, node[2][1]
1.881 +
1.882 + def com_dotted_as_names(self, node):
1.883 + assert node[0] == symbol.dotted_as_names
1.884 + node = node[1:]
1.885 + names = [self.com_dotted_as_name(node[0])]
1.886 + for i in range(2, len(node), 2):
1.887 + names.append(self.com_dotted_as_name(node[i]))
1.888 + return names
1.889 +
1.890 + def com_import_as_name(self, node):
1.891 + assert node[0] == symbol.import_as_name
1.892 + node = node[1:]
1.893 + assert node[0][0] == token.NAME
1.894 + if len(node) == 1:
1.895 + return node[0][1], None
1.896 + assert node[1][1] == 'as', node
1.897 + assert node[2][0] == token.NAME
1.898 + return node[0][1], node[2][1]
1.899 +
1.900 + def com_import_as_names(self, node):
1.901 + assert node[0] == symbol.import_as_names
1.902 + node = node[1:]
1.903 + names = [self.com_import_as_name(node[0])]
1.904 + for i in range(2, len(node), 2):
1.905 + names.append(self.com_import_as_name(node[i]))
1.906 + return names
1.907 +
1.908 + def com_bases(self, node):
1.909 + bases = []
1.910 + for i in range(1, len(node), 2):
1.911 + bases.append(self.com_node(node[i]))
1.912 + return bases
1.913 +
1.914 + def com_try_except_finally(self, nodelist):
1.915 + # ('try' ':' suite
1.916 + # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite]
1.917 + # | 'finally' ':' suite))
1.918 +
1.919 + if nodelist[3][0] == token.NAME:
1.920 + # first clause is a finally clause: only try-finally
1.921 + return TryFinally(self.com_node(nodelist[2]),
1.922 + self.com_node(nodelist[5]),
1.923 + lineno=nodelist[0][2])
1.924 +
1.925 + #tryexcept: [TryNode, [except_clauses], elseNode)]
1.926 + clauses = []
1.927 + elseNode = None
1.928 + finallyNode = None
1.929 + for i in range(3, len(nodelist), 3):
1.930 + node = nodelist[i]
1.931 + if node[0] == symbol.except_clause:
1.932 + # except_clause: 'except' [expr [(',' | 'as') expr]] */
1.933 + if len(node) > 2:
1.934 + expr1 = self.com_node(node[2])
1.935 + if len(node) > 4:
1.936 + expr2 = self.com_assign(node[4], OP_ASSIGN)
1.937 + else:
1.938 + expr2 = None
1.939 + else:
1.940 + expr1 = expr2 = None
1.941 + clauses.append((expr1, expr2, self.com_node(nodelist[i+2])))
1.942 +
1.943 + if node[0] == token.NAME:
1.944 + if node[1] == 'else':
1.945 + elseNode = self.com_node(nodelist[i+2])
1.946 + elif node[1] == 'finally':
1.947 + finallyNode = self.com_node(nodelist[i+2])
1.948 + try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode,
1.949 + lineno=nodelist[0][2])
1.950 + if finallyNode:
1.951 + return TryFinally(try_except, finallyNode, lineno=nodelist[0][2])
1.952 + else:
1.953 + return try_except
1.954 +
1.955 + def com_with(self, nodelist):
1.956 + # with_stmt: 'with' with_item (',' with_item)* ':' suite
1.957 + body = self.com_node(nodelist[-1])
1.958 + for i in range(len(nodelist) - 3, 0, -2):
1.959 + ret = self.com_with_item(nodelist[i], body, nodelist[0][2])
1.960 + if i == 1:
1.961 + return ret
1.962 + body = ret
1.963 +
1.964 + def com_with_item(self, nodelist, body, lineno):
1.965 + # with_item: test ['as' expr]
1.966 + if len(nodelist) == 4:
1.967 + var = self.com_assign(nodelist[3], OP_ASSIGN)
1.968 + else:
1.969 + var = None
1.970 + expr = self.com_node(nodelist[1])
1.971 + return With(expr, var, body, lineno=lineno)
1.972 +
1.973 + def com_augassign_op(self, node):
1.974 + assert node[0] == symbol.augassign
1.975 + return node[1]
1.976 +
1.977 + def com_augassign(self, node):
1.978 + """Return node suitable for lvalue of augmented assignment
1.979 +
1.980 + Names, slices, and attributes are the only allowable nodes.
1.981 + """
1.982 + l = self.com_node(node)
1.983 + if l.__class__ in (Name, Slice, Subscript, Getattr):
1.984 + return l
1.985 + raise SyntaxError, "can't assign to %s" % l.__class__.__name__
1.986 +
1.987 + def com_assign(self, node, assigning):
1.988 + # return a node suitable for use as an "lvalue"
1.989 + # loop to avoid trivial recursion
1.990 + while 1:
1.991 + t = node[0]
1.992 + if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp):
1.993 + if len(node) > 2:
1.994 + return self.com_assign_tuple(node, assigning)
1.995 + node = node[1]
1.996 + elif t in _assign_types:
1.997 + if len(node) > 2:
1.998 + raise SyntaxError, "can't assign to operator"
1.999 + node = node[1]
1.1000 + elif t == symbol.power:
1.1001 + if node[1][0] != symbol.atom:
1.1002 + raise SyntaxError, "can't assign to operator"
1.1003 + if len(node) > 2:
1.1004 + primary = self.com_node(node[1])
1.1005 + for i in range(2, len(node)-1):
1.1006 + ch = node[i]
1.1007 + if ch[0] == token.DOUBLESTAR:
1.1008 + raise SyntaxError, "can't assign to operator"
1.1009 + primary = self.com_apply_trailer(primary, ch)
1.1010 + return self.com_assign_trailer(primary, node[-1],
1.1011 + assigning)
1.1012 + node = node[1]
1.1013 + elif t == symbol.atom:
1.1014 + t = node[1][0]
1.1015 + if t == token.LPAR:
1.1016 + node = node[2]
1.1017 + if node[0] == token.RPAR:
1.1018 + raise SyntaxError, "can't assign to ()"
1.1019 + elif t == token.LSQB:
1.1020 + node = node[2]
1.1021 + if node[0] == token.RSQB:
1.1022 + raise SyntaxError, "can't assign to []"
1.1023 + return self.com_assign_list(node, assigning)
1.1024 + elif t == token.NAME:
1.1025 + return self.com_assign_name(node[1], assigning)
1.1026 + else:
1.1027 + raise SyntaxError, "can't assign to literal"
1.1028 + else:
1.1029 + raise SyntaxError, "bad assignment (%s)" % t
1.1030 +
1.1031 + def com_assign_tuple(self, node, assigning):
1.1032 + assigns = []
1.1033 + for i in range(1, len(node), 2):
1.1034 + assigns.append(self.com_assign(node[i], assigning))
1.1035 + return AssTuple(assigns, lineno=extractLineNo(node))
1.1036 +
1.1037 + def com_assign_list(self, node, assigning):
1.1038 + assigns = []
1.1039 + for i in range(1, len(node), 2):
1.1040 + if i + 1 < len(node):
1.1041 + if node[i + 1][0] == symbol.list_for:
1.1042 + raise SyntaxError, "can't assign to list comprehension"
1.1043 + assert node[i + 1][0] == token.COMMA, node[i + 1]
1.1044 + assigns.append(self.com_assign(node[i], assigning))
1.1045 + return AssList(assigns, lineno=extractLineNo(node))
1.1046 +
1.1047 + def com_assign_name(self, node, assigning):
1.1048 + return AssName(node[1], assigning, lineno=node[2])
1.1049 +
1.1050 + def com_assign_trailer(self, primary, node, assigning):
1.1051 + t = node[1][0]
1.1052 + if t == token.DOT:
1.1053 + return self.com_assign_attr(primary, node[2], assigning)
1.1054 + if t == token.LSQB:
1.1055 + return self.com_subscriptlist(primary, node[2], assigning)
1.1056 + if t == token.LPAR:
1.1057 + raise SyntaxError, "can't assign to function call"
1.1058 + raise SyntaxError, "unknown trailer type: %s" % t
1.1059 +
1.1060 + def com_assign_attr(self, primary, node, assigning):
1.1061 + return AssAttr(primary, node[1], assigning, lineno=node[-1])
1.1062 +
1.1063 + def com_binary(self, constructor, nodelist):
1.1064 + "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
1.1065 + l = len(nodelist)
1.1066 + if l == 1:
1.1067 + n = nodelist[0]
1.1068 + return self.lookup_node(n)(n[1:])
1.1069 + items = []
1.1070 + for i in range(0, l, 2):
1.1071 + n = nodelist[i]
1.1072 + items.append(self.lookup_node(n)(n[1:]))
1.1073 + return constructor(items, lineno=extractLineNo(nodelist))
1.1074 +
1.1075 + def com_stmt(self, node):
1.1076 + result = self.lookup_node(node)(node[1:])
1.1077 + assert result is not None
1.1078 + if isinstance(result, Stmt):
1.1079 + return result
1.1080 + return Stmt([result])
1.1081 +
1.1082 + def com_append_stmt(self, stmts, node):
1.1083 + result = self.lookup_node(node)(node[1:])
1.1084 + assert result is not None
1.1085 + if isinstance(result, Stmt):
1.1086 + stmts.extend(result.nodes)
1.1087 + else:
1.1088 + stmts.append(result)
1.1089 +
1.1090 + def com_list_constructor(self, nodelist):
1.1091 + # listmaker: test ( list_for | (',' test)* [','] )
1.1092 + values = []
1.1093 + for i in range(1, len(nodelist)):
1.1094 + if nodelist[i][0] == symbol.list_for:
1.1095 + assert len(nodelist[i:]) == 1
1.1096 + return self.com_list_comprehension(values[0],
1.1097 + nodelist[i])
1.1098 + elif nodelist[i][0] == token.COMMA:
1.1099 + continue
1.1100 + values.append(self.com_node(nodelist[i]))
1.1101 + return List(values, lineno=values[0].lineno)
1.1102 +
1.1103 + def com_list_comprehension(self, expr, node):
1.1104 + return self.com_comprehension(expr, None, node, 'list')
1.1105 +
1.1106 + def com_comprehension(self, expr1, expr2, node, type):
1.1107 + # list_iter: list_for | list_if
1.1108 + # list_for: 'for' exprlist 'in' testlist [list_iter]
1.1109 + # list_if: 'if' test [list_iter]
1.1110 +
1.1111 + # XXX should raise SyntaxError for assignment
1.1112 + # XXX(avassalotti) Set and dict comprehensions should have generator
1.1113 + # semantics. In other words, they shouldn't leak
1.1114 + # variables outside of the comprehension's scope.
1.1115 +
1.1116 + lineno = node[1][2]
1.1117 + fors = []
1.1118 + while node:
1.1119 + t = node[1][1]
1.1120 + if t == 'for':
1.1121 + assignNode = self.com_assign(node[2], OP_ASSIGN)
1.1122 + compNode = self.com_node(node[4])
1.1123 + newfor = ListCompFor(assignNode, compNode, [])
1.1124 + newfor.lineno = node[1][2]
1.1125 + fors.append(newfor)
1.1126 + if len(node) == 5:
1.1127 + node = None
1.1128 + elif type == 'list':
1.1129 + node = self.com_list_iter(node[5])
1.1130 + else:
1.1131 + node = self.com_comp_iter(node[5])
1.1132 + elif t == 'if':
1.1133 + test = self.com_node(node[2])
1.1134 + newif = ListCompIf(test, lineno=node[1][2])
1.1135 + newfor.ifs.append(newif)
1.1136 + if len(node) == 3:
1.1137 + node = None
1.1138 + elif type == 'list':
1.1139 + node = self.com_list_iter(node[3])
1.1140 + else:
1.1141 + node = self.com_comp_iter(node[3])
1.1142 + else:
1.1143 + raise SyntaxError, \
1.1144 + ("unexpected comprehension element: %s %d"
1.1145 + % (node, lineno))
1.1146 + if type == 'list':
1.1147 + return ListComp(expr1, fors, lineno=lineno)
1.1148 + elif type == 'set':
1.1149 + return SetComp(expr1, fors, lineno=lineno)
1.1150 + elif type == 'dict':
1.1151 + return DictComp(expr1, expr2, fors, lineno=lineno)
1.1152 + else:
1.1153 + raise ValueError("unexpected comprehension type: " + repr(type))
1.1154 +
1.1155 + def com_list_iter(self, node):
1.1156 + assert node[0] == symbol.list_iter
1.1157 + return node[1]
1.1158 +
1.1159 + def com_comp_iter(self, node):
1.1160 + assert node[0] == symbol.comp_iter
1.1161 + return node[1]
1.1162 +
1.1163 + def com_generator_expression(self, expr, node):
1.1164 + # comp_iter: comp_for | comp_if
1.1165 + # comp_for: 'for' exprlist 'in' test [comp_iter]
1.1166 + # comp_if: 'if' test [comp_iter]
1.1167 +
1.1168 + lineno = node[1][2]
1.1169 + fors = []
1.1170 + while node:
1.1171 + t = node[1][1]
1.1172 + if t == 'for':
1.1173 + assignNode = self.com_assign(node[2], OP_ASSIGN)
1.1174 + genNode = self.com_node(node[4])
1.1175 + newfor = GenExprFor(assignNode, genNode, [],
1.1176 + lineno=node[1][2])
1.1177 + fors.append(newfor)
1.1178 + if (len(node)) == 5:
1.1179 + node = None
1.1180 + else:
1.1181 + node = self.com_comp_iter(node[5])
1.1182 + elif t == 'if':
1.1183 + test = self.com_node(node[2])
1.1184 + newif = GenExprIf(test, lineno=node[1][2])
1.1185 + newfor.ifs.append(newif)
1.1186 + if len(node) == 3:
1.1187 + node = None
1.1188 + else:
1.1189 + node = self.com_comp_iter(node[3])
1.1190 + else:
1.1191 + raise SyntaxError, \
1.1192 + ("unexpected generator expression element: %s %d"
1.1193 + % (node, lineno))
1.1194 + fors[0].is_outmost = True
1.1195 + return GenExpr(GenExprInner(expr, fors), lineno=lineno)
1.1196 +
1.1197 + def com_dictorsetmaker(self, nodelist):
1.1198 + # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
1.1199 + # (test (comp_for | (',' test)* [','])) )
1.1200 + assert nodelist[0] == symbol.dictorsetmaker
1.1201 + nodelist = nodelist[1:]
1.1202 + if len(nodelist) == 1 or nodelist[1][0] == token.COMMA:
1.1203 + # set literal
1.1204 + items = []
1.1205 + for i in range(0, len(nodelist), 2):
1.1206 + items.append(self.com_node(nodelist[i]))
1.1207 + return Set(items, lineno=items[0].lineno)
1.1208 + elif nodelist[1][0] == symbol.comp_for:
1.1209 + # set comprehension
1.1210 + expr = self.com_node(nodelist[0])
1.1211 + return self.com_comprehension(expr, None, nodelist[1], 'set')
1.1212 + elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for:
1.1213 + # dict comprehension
1.1214 + assert nodelist[1][0] == token.COLON
1.1215 + key = self.com_node(nodelist[0])
1.1216 + value = self.com_node(nodelist[2])
1.1217 + return self.com_comprehension(key, value, nodelist[3], 'dict')
1.1218 + else:
1.1219 + # dict literal
1.1220 + items = []
1.1221 + for i in range(0, len(nodelist), 4):
1.1222 + items.append((self.com_node(nodelist[i]),
1.1223 + self.com_node(nodelist[i+2])))
1.1224 + return Dict(items, lineno=items[0][0].lineno)
1.1225 +
1.1226 + def com_apply_trailer(self, primaryNode, nodelist):
1.1227 + t = nodelist[1][0]
1.1228 + if t == token.LPAR:
1.1229 + return self.com_call_function(primaryNode, nodelist[2])
1.1230 + if t == token.DOT:
1.1231 + return self.com_select_member(primaryNode, nodelist[2])
1.1232 + if t == token.LSQB:
1.1233 + return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY)
1.1234 +
1.1235 + raise SyntaxError, 'unknown node type: %s' % t
1.1236 +
1.1237 + def com_select_member(self, primaryNode, nodelist):
1.1238 + if nodelist[0] != token.NAME:
1.1239 + raise SyntaxError, "member must be a name"
1.1240 + return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
1.1241 +
1.1242 + def com_call_function(self, primaryNode, nodelist):
1.1243 + if nodelist[0] == token.RPAR:
1.1244 + return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
1.1245 + args = []
1.1246 + kw = 0
1.1247 + star_node = dstar_node = None
1.1248 + len_nodelist = len(nodelist)
1.1249 + i = 1
1.1250 + while i < len_nodelist:
1.1251 + node = nodelist[i]
1.1252 +
1.1253 + if node[0]==token.STAR:
1.1254 + if star_node is not None:
1.1255 + raise SyntaxError, 'already have the varargs indentifier'
1.1256 + star_node = self.com_node(nodelist[i+1])
1.1257 + i = i + 3
1.1258 + continue
1.1259 + elif node[0]==token.DOUBLESTAR:
1.1260 + if dstar_node is not None:
1.1261 + raise SyntaxError, 'already have the kwargs indentifier'
1.1262 + dstar_node = self.com_node(nodelist[i+1])
1.1263 + i = i + 3
1.1264 + continue
1.1265 +
1.1266 + # positional or named parameters
1.1267 + kw, result = self.com_argument(node, kw, star_node)
1.1268 +
1.1269 + if len_nodelist != 2 and isinstance(result, GenExpr) \
1.1270 + and len(node) == 3 and node[2][0] == symbol.comp_for:
1.1271 + # allow f(x for x in y), but reject f(x for x in y, 1)
1.1272 + # should use f((x for x in y), 1) instead of f(x for x in y, 1)
1.1273 + raise SyntaxError, 'generator expression needs parenthesis'
1.1274 +
1.1275 + args.append(result)
1.1276 + i = i + 2
1.1277 +
1.1278 + return CallFunc(primaryNode, args, star_node, dstar_node,
1.1279 + lineno=extractLineNo(nodelist))
1.1280 +
1.1281 + def com_argument(self, nodelist, kw, star_node):
1.1282 + if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for:
1.1283 + test = self.com_node(nodelist[1])
1.1284 + return 0, self.com_generator_expression(test, nodelist[2])
1.1285 + if len(nodelist) == 2:
1.1286 + if kw:
1.1287 + raise SyntaxError, "non-keyword arg after keyword arg"
1.1288 + if star_node:
1.1289 + raise SyntaxError, "only named arguments may follow *expression"
1.1290 + return 0, self.com_node(nodelist[1])
1.1291 + result = self.com_node(nodelist[3])
1.1292 + n = nodelist[1]
1.1293 + while len(n) == 2 and n[0] != token.NAME:
1.1294 + n = n[1]
1.1295 + if n[0] != token.NAME:
1.1296 + raise SyntaxError, "keyword can't be an expression (%s)"%n[0]
1.1297 + node = Keyword(n[1], result, lineno=n[2])
1.1298 + return 1, node
1.1299 +
1.1300 + def com_subscriptlist(self, primary, nodelist, assigning):
1.1301 + # slicing: simple_slicing | extended_slicing
1.1302 + # simple_slicing: primary "[" short_slice "]"
1.1303 + # extended_slicing: primary "[" slice_list "]"
1.1304 + # slice_list: slice_item ("," slice_item)* [","]
1.1305 +
1.1306 + # backwards compat slice for '[i:j]'
1.1307 + if len(nodelist) == 2:
1.1308 + sub = nodelist[1]
1.1309 + if (sub[1][0] == token.COLON or \
1.1310 + (len(sub) > 2 and sub[2][0] == token.COLON)) and \
1.1311 + sub[-1][0] != symbol.sliceop:
1.1312 + return self.com_slice(primary, sub, assigning)
1.1313 +
1.1314 + subscripts = []
1.1315 + for i in range(1, len(nodelist), 2):
1.1316 + subscripts.append(self.com_subscript(nodelist[i]))
1.1317 + return Subscript(primary, assigning, subscripts,
1.1318 + lineno=extractLineNo(nodelist))
1.1319 +
1.1320 + def com_subscript(self, node):
1.1321 + # slice_item: expression | proper_slice | ellipsis
1.1322 + ch = node[1]
1.1323 + t = ch[0]
1.1324 + if t == token.DOT and node[2][0] == token.DOT:
1.1325 + return Ellipsis()
1.1326 + if t == token.COLON or len(node) > 2:
1.1327 + return self.com_sliceobj(node)
1.1328 + return self.com_node(ch)
1.1329 +
1.1330 + def com_sliceobj(self, node):
1.1331 + # proper_slice: short_slice | long_slice
1.1332 + # short_slice: [lower_bound] ":" [upper_bound]
1.1333 + # long_slice: short_slice ":" [stride]
1.1334 + # lower_bound: expression
1.1335 + # upper_bound: expression
1.1336 + # stride: expression
1.1337 + #
1.1338 + # Note: a stride may be further slicing...
1.1339 +
1.1340 + items = []
1.1341 +
1.1342 + if node[1][0] == token.COLON:
1.1343 + items.append(Const(None))
1.1344 + i = 2
1.1345 + else:
1.1346 + items.append(self.com_node(node[1]))
1.1347 + # i == 2 is a COLON
1.1348 + i = 3
1.1349 +
1.1350 + if i < len(node) and node[i][0] == symbol.test:
1.1351 + items.append(self.com_node(node[i]))
1.1352 + i = i + 1
1.1353 + else:
1.1354 + items.append(Const(None))
1.1355 +
1.1356 + # a short_slice has been built. look for long_slice now by looking
1.1357 + # for strides...
1.1358 + for j in range(i, len(node)):
1.1359 + ch = node[j]
1.1360 + if len(ch) == 2:
1.1361 + items.append(Const(None))
1.1362 + else:
1.1363 + items.append(self.com_node(ch[2]))
1.1364 + return Sliceobj(items, lineno=extractLineNo(node))
1.1365 +
1.1366 + def com_slice(self, primary, node, assigning):
1.1367 + # short_slice: [lower_bound] ":" [upper_bound]
1.1368 + lower = upper = None
1.1369 + if len(node) == 3:
1.1370 + if node[1][0] == token.COLON:
1.1371 + upper = self.com_node(node[2])
1.1372 + else:
1.1373 + lower = self.com_node(node[1])
1.1374 + elif len(node) == 4:
1.1375 + lower = self.com_node(node[1])
1.1376 + upper = self.com_node(node[3])
1.1377 + return Slice(primary, assigning, lower, upper,
1.1378 + lineno=extractLineNo(node))
1.1379 +
1.1380 + def get_docstring(self, node, n=None):
1.1381 + if n is None:
1.1382 + n = node[0]
1.1383 + node = node[1:]
1.1384 + if n == symbol.suite:
1.1385 + if len(node) == 1:
1.1386 + return self.get_docstring(node[0])
1.1387 + for sub in node:
1.1388 + if sub[0] == symbol.stmt:
1.1389 + return self.get_docstring(sub)
1.1390 + return None
1.1391 + if n == symbol.file_input:
1.1392 + for sub in node:
1.1393 + if sub[0] == symbol.stmt:
1.1394 + return self.get_docstring(sub)
1.1395 + return None
1.1396 + if n == symbol.atom:
1.1397 + if node[0][0] == token.STRING:
1.1398 + s = ''
1.1399 + for t in node:
1.1400 + s = s + eval(t[1])
1.1401 + return s
1.1402 + return None
1.1403 + if n == symbol.stmt or n == symbol.simple_stmt \
1.1404 + or n == symbol.small_stmt:
1.1405 + return self.get_docstring(node[0])
1.1406 + if n in _doc_nodes and len(node) == 1:
1.1407 + return self.get_docstring(node[0])
1.1408 + return None
1.1409 +
1.1410 +
1.1411 +_doc_nodes = [
1.1412 + symbol.expr_stmt,
1.1413 + symbol.testlist,
1.1414 + symbol.testlist_safe,
1.1415 + symbol.test,
1.1416 + symbol.or_test,
1.1417 + symbol.and_test,
1.1418 + symbol.not_test,
1.1419 + symbol.comparison,
1.1420 + symbol.expr,
1.1421 + symbol.xor_expr,
1.1422 + symbol.and_expr,
1.1423 + symbol.shift_expr,
1.1424 + symbol.arith_expr,
1.1425 + symbol.term,
1.1426 + symbol.factor,
1.1427 + symbol.power,
1.1428 + ]
1.1429 +
1.1430 +# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
1.1431 +# | 'in' | 'not' 'in' | 'is' | 'is' 'not'
1.1432 +_cmp_types = {
1.1433 + token.LESS : '<',
1.1434 + token.GREATER : '>',
1.1435 + token.EQEQUAL : '==',
1.1436 + token.EQUAL : '==',
1.1437 + token.LESSEQUAL : '<=',
1.1438 + token.GREATEREQUAL : '>=',
1.1439 + token.NOTEQUAL : '!=',
1.1440 + }
1.1441 +
1.1442 +_legal_node_types = [
1.1443 + symbol.funcdef,
1.1444 + symbol.classdef,
1.1445 + symbol.stmt,
1.1446 + symbol.small_stmt,
1.1447 + symbol.flow_stmt,
1.1448 + symbol.simple_stmt,
1.1449 + symbol.compound_stmt,
1.1450 + symbol.expr_stmt,
1.1451 + symbol.print_stmt,
1.1452 + symbol.del_stmt,
1.1453 + symbol.pass_stmt,
1.1454 + symbol.break_stmt,
1.1455 + symbol.continue_stmt,
1.1456 + symbol.return_stmt,
1.1457 + symbol.raise_stmt,
1.1458 + symbol.import_stmt,
1.1459 + symbol.global_stmt,
1.1460 + symbol.exec_stmt,
1.1461 + symbol.assert_stmt,
1.1462 + symbol.if_stmt,
1.1463 + symbol.while_stmt,
1.1464 + symbol.for_stmt,
1.1465 + symbol.try_stmt,
1.1466 + symbol.with_stmt,
1.1467 + symbol.suite,
1.1468 + symbol.testlist,
1.1469 + symbol.testlist_safe,
1.1470 + symbol.test,
1.1471 + symbol.and_test,
1.1472 + symbol.not_test,
1.1473 + symbol.comparison,
1.1474 + symbol.exprlist,
1.1475 + symbol.expr,
1.1476 + symbol.xor_expr,
1.1477 + symbol.and_expr,
1.1478 + symbol.shift_expr,
1.1479 + symbol.arith_expr,
1.1480 + symbol.term,
1.1481 + symbol.factor,
1.1482 + symbol.power,
1.1483 + symbol.atom,
1.1484 + symbol.yield_stmt,
1.1485 + symbol.yield_expr,
1.1486 + ]
1.1487 +
1.1488 +_assign_types = [
1.1489 + symbol.test,
1.1490 + symbol.or_test,
1.1491 + symbol.and_test,
1.1492 + symbol.not_test,
1.1493 + symbol.comparison,
1.1494 + symbol.expr,
1.1495 + symbol.xor_expr,
1.1496 + symbol.and_expr,
1.1497 + symbol.shift_expr,
1.1498 + symbol.arith_expr,
1.1499 + symbol.term,
1.1500 + symbol.factor,
1.1501 + ]
1.1502 +
1.1503 +_names = {}
1.1504 +for k, v in symbol.sym_name.items():
1.1505 + _names[k] = v
1.1506 +for k, v in token.tok_name.items():
1.1507 + _names[k] = v
1.1508 +
1.1509 +def debug_tree(tree):
1.1510 + l = []
1.1511 + for elt in tree:
1.1512 + if isinstance(elt, int):
1.1513 + l.append(_names.get(elt, elt))
1.1514 + elif isinstance(elt, str):
1.1515 + l.append(elt)
1.1516 + else:
1.1517 + l.append(debug_tree(elt))
1.1518 + return l