1.1 --- a/compiler/transformer.py Mon Jan 09 16:05:35 2017 +0100
1.2 +++ b/compiler/transformer.py Mon Jan 09 19:10:12 2017 +0100
1.3 @@ -178,68 +178,13 @@
1.4 ### is this sufficient?
1.5 return Expression(self.com_node(nodelist[0]))
1.6
1.7 - def decorator_name(self, nodelist):
1.8 - listlen = len(nodelist)
1.9 - assert listlen >= 1 and listlen % 2 == 1
1.10 -
1.11 - item = self.atom_name(nodelist)
1.12 - i = 1
1.13 - while i < listlen:
1.14 - assert nodelist[i][0] == token["DOT"]
1.15 - assert nodelist[i + 1][0] == token["NAME"]
1.16 - item = Getattr(item, nodelist[i + 1][1])
1.17 - i += 2
1.18 -
1.19 - return item
1.20 -
1.21 - def decorator(self, nodelist):
1.22 - # '@' dotted_name [ '(' [arglist] ')' ]
1.23 - assert len(nodelist) in (3, 5, 6)
1.24 - assert nodelist[0][0] == token["AT"]
1.25 - assert nodelist[-1][0] == token["NEWLINE"]
1.26 -
1.27 - assert nodelist[1][0] == symbol["dotted_name"]
1.28 - funcname = self.decorator_name(nodelist[1][1:])
1.29 -
1.30 - if len(nodelist) > 3:
1.31 - assert nodelist[2][0] == token["LPAR"]
1.32 - expr = self.com_call_function(funcname, nodelist[3])
1.33 - else:
1.34 - expr = funcname
1.35 -
1.36 - return expr
1.37 -
1.38 - def decorators(self, nodelist):
1.39 - # decorators: decorator ([NEWLINE] decorator)* NEWLINE
1.40 - items = []
1.41 - for dec_nodelist in nodelist:
1.42 - assert dec_nodelist[0] == symbol["decorator"]
1.43 - items.append(self.decorator(dec_nodelist[1:]))
1.44 - return Decorators(items)
1.45 -
1.46 - def decorated(self, nodelist):
1.47 - assert nodelist[0][0] == symbol["decorators"]
1.48 - if nodelist[1][0] == symbol["funcdef"]:
1.49 - n = [nodelist[0]] + list(nodelist[1][1:])
1.50 - return self.funcdef(n)
1.51 - elif nodelist[1][0] == symbol["classdef"]:
1.52 - decorators = self.decorators(nodelist[0][1:])
1.53 - cls = self.classdef(nodelist[1][1:])
1.54 - cls.decorators = decorators
1.55 - return cls
1.56 - raise WalkerError()
1.57 -
1.58 def funcdef(self, nodelist):
1.59 - # -6 -5 -4 -3 -2 -1
1.60 - # funcdef: [decorators] 'def' NAME parameters ':' suite
1.61 + # -5 -4 -3 -2 -1
1.62 + # funcdef: 'def' NAME parameters ':' suite
1.63 # parameters: '(' [varargslist] ')'
1.64
1.65 - if len(nodelist) == 6:
1.66 - assert nodelist[0][0] == symbol["decorators"]
1.67 - decorators = self.decorators(nodelist[0][1:])
1.68 - else:
1.69 - assert len(nodelist) == 5
1.70 - decorators = None
1.71 + assert len(nodelist) == 5
1.72 + decorators = None
1.73
1.74 lineno = nodelist[-4][2]
1.75 name = nodelist[-4][1]
1.76 @@ -399,17 +344,6 @@
1.77 return Return(Const(None), lineno=nodelist[0][2])
1.78 return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
1.79
1.80 - def yield_stmt(self, nodelist):
1.81 - expr = self.com_node(nodelist[0])
1.82 - return Discard(expr, lineno=expr.lineno)
1.83 -
1.84 - def yield_expr(self, nodelist):
1.85 - if len(nodelist) > 1:
1.86 - value = self.com_node(nodelist[1])
1.87 - else:
1.88 - value = Const(None)
1.89 - return Yield(value, lineno=nodelist[0][2])
1.90 -
1.91 def raise_stmt(self, nodelist):
1.92 # raise: [test [',' test [',' test]]]
1.93 if len(nodelist) > 5:
1.94 @@ -564,11 +498,8 @@
1.95 exprlist = testlist
1.96
1.97 def testlist_comp(self, nodelist):
1.98 - # test ( comp_for | (',' test)* [','] )
1.99 + # test ( (',' test)* [','] )
1.100 assert nodelist[0][0] == symbol["test"]
1.101 - if len(nodelist) == 2 and nodelist[1][0] == symbol["comp_for"]:
1.102 - test = self.com_node(nodelist[0])
1.103 - return self.com_generator_expression(test, nodelist[1])
1.104 return self.testlist(nodelist)
1.105
1.106 def test(self, nodelist):
1.107 @@ -1084,115 +1015,17 @@
1.108 stmts.append(result)
1.109
1.110 def com_list_constructor(self, nodelist):
1.111 - # listmaker: test ( list_for | (',' test)* [','] )
1.112 + # listmaker: test ( (',' test)* [','] )
1.113 values = []
1.114 for i in range(1, len(nodelist)):
1.115 - if nodelist[i][0] == symbol["list_for"]:
1.116 - assert len(nodelist[i:]) == 1
1.117 - return self.com_list_comprehension(values[0],
1.118 - nodelist[i])
1.119 - elif nodelist[i][0] == token["COMMA"]:
1.120 + if nodelist[i][0] == token["COMMA"]:
1.121 continue
1.122 values.append(self.com_node(nodelist[i]))
1.123 return List(values, lineno=values[0].lineno)
1.124
1.125 - def com_list_comprehension(self, expr, node):
1.126 - return self.com_comprehension(expr, None, node, 'list')
1.127 -
1.128 - def com_comprehension(self, expr1, expr2, node, type):
1.129 - # list_iter: list_for | list_if
1.130 - # list_for: 'for' exprlist 'in' testlist [list_iter]
1.131 - # list_if: 'if' test [list_iter]
1.132 -
1.133 - # XXX should raise SyntaxError for assignment
1.134 - # XXX(avassalotti) Set and dict comprehensions should have generator
1.135 - # semantics. In other words, they shouldn't leak
1.136 - # variables outside of the comprehension's scope.
1.137 -
1.138 - lineno = node[1][2]
1.139 - fors = []
1.140 - while node:
1.141 - t = node[1][1]
1.142 - if t == 'for':
1.143 - assignNode = self.com_assign(node[2], OP_ASSIGN)
1.144 - compNode = self.com_node(node[4])
1.145 - newfor = ListCompFor(assignNode, compNode, [])
1.146 - newfor.lineno = node[1][2]
1.147 - fors.append(newfor)
1.148 - if len(node) == 5:
1.149 - node = None
1.150 - elif type == 'list':
1.151 - node = self.com_list_iter(node[5])
1.152 - else:
1.153 - node = self.com_comp_iter(node[5])
1.154 - elif t == 'if':
1.155 - test = self.com_node(node[2])
1.156 - newif = ListCompIf(test, lineno=node[1][2])
1.157 - newfor.ifs.append(newif)
1.158 - if len(node) == 3:
1.159 - node = None
1.160 - elif type == 'list':
1.161 - node = self.com_list_iter(node[3])
1.162 - else:
1.163 - node = self.com_comp_iter(node[3])
1.164 - else:
1.165 - raise SyntaxError, \
1.166 - ("unexpected comprehension element: %s %d"
1.167 - % (node, lineno))
1.168 - if type == 'list':
1.169 - return ListComp(expr1, fors, lineno=lineno)
1.170 - elif type == 'set':
1.171 - return SetComp(expr1, fors, lineno=lineno)
1.172 - elif type == 'dict':
1.173 - return DictComp(expr1, expr2, fors, lineno=lineno)
1.174 - else:
1.175 - raise ValueError("unexpected comprehension type: " + repr(type))
1.176 -
1.177 - def com_list_iter(self, node):
1.178 - assert node[0] == symbol["list_iter"]
1.179 - return node[1]
1.180 -
1.181 - def com_comp_iter(self, node):
1.182 - assert node[0] == symbol["comp_iter"]
1.183 - return node[1]
1.184 -
1.185 - def com_generator_expression(self, expr, node):
1.186 - # comp_iter: comp_for | comp_if
1.187 - # comp_for: 'for' exprlist 'in' test [comp_iter]
1.188 - # comp_if: 'if' test [comp_iter]
1.189 -
1.190 - lineno = node[1][2]
1.191 - fors = []
1.192 - while node:
1.193 - t = node[1][1]
1.194 - if t == 'for':
1.195 - assignNode = self.com_assign(node[2], OP_ASSIGN)
1.196 - genNode = self.com_node(node[4])
1.197 - newfor = GenExprFor(assignNode, genNode, [],
1.198 - lineno=node[1][2])
1.199 - fors.append(newfor)
1.200 - if (len(node)) == 5:
1.201 - node = None
1.202 - else:
1.203 - node = self.com_comp_iter(node[5])
1.204 - elif t == 'if':
1.205 - test = self.com_node(node[2])
1.206 - newif = GenExprIf(test, lineno=node[1][2])
1.207 - newfor.ifs.append(newif)
1.208 - if len(node) == 3:
1.209 - node = None
1.210 - else:
1.211 - node = self.com_comp_iter(node[3])
1.212 - else:
1.213 - raise SyntaxError, \
1.214 - ("unexpected generator expression element: %s %d"
1.215 - % (node, lineno))
1.216 - fors[0].is_outmost = True
1.217 - return GenExpr(GenExprInner(expr, fors), lineno=lineno)
1.218 -
1.219 def com_dictorsetmaker(self, nodelist):
1.220 - # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
1.221 - # (test (comp_for | (',' test)* [','])) )
1.222 + # dictorsetmaker: ( (test ':' test ( (',' test ':' test)* [','])) |
1.223 + # (test ( (',' test)* [','])) )
1.224 assert nodelist[0] == symbol["dictorsetmaker"]
1.225 nodelist = nodelist[1:]
1.226 if len(nodelist) == 1 or nodelist[1][0] == token["COMMA"]:
1.227 @@ -1201,16 +1034,6 @@
1.228 for i in range(0, len(nodelist), 2):
1.229 items.append(self.com_node(nodelist[i]))
1.230 return Set(items, lineno=items[0].lineno)
1.231 - elif nodelist[1][0] == symbol["comp_for"]:
1.232 - # set comprehension
1.233 - expr = self.com_node(nodelist[0])
1.234 - return self.com_comprehension(expr, None, nodelist[1], 'set')
1.235 - elif len(nodelist) > 3 and nodelist[3][0] == symbol["comp_for"]:
1.236 - # dict comprehension
1.237 - assert nodelist[1][0] == token["COLON"]
1.238 - key = self.com_node(nodelist[0])
1.239 - value = self.com_node(nodelist[2])
1.240 - return self.com_comprehension(key, value, nodelist[3], 'dict')
1.241 else:
1.242 # dict literal
1.243 items = []
1.244 @@ -1262,12 +1085,6 @@
1.245 # positional or named parameters
1.246 kw, result = self.com_argument(node, kw, star_node)
1.247
1.248 - if len_nodelist != 2 and isinstance(result, GenExpr) \
1.249 - and len(node) == 3 and node[2][0] == symbol["comp_for"]:
1.250 - # allow f(x for x in y), but reject f(x for x in y, 1)
1.251 - # should use f((x for x in y), 1) instead of f(x for x in y, 1)
1.252 - raise SyntaxError, 'generator expression needs parenthesis'
1.253 -
1.254 args.append(result)
1.255 i = i + 2
1.256
1.257 @@ -1275,9 +1092,6 @@
1.258 lineno=extractLineNo(nodelist))
1.259
1.260 def com_argument(self, nodelist, kw, star_node):
1.261 - if len(nodelist) == 3 and nodelist[2][0] == symbol["comp_for"]:
1.262 - test = self.com_node(nodelist[1])
1.263 - return 0, self.com_generator_expression(test, nodelist[2])
1.264 if len(nodelist) == 2:
1.265 if kw:
1.266 raise SyntaxError, "non-keyword arg after keyword arg"
1.267 @@ -1477,8 +1291,6 @@
1.268 symbol["factor"],
1.269 symbol["power"],
1.270 symbol["atom"],
1.271 - symbol["yield_stmt"],
1.272 - symbol["yield_expr"],
1.273 ]
1.274
1.275 _assign_types = [
2.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
2.2 +++ b/pyparser/data/Grammar-Lichen Mon Jan 09 19:10:12 2017 +0100
2.3 @@ -0,0 +1,119 @@
2.4 +# Grammar for Lichen
2.5 +
2.6 +# Start symbols for the grammar:
2.7 +# single_input is a single interactive statement;
2.8 +# file_input is a module or sequence of commands read from an input file;
2.9 +# eval_input is the input for the eval() and input() functions.
2.10 +# NB: compound_stmt in single_input is followed by extra NEWLINE!
2.11 +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
2.12 +file_input: (NEWLINE | stmt)* ENDMARKER
2.13 +eval_input: testlist NEWLINE* ENDMARKER
2.14 +
2.15 +funcdef: 'def' NAME parameters ':' suite
2.16 +parameters: '(' [varargslist] ')'
2.17 +varargslist: ((fpdef ['=' test] ',')*
2.18 + ('*' NAME [',' '**' NAME] | '**' NAME) |
2.19 + fpdef ['=' test] (',' fpdef ['=' test])* [','])
2.20 +fpdef: NAME | '(' fplist ')'
2.21 +fplist: fpdef (',' fpdef)* [',']
2.22 +
2.23 +stmt: simple_stmt | compound_stmt
2.24 +simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
2.25 +small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |
2.26 + import_stmt | global_stmt | exec_stmt | assert_stmt)
2.27 +expr_stmt: testlist (augassign testlist |
2.28 + ('=' testlist)*)
2.29 +augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
2.30 + '<<=' | '>>=' | '**=' | '//=')
2.31 +# For normal assignments, additional restrictions enforced by the interpreter
2.32 +print_stmt: 'print' ( [ test (',' test)* [','] ] |
2.33 + '>>' test [ (',' test)+ [','] ] )
2.34 +del_stmt: 'del' exprlist
2.35 +pass_stmt: 'pass'
2.36 +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
2.37 +break_stmt: 'break'
2.38 +continue_stmt: 'continue'
2.39 +return_stmt: 'return' [testlist]
2.40 +raise_stmt: 'raise' [test [',' test [',' test]]]
2.41 +import_stmt: import_name | import_from
2.42 +import_name: 'import' dotted_as_names
2.43 +import_from: ('from' ('.'* dotted_name | '.'+)
2.44 + 'import' ('*' | '(' import_as_names ')' | import_as_names))
2.45 +import_as_name: NAME ['as' NAME]
2.46 +dotted_as_name: dotted_name ['as' NAME]
2.47 +import_as_names: import_as_name (',' import_as_name)* [',']
2.48 +dotted_as_names: dotted_as_name (',' dotted_as_name)*
2.49 +dotted_name: NAME ('.' NAME)*
2.50 +global_stmt: 'global' NAME (',' NAME)*
2.51 +exec_stmt: 'exec' expr ['in' test [',' test]]
2.52 +assert_stmt: 'assert' test [',' test]
2.53 +
2.54 +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef
2.55 +if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
2.56 +while_stmt: 'while' test ':' suite ['else' ':' suite]
2.57 +for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
2.58 +try_stmt: ('try' ':' suite
2.59 + ((except_clause ':' suite)+
2.60 + ['else' ':' suite]
2.61 + ['finally' ':' suite] |
2.62 + 'finally' ':' suite))
2.63 +with_stmt: 'with' with_item (',' with_item)* ':' suite
2.64 +with_item: test ['as' expr]
2.65 +# NB compile.c makes sure that the default except clause is last
2.66 +except_clause: 'except' [test [('as' | ',') test]]
2.67 +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
2.68 +
2.69 +# Backward compatibility cruft to support:
2.70 +# [ x for x in lambda: True, lambda: False if x() ]
2.71 +# even while also allowing:
2.72 +# lambda x: 5 if x else 2
2.73 +# (But not a mix of the two)
2.74 +testlist_safe: old_test [(',' old_test)+ [',']]
2.75 +old_test: or_test | old_lambdef
2.76 +old_lambdef: 'lambda' [varargslist] ':' old_test
2.77 +
2.78 +test: or_test ['if' or_test 'else' test] | lambdef
2.79 +or_test: and_test ('or' and_test)*
2.80 +and_test: not_test ('and' not_test)*
2.81 +not_test: 'not' not_test | comparison
2.82 +comparison: expr (comp_op expr)*
2.83 +comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
2.84 +expr: xor_expr ('|' xor_expr)*
2.85 +xor_expr: and_expr ('^' and_expr)*
2.86 +and_expr: shift_expr ('&' shift_expr)*
2.87 +shift_expr: arith_expr (('<<'|'>>') arith_expr)*
2.88 +arith_expr: term (('+'|'-') term)*
2.89 +term: factor (('*'|'/'|'%'|'//') factor)*
2.90 +factor: ('+'|'-'|'~') factor | power
2.91 +power: atom trailer* ['**' factor]
2.92 +atom: ('(' [testlist_comp] ')' |
2.93 + '[' [listmaker] ']' |
2.94 + '{' [dictorsetmaker] '}' |
2.95 + '`' testlist1 '`' |
2.96 + NAME | NUMBER | STRING+)
2.97 +listmaker: test ( (',' test)* [','] )
2.98 +testlist_comp: test ( (',' test)* [','] )
2.99 +lambdef: 'lambda' [varargslist] ':' test
2.100 +trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
2.101 +subscriptlist: subscript (',' subscript)* [',']
2.102 +subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
2.103 +sliceop: ':' [test]
2.104 +exprlist: expr (',' expr)* [',']
2.105 +testlist: test (',' test)* [',']
2.106 +dictmaker: test ':' test (',' test ':' test)* [',']
2.107 +dictorsetmaker: ( (test ':' test ( (',' test ':' test)* [','])) |
2.108 + (test ( (',' test)* [','])) )
2.109 +
2.110 +classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
2.111 +
2.112 +arglist: (argument ',')* (argument [',']
2.113 + |'*' test (',' argument)* [',' '**' test]
2.114 + |'**' test)
2.115 +# The reason that keywords are test nodes instead of NAME is that using NAME
2.116 +# results in an ambiguity. ast.c makes sure it's a NAME.
2.117 +argument: test | test '=' test
2.118 +
2.119 +testlist1: test (',' test)*
2.120 +
2.121 +# not used in grammar, but may appear in "node" passed from Parser to Compiler
2.122 +encoding_decl: NAME