# HG changeset patch # User paulb@jeremy # Date 1160243181 -7200 # Node ID fe0d6544097093886f076a2e4eb7379c391ad26e # Parent cf98483930591c5fb6862eeca90497ead73f5a75 Attempted to handle invocation arguments properly, introducing argument filtering in the InvokeFunction class, and enforcing the usage of InvokeFunction in all non-block cases. Attempted to prevent infinite recursion in annotation. Renamed FailureError to AnnotationError and introduced AnnotationMessage. diff -r cf9848393059 -r fe0d65440970 annotate.py --- a/annotate.py Sat Oct 07 01:50:03 2006 +0200 +++ b/annotate.py Sat Oct 07 19:46:21 2006 +0200 @@ -69,7 +69,7 @@ # Exceptions. -class FailureError(Exception): +class AnnotationError(Exception): def __init__(self, exc, node, *args): Exception.__init__(self, *args) self.nodes = [node] @@ -79,6 +79,9 @@ def __str__(self): return "%s, %s" % (self.exc, self.nodes) +class AnnotationMessage(Exception): + pass + # Annotation. class Annotator(Visitor): @@ -146,6 +149,13 @@ mutate nodes in the original program. """ + # Prevent infinite recursion. + + if node in self.current_subprograms: + return node + + # Determine the namespace. + if locals: self.namespace = locals else: @@ -219,11 +229,11 @@ def dispatch(self, node, *args): try: return Visitor.dispatch(self, node, *args) - except FailureError, exc: + except AnnotationError, exc: exc.add(node) raise - except Exception, exc: - raise FailureError(exc, node) + except AnnotationMessage, exc: + raise AnnotationError(exc, node) def visitLoadRef(self, loadref): self.namespace.set_types([Attribute(None, loadref.ref)]) @@ -327,7 +337,8 @@ invoke.expr = self.dispatch(invoke.expr) invocation_types = self.namespace.types - self.process_args(invoke) + if isinstance(invoke, InvokeFunction): + self.process_args(invoke) # Now locate and invoke the subprogram. This can be complicated because # the target may be a class or object, and there may be many different @@ -403,6 +414,9 @@ return invoke + visitInvokeFunction = visitInvoke + visitInvokeBlock = visitInvoke + # Utility methods. def invoke_subprogram(self, invoke, subprogram): @@ -493,56 +507,69 @@ else: args = invocation.args + # Sort the arguments into positional and keyword arguments. + + pos_args = [] + kw_args = [] + add_kw = 0 + for arg in args: + if not add_kw: + if not isinstance(arg, Keyword): + pos_args.append(arg) + else: + add_kw = 1 + if add_kw: + if isinstance(arg, Keyword): + kw_args.append(arg) + else: + raise AnnotationMessage, "Positional argument appears after keyword arguments in '%s'." % callfunc + params = subprogram.params items = [] - keywords = {} - - # Process the specified arguments. + star_args = [] - for arg in args: - if isinstance(arg, Keyword): - keywords[arg.name] = arg.expr - continue - elif params: + # Match each positional argument, taking excess arguments as star args. + + for arg in pos_args: + if params: param, default = params[0] if arg is None: arg = default + items.append((param, arg.types)) + params = params[1:] else: - raise TypeError, "Invocation of '%s' has too many arguments for %s, *%s, **%s." % ( - subprogram, subprogram.params, subprogram.star, subprogram.dstar) - items.append((param, arg.types)) - params = params[1:] + star_args.append(arg) # Collect the remaining defaults. while params: param, default = params[0] - if keywords.has_key(param): - arg = keywords[param] - else: - arg = self.dispatch(default) # NOTE: Review reprocessing. + if kw_args.has_key(param): + arg = kw_args[param] + elif default is None: + raise AnnotationMessage, "No argument supplied in '%s' for parameter '%s'." % (subprogram, param) items.append((param, arg.types)) params = params[1:] # Add star and dstar. - if invocation.star is not None: + if star_args or invocation.star is not None: if subprogram.star is not None: param, default = subprogram.star items.append((param, invocation.star.types)) else: - raise TypeError, "Invocation provides unwanted *args." + raise AnnotationMessage, "Invocation provides unwanted *args." elif subprogram.star is not None: param, default = subprogram.star arg = self.dispatch(default) # NOTE: Review reprocessing. items.append((param, arg.types)) - if invocation.dstar is not None: + if kw_args or invocation.dstar is not None: if subprogram.dstar is not None: param, default = subprogram.dstar items.append((param, invocation.dstar.types)) else: - raise TypeError, "Invocation provides unwanted **args." + raise AnnotationMessage, "Invocation provides unwanted **args." elif subprogram.dstar is not None: param, default = subprogram.dstar arg = self.dispatch(default) # NOTE: Review reprocessing. diff -r cf9848393059 -r fe0d65440970 simplified.py --- a/simplified.py Sat Oct 07 01:50:03 2006 +0200 +++ b/simplified.py Sat Oct 07 19:46:21 2006 +0200 @@ -27,11 +27,16 @@ # Unique name registration. class Naming: + + "Maintain records of unique names for each simple name." + def __init__(self): self.obj_to_name = {} self.names = {} + def get(self, obj): return self.obj_to_name[obj] + def set(self, obj, name): if self.obj_to_name.has_key(obj): return @@ -200,23 +205,7 @@ self._pprint(indent + 2, "| ", "when %s: %s" % (ref, attribute), stream=stream) self._pprint(indent, "", "--------", stream=stream) -class NamedNode(Node): - - "A named node." - - def __init__(self, *args, **kw): - Node.__init__(self, *args, **kw) - self.full_name = name(self, self.name or "$untitled") - -class Module(NamedNode): "A Python module." -class Subprogram(NamedNode): "A subprogram: functions, methods and loops." - class Pass(Node): "A placeholder node corresponding to pass." - -class Invoke(Node): "An invocation." -class InvokeFunction(Invoke): "A function or method invocation." -class InvokeBlock(Invoke): "A block or loop invocation." - class Return(Node): "Return an evaluated expression." class Assign(Node): "A grouping node for assignment-related operations." class Keyword(Node): "A grouping node for keyword arguments." @@ -237,35 +226,78 @@ class Not(Node): "A negation of an expression." class Choice(Node): "A special node which indicates a choice of expressions." +# Invocations involve some more work to process calculated attributes. + +class Invoke(Node): "An invocation." + +class InvokeFunction(Invoke): + + "A function or method invocation." + + def __init__(self, *args, **kw): + Node.__init__(self, *args, **kw) + if hasattr(self, "args"): + self.set_args(self.args) + + def set_args(self, args): + + "Sort the 'args' into positional and keyword arguments." + + self.pos_args = [] + self.kw_args = [] + add_kw = 0 + for arg in args: + if not add_kw: + if not isinstance(arg, Keyword): + self.pos_args.append(arg) + else: + add_kw = 1 + if add_kw: + if isinstance(arg, Keyword): + self.kw_args.append(arg) + else: + raise TypeError, "Positional argument appears after keyword arguments in '%s'." % self + +class InvokeBlock(Invoke): "A block or loop invocation." + +# Named nodes are those which can be referenced in some way. + +class WithName: + + "Node naming." + + def __init__(self): + self.full_name = name(self, self.name or "$untitled") + +class Module(Node, WithName): + + "A Python module." + + def __init__(self, *args, **kw): + Node.__init__(self, *args, **kw) + WithName.__init__(self) + +class Subprogram(Node, WithName): + + "A subprogram: functions, methods and loops." + + def __init__(self, *args, **kw): + Node.__init__(self, *args, **kw) + WithName.__init__(self) + # Special non-program nodes. -class Structure: - - "A non-program node containing some kind of namespace." +class Structure(Node): "A non-program node containing some kind of namespace." - def __init__(self, original=None, **kw): - self.original = original - for name, value in kw.items(): - setattr(self, name, value) - - def __repr__(self): - if hasattr(self, "name"): - return "%s '%s'" % (self.__class__.__name__, name(self, self.name)) - else: - return "%s" % (self.__class__.__name__,) - -class Class(Structure): +class Class(Structure, WithName): "A Python class." - pass - -class Instance(Structure): - - "An instance." - def __init__(self, *args, **kw): Structure.__init__(self, *args, **kw) + WithName.__init__(self) + +class Instance(Structure): "An instance." class Constant(Instance): diff -r cf9848393059 -r fe0d65440970 simplify.py --- a/simplify.py Sat Oct 07 01:50:03 2006 +0200 +++ b/simplify.py Sat Oct 07 19:46:21 2006 +0200 @@ -218,7 +218,7 @@ return result def _visitBuiltin(self, builtin, name): - result = Invoke(builtin, expr=LoadName(builtin, name=name), args=self.dispatches(builtin.nodes), star=None, dstar=None) + result = InvokeFunction(builtin, expr=LoadName(builtin, name=name), args=self.dispatches(builtin.nodes), star=None, dstar=None) return result def visitTuple(self, tuple): @@ -228,13 +228,13 @@ return self._visitBuiltin(list, "list") def visitDict(self, dict): - result = Invoke(dict, expr=LoadName(dict, name="dict"), star=None, dstar=None) + result = InvokeFunction(dict, expr=LoadName(dict, name="dict"), star=None, dstar=None) args = [] for key, value in dict.items: - tuple = Invoke(dict, expr=LoadName(dict, name="tuple"), star=None, dstar=None) - tuple.args = [self.dispatch(key), self.dispatch(value)] + tuple = InvokeFunction(dict, expr=LoadName(dict, name="tuple"), star=None, dstar=None) + tuple.set_args([self.dispatch(key), self.dispatch(value)]) args.append(tuple) - result.args = args + result.set_args(args) return result # Logical and comparison operations plus chained statements. @@ -268,7 +268,7 @@ for compare, stmt in if_.tests: test = Conditional(if_, - test=Invoke(if_, + test=InvokeFunction(if_, expr=LoadAttr(if_, expr=self.dispatch(compare), name="__true__" @@ -333,7 +333,7 @@ else: new_spec = self.dispatch(spec) test = Conditional(tryexcept, - test=Invoke(tryexcept, + test=InvokeFunction(tryexcept, expr=LoadName(tryexcept, name="isinstance"), args=[LoadExc(tryexcept), new_spec], star=None, @@ -404,11 +404,11 @@ method_name = self.comparison_methods[op_name] if method_name: - invocation = Invoke(compare, expr=LoadAttr(compare, expr=previous, name=method_name), args=[expr], star=None, dstar=None) + invocation = InvokeFunction(compare, expr=LoadAttr(compare, expr=previous, name=method_name), args=[expr], star=None, dstar=None) elif op_name == "is": - invocation = Invoke(compare, expr=LoadName(compare, name="__is__"), args=[previous, expr], star=None, dstar=None) + invocation = InvokeFunction(compare, expr=LoadName(compare, name="__is__"), args=[previous, expr], star=None, dstar=None) elif op_name == "is not": - invocation = Not(compare, expr=Invoke(compare, expr=LoadName(compare, name="__is__"), args=[previous, expr], star=None, dstar=None)) + invocation = Not(compare, expr=InvokeFunction(compare, expr=LoadName(compare, name="__is__"), args=[previous, expr], star=None, dstar=None)) else: raise NotImplementedError, op_name nodes.append(StoreTemp(compare, expr=invocation)) @@ -477,7 +477,7 @@ if node is not last: nodes.append(StoreTemp(and_, expr=expr)) - invocation = Invoke(and_, expr=LoadAttr(and_, expr=LoadTemp(and_), name="__true__"), args=[], star=None, dstar=None) + invocation = InvokeFunction(and_, expr=LoadAttr(and_, expr=LoadTemp(and_), name="__true__"), args=[], star=None, dstar=None) test = Conditional(and_, test=Not(and_, expr=invocation), body=[Return(and_, expr=LoadTemp(and_))]) nodes.append(test) @@ -537,7 +537,7 @@ if node is not last: nodes.append(StoreTemp(or_, expr=expr)) - invocation = Invoke(or_, expr=LoadAttr(or_, expr=LoadTemp(or_), name="__true__"), args=[], star=None, dstar=None) + invocation = InvokeFunction(or_, expr=LoadAttr(or_, expr=LoadTemp(or_), name="__true__"), args=[], star=None, dstar=None) test = Conditional(or_, test=invocation, body=[Return(or_, expr=LoadTemp(or_))]) nodes.append(test) @@ -566,19 +566,19 @@ return result def visitNot(self, not_): - result = Not(not_, expr=Invoke(not_, expr=LoadAttr(not_, expr=self.dispatch(not_.expr), name="__true__"), args=[], star=None, dstar=None)) + result = Not(not_, expr=InvokeFunction(not_, expr=LoadAttr(not_, expr=self.dispatch(not_.expr), name="__true__"), args=[], star=None, dstar=None)) return result # Operators. def visitUnaryAdd(self, unaryadd): - return Invoke(unaryadd, expr=LoadAttr(unaryadd, expr=self.dispatch(unaryadd.expr), name="__pos__"), args=[], star=None, dstar=None) + return InvokeFunction(unaryadd, expr=LoadAttr(unaryadd, expr=self.dispatch(unaryadd.expr), name="__pos__"), args=[], star=None, dstar=None) def visitUnarySub(self, unarysub): - return Invoke(unarysub, expr=LoadAttr(unarysub, expr=self.dispatch(unarysub.expr), name="__neg__"), args=[], star=None, dstar=None) + return InvokeFunction(unarysub, expr=LoadAttr(unarysub, expr=self.dispatch(unarysub.expr), name="__neg__"), args=[], star=None, dstar=None) def visitInvert(self, invert): - return Invoke(invert, expr=LoadAttr(invert, expr=self.dispatch(invert.expr), name="__invert__"), args=[], star=None, dstar=None) + return InvokeFunction(invert, expr=LoadAttr(invert, expr=self.dispatch(invert.expr), name="__invert__"), args=[], star=None, dstar=None) def visitAdd(self, add): @@ -594,7 +594,7 @@ result = Try(add, body=[ - Invoke(add, + InvokeFunction(add, expr=LoadAttr(add, expr=self.dispatch(add.left), name="__add__"), args=[self.dispatch(add.right)], star=None, @@ -604,13 +604,13 @@ finally_=[], handler=[ Conditional(add, - test=Invoke(add, + test=InvokeFunction(add, expr=LoadName(add, name="isinstance"), args=[LoadExc(add), LoadName(add, name="TypeError")], star=None, dstar=None), body=[ - Invoke(add, + InvokeFunction(add, expr=LoadAttr(add, expr=self.dispatch(add.right), name="__radd__"), args=[self.dispatch(add.left)], star=None, @@ -640,7 +640,7 @@ if isinstance(augassign.node, compiler.ast.Name): result.code = [ StoreTemp(augassign, - expr=Invoke(augassign, + expr=InvokeFunction(augassign, args=[expr], star=None, dstar=None, @@ -668,7 +668,7 @@ expr=self.dispatch(augassign.node.expr) ), StoreTemp(augassign, - expr=Invoke(augassign, + expr=InvokeFunction(augassign, args=[expr], star=None, dstar=None, expr=LoadAttr(augassign, expr=LoadAttr(augassign, @@ -708,7 +708,7 @@ expr=self.dispatch_or_none(augassign.node.upper) ), StoreTemp(augassign, - expr=Invoke(augassign, + expr=InvokeFunction(augassign, args=[expr], star=None, dstar=None, expr=LoadAttr(augassign, expr=self._visitSlice( @@ -745,7 +745,7 @@ StoreTemp(augassign, index="expr", expr=self.dispatch(augassign.node.expr)), StoreTemp(augassign, index="subs", expr=self._visitSubscriptSubs(augassign.node.subs)), StoreTemp(augassign, - expr=Invoke(augassign, + expr=InvokeFunction(augassign, args=[expr], star=None, dstar=None, expr=LoadAttr(augassign, expr=self._visitSubscript( @@ -786,9 +786,9 @@ if not in_sequence: expr = LoadTemp(asslist) else: - expr = Invoke(asslist, expr=LoadAttr(asslist, expr=LoadTemp(asslist), name="next"), star=None, dstar=None, args=[]) + expr = InvokeFunction(asslist, expr=LoadAttr(asslist, expr=LoadTemp(asslist), name="next"), star=None, dstar=None, args=[]) result = Assign(asslist) - store = StoreTemp(asslist, expr=Invoke(asslist, expr=LoadAttr(asslist, name="__iter__", expr=expr), star=None, dstar=None, args=[])) + store = StoreTemp(asslist, expr=InvokeFunction(asslist, expr=LoadAttr(asslist, name="__iter__", expr=expr), star=None, dstar=None, args=[])) release = ReleaseTemp(asslist) result.code = [store] + self.dispatches(asslist.nodes, 1) + [release] return result @@ -799,7 +799,7 @@ if not in_sequence: return LoadTemp(node) else: - return Invoke(node, expr=LoadAttr(node, expr=LoadTemp(node), name="next"), star=None, dstar=None, args=[]) + return InvokeFunction(node, expr=LoadAttr(node, expr=LoadTemp(node), name="next"), star=None, dstar=None, args=[]) def visitAssName(self, assname, in_sequence=0): expr = self._visitAssNameOrAttr(assname, in_sequence) @@ -815,13 +815,13 @@ def _visitSlice(self, slice, expr, lower, upper, flags, value=None): if flags == "OP_ASSIGN": args = [value] - result = Invoke(slice, expr=LoadAttr(slice, expr=expr, name="__setslice__"), star=None, dstar=None, args=[]) + result = InvokeFunction(slice, expr=LoadAttr(slice, expr=expr, name="__setslice__"), star=None, dstar=None, args=[]) elif flags == "OP_APPLY": args = [] - result = Invoke(slice, expr=LoadAttr(slice, expr=expr, name="__getslice__"), star=None, dstar=None, args=[]) + result = InvokeFunction(slice, expr=LoadAttr(slice, expr=expr, name="__getslice__"), star=None, dstar=None, args=[]) elif flags == "OP_DELETE": args = [] - result = Invoke(slice, expr=LoadAttr(slice, expr=expr, name="__delslice__"), star=None, dstar=None, args=[]) + result = InvokeFunction(slice, expr=LoadAttr(slice, expr=expr, name="__delslice__"), star=None, dstar=None, args=[]) else: raise NotImplementedError, flags @@ -831,7 +831,7 @@ args.insert(1, upper) result.original = slice - result.args = args + result.set_args(args) return result def visitSlice(self, slice, in_sequence=0): @@ -841,13 +841,13 @@ def _visitSubscript(self, subscript, expr, subs, flags, value=None): if flags == "OP_ASSIGN": args = [value] - result = Invoke(subscript, expr=LoadAttr(subscript, expr=expr, name="__setitem__"), star=None, dstar=None, args=[]) + result = InvokeFunction(subscript, expr=LoadAttr(subscript, expr=expr, name="__setitem__"), star=None, dstar=None, args=[]) elif flags == "OP_APPLY": args = [] - result = Invoke(subscript, expr=LoadAttr(subscript, expr=expr, name="__getitem__"), star=None, dstar=None, args=[]) + result = InvokeFunction(subscript, expr=LoadAttr(subscript, expr=expr, name="__getitem__"), star=None, dstar=None, args=[]) elif flags == "OP_DELETE": args = [] - result = Invoke(subscript, expr=LoadAttr(subscript, expr=expr, name="__delitem__"), star=None, dstar=None, args=[]) + result = InvokeFunction(subscript, expr=LoadAttr(subscript, expr=expr, name="__delitem__"), star=None, dstar=None, args=[]) else: raise NotImplementedError, flags @@ -856,14 +856,14 @@ args.insert(0, subs) result.original = subscript - result.args = args + result.set_args(args) return result def _visitSubscriptSubs(self, subs): if len(subs) == 1: return self.dispatch(subs[0]) else: - return Invoke(subs, expr=LoadName(subs, name="tuple"), args=self.dispatches(subs), star=None, dstar=None) + return InvokeFunction(subs, expr=LoadName(subs, name="tuple"), args=self.dispatches(subs), star=None, dstar=None) def visitSubscript(self, subscript, in_sequence=0): return self._visitSubscript( @@ -897,8 +897,7 @@ name=class_.name, expr=LoadRef(class_, ref=structure) ), - Invoke(class_, - args=[], star=None, dstar=None, + InvokeBlock(class_, expr=LoadRef(class_, ref=subprogram) ) ] @@ -931,11 +930,11 @@ # Produce star and dstar parameters with appropriate defaults. if has_star: - star = (function.argnames[npositional], Invoke(function, expr=LoadName(function, name="list"), args=[], star=None, dstar=None)) + star = (function.argnames[npositional], InvokeFunction(function, expr=LoadName(function, name="list"), args=[], star=None, dstar=None)) else: star = None if has_dstar: - dstar = (function.argnames[npositional + has_star], Invoke(function, expr=LoadName(function, name="dict"), args=[], star=None, dstar=None)) + dstar = (function.argnames[npositional + has_star], InvokeFunction(function, expr=LoadName(function, name="dict"), args=[], star=None, dstar=None)) else: dstar = None @@ -989,8 +988,7 @@ return LoadRef(lambda_, ref=subprogram) def visitCallFunc(self, callfunc): - result = InvokeFunction(callfunc, star=None, dstar=None) - result.args = self.dispatches(callfunc.args) + result = InvokeFunction(callfunc, star=None, dstar=None, args=self.dispatches(callfunc.args)) if callfunc.star_args is not None: result.star = self.dispatch(callfunc.star_args) if callfunc.dstar_args is not None: @@ -1020,7 +1018,7 @@ # Include a conditional statement in the subprogram. test = Conditional(while_, else_=[]) - test.test = Invoke(while_, expr=LoadAttr(while_, expr=self.dispatch(while_.test), name="__true__"), args=[], star=None, dstar=None) + test.test = InvokeFunction(while_, expr=LoadAttr(while_, expr=self.dispatch(while_.test), name="__true__"), args=[], star=None, dstar=None) # Inside the conditional, add a recursive invocation to the subprogram # if the test condition was satisfied. @@ -1084,14 +1082,18 @@ try_except = Try(for_, body=[], else_=[], finally_=[]) test = Conditional(for_, - test=Invoke(for_, expr=LoadName(for_, name="isinstance"), args=[LoadExc(for_), LoadName(for_, name="StopIteration")], star=None, dstar=None), + test=InvokeFunction(for_, + expr=LoadName(for_, name="isinstance"), + args=[LoadExc(for_), LoadName(for_, name="StopIteration")], + star=None, + dstar=None), body=else_stmt, else_=[Raise(for_, expr=LoadExc(for_))]) try_except.handler = [test] assign = Assign(for_, code=[ - StoreTemp(for_, expr=Invoke(for_, expr=LoadAttr(for_, expr=LoadTemp(for_), name="next"), args=[], star=None, dstar=None)), + StoreTemp(for_, expr=InvokeFunction(for_, expr=LoadAttr(for_, expr=LoadTemp(for_), name="next"), args=[], star=None, dstar=None)), self.dispatch(for_.assign), ReleaseTemp(for_) ]) @@ -1114,7 +1116,7 @@ result = Assign(for_) result.code = [ - StoreTemp(for_, expr=Invoke(for_, expr=LoadAttr(for_, name="__iter__", expr=self.dispatch(for_.list)), args=[], star=None, dstar=None)), + StoreTemp(for_, expr=InvokeFunction(for_, expr=LoadAttr(for_, name="__iter__", expr=self.dispatch(for_.list)), args=[], star=None, dstar=None)), InvokeBlock(for_, expr=LoadRef(for_, ref=subprogram)), ReleaseTemp(for_) ] diff -r cf9848393059 -r fe0d65440970 test.py --- a/test.py Sat Oct 07 01:50:03 2006 +0200 +++ b/test.py Sat Oct 07 19:46:21 2006 +0200 @@ -4,7 +4,7 @@ import simplify import fixnames import viewer -from annotate import FailureError, annotate_all as aa +from annotate import AnnotationError, annotate_all as aa builtins = simplify.simplify(os.path.join("lib", "builtins.py"), 1) module = simplify.simplify(sys.argv[1]) @@ -17,7 +17,7 @@ if "-a" in sys.argv: try: aa([module], builtins) - except FailureError, exc: + except AnnotationError, exc: v.report(exc) # vim: tabstop=4 expandtab shiftwidth=4