# HG changeset patch
# User paulb@jeremy
# Date 1154300494 -7200
# Node ID adf464d8a1883d9ed88334f6562209a0663bc2e3
# Parent  f60941bcfca112bcfd900793e4da1cadc2c871c8
Fixed/improved namespace usage in the annotation process.
Changed find_methods to become find_attributes, along with a get_attributes
function which knows how to traverse class and object namespaces.
Changed the fixer/annotator process methods, introducing process_all methods
which accept visitors/simplifiers.
Added conditional annotation support, and tentative invocation support, along
with locals snapshot support (upon which conditionals depend).
Added tentative constant and instance support, along with initialisation of
constant attributes.
Added Return nodes in most Conditional node sections in order to facilitate
locals merging.


diff -r f60941bcfca1 -r adf464d8a188 annotate.py
--- a/annotate.py	Sun Jul 30 16:15:57 2006 +0200
+++ b/annotate.py	Mon Jul 31 01:01:34 2006 +0200
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Annotate simplified AST structures. The code in this module operates upon nodes
+Annotate program node structures. The code in this module operates upon nodes
 which are produced when simplifying AST node trees originating from the compiler
 module.
 
@@ -44,7 +44,7 @@
 
 system = System()
 
-# Namespaces and related abstractions.
+# Namespace-related abstractions.
 
 class Namespace:
 
@@ -53,8 +53,7 @@
     locals or the initialisation of a structure.
     """
 
-    def __init__(self, structure=None):
-        self.structure = structure
+    def __init__(self):
         self.names = {}
 
     def store(self, name, types):
@@ -65,7 +64,7 @@
 
     def merge(self, name, types):
         if not self.names.has_key(name):
-            self.names[name] = types
+            self.names[name] = types[:]
         else:
             existing = self.names[name]
             for type in types:
@@ -79,6 +78,9 @@
         for name, types in items:
             self.merge(name, types)
 
+    def __repr__(self):
+        return repr(self.names)
+
 class Attribute:
 
     """
@@ -93,10 +95,13 @@
     def __eq__(self, other):
         return hasattr(other, "type") and other.type == self.type or other == self.type
 
-def find_methods(structure, name):
+    def __repr__(self):
+        return "Attribute of type %s (context %s)" % (self.type, self.context)
+
+def find_attributes(structure, name):
 
     """
-    Find for the given 'structure' all methods for the given 'name', visiting
+    Find for the given 'structure' all attributes for the given 'name', visiting
     base classes where appropriate and returning the methods in order of
     descending precedence for all possible base classes.
     """
@@ -104,16 +109,43 @@
     try:
         return structure.namespace.load(name)
     except KeyError:
-        methods = []
-        if hasattr(structure, "base_refs"):
+        attributes = []
+        if isinstance(structure, Instance):
+            for cls in structure.namespace.load("__class__"):
+                l = find_attributes(cls, name)
+                for attribute in l:
+                    if attribute not in attributes:
+                        attributes.append(attribute)
+        elif isinstance(structure, Class):
             for base_refs in structure.base_refs:
+                base_attributes = []
                 for base_ref in base_refs:
-                    l = find_methods(base_ref, name)
+                    l = find_attributes(base_ref, name)
                     if l:
-                        for method in l:
-                            if method not in methods:
-                                methods.append(method)
-        return methods
+                        for attribute in l:
+                            if attribute not in base_attributes:
+                                base_attributes.append(attribute)
+                    elif None not in base_attributes:
+                        base_attributes.append(None)
+                if base_attributes != [None]:
+                    attributes += base_attributes
+        return attributes
+
+def get_attributes(structure, name):
+
+    """
+    Return all possible attributes for the given 'structure' having the given
+    'name', wrapping each attribute in an Attribute object which includes
+    context information for the attribute access.
+    """
+
+    if isinstance(structure, Attribute):
+        structure = structure.type
+    attributes = find_attributes(structure, name)
+    for i, attribute in enumerate(attributes):
+        if attribute is not None:
+            attributes[i] = Attribute(structure, attribute)
+    return attributes
 
 # Annotation.
 
@@ -137,35 +169,49 @@
 
         self.visitor = self
 
+    def process_all(self, visitor, builtins_visitor=None):
+
+        # Give constants their own namespace.
+
+        for value, constant in visitor.constants.items():
+            constant.namespace = Namespace()
+
+        # Process the module, supplying builtins if possible.
+
+        if builtins_visitor is not None:
+            return self.process(visitor.result, builtins=builtins_visitor.result.namespace)
+        else:
+            return self.process(visitor.result)
+
     def process(self, node, locals=None, globals=None, builtins=None):
 
         """
-        Process a subprogram or module 'node', indicating any initial 'locals',
-        'globals' and 'builtins' if any are defined. Return an annotated
-        subprogram or module. Note that this method may mutate nodes in the
-        original program.
+        Process a subprogram or module 'node', indicating any initial 'locals'.
+        Return an annotated subprogram or module. Note that this method may
+        mutate nodes in the original program.
         """
 
-        # Obtain a namespace either based on locals or on a structure.
+        # Determine the global namespace.
+        # NOTE: Improve this.
 
-        self.namespace = Namespace(structure=getattr(node, "structure", None))
-        if locals is not None:
-            self.namespace.merge_namespace(locals)
+        self.global_namespace = globals or Namespace()
+        self.builtins_namespace = builtins or self.global_namespace
+        self.namespace = locals or self.global_namespace
 
-        # Determine the global namespace.
+        # Record the namespace on the node.
+        # NOTE: This may eventually be a specialisation node.
 
-        self.global_namespace = globals or self.namespace # NOTE: Improve this.
-        self.builtins_namespace = builtins or self.namespace # NOTE: Improve this.
         node.namespace = self.namespace
 
-        # Remember return values.
+        # Remember return values and locals snapshots.
 
         self.returns = []
+        self.return_locals = []
 
         # Add namespace details to any structure involved.
 
-        if hasattr(node, "structure") and node.structure is not None:
-            node.structure.namespace = self.namespace
+        if getattr(node, "structure", None) is not None:
+            node.structure.namespace = Namespace()
 
             # Initialise bases where appropriate.
 
@@ -179,7 +225,6 @@
         # Dispatch to the code itself.
 
         result = self.dispatch(node)
-
         return result
 
     def annotate(self, node):
@@ -188,6 +233,14 @@
 
         self.system.annotate(node, self.types)
 
+    def add_locals_snapshot(self):
+
+        "Make a snapshot of the locals and remember them."
+
+        namespace = Namespace()
+        namespace.merge_namespace(self.namespace)
+        self.return_locals.append(namespace)
+
     # Visitor methods.
 
     def default(self, node):
@@ -208,7 +261,11 @@
         return node
 
     def dispatch(self, node, *args):
-        return Visitor.dispatch(self, node, *args)
+        try:
+            return Visitor.dispatch(self, node, *args)
+        except:
+            print "Failed using node", node
+            raise
 
     def visitLoadRef(self, loadref):
         self.types = [loadref.ref]
@@ -260,8 +317,8 @@
         loadattr.expr = self.dispatch(loadattr.expr)
         types = []
         for ref in self.types:
-            for type in ref.namespace.load(loadattr.name):
-                types.append(Attribute(ref, type))
+            for type in get_attributes(ref, loadattr.name):
+                types.append(type)
         self.types = types
         self.annotate(loadattr)
         return loadattr
@@ -274,10 +331,31 @@
             ref.namespace.store(storeattr.name, expr)
         return storeattr
 
+    def visitConditional(self, conditional):
+
+        # Conditionals keep local namespace changes isolated.
+        # With Return nodes inside the body/else sections, the changes are
+        # communicated to the caller.
+
+        conditional.test = self.dispatch(conditional.test)
+        saved_namespace = self.namespace
+
+        self.namespace = Namespace()
+        self.namespace.merge_namespace(saved_namespace)
+        conditional.body = self.dispatches(conditional.body)
+
+        self.namespace = Namespace()
+        self.namespace.merge_namespace(saved_namespace)
+        conditional.else_ = self.dispatches(conditional.else_)
+
+        self.namespace = saved_namespace
+        return conditional
+
     def visitReturn(self, return_):
         if hasattr(return_, "expr"):
             return_.expr = self.dispatch(return_.expr)
             self.returns += self.types
+        self.add_locals_snapshot()
         return return_
 
     def visitInvoke(self, invoke):
@@ -312,36 +390,94 @@
         invoke.args = args
         invoke.types = expr
 
-        # NOTE: Now locate and invoke the subprogram.
+        # Now locate and invoke the subprogram. This can be complicated because
+        # the target may be a class or object, and there may be many different
+        # related subprograms.
 
-        for subprogram in expr:
+        invocations = []
+
+        # Visit each callable in turn
+
+        for callable in expr:
 
-            # NOTE: Deal with class invocations by providing instance objects,
-            # NOTE: and with object invocations by using __call__ methods.
+            # Deal with class invocations by providing instance objects.
+            # Here, each class is queried for the __init__ method, which may
+            # exist for some combinations of classes in a hierarchy but not for
+            # others.
+
+            if isinstance(callable, Class):
+                subprograms = get_attributes(callable, "__init__")
 
-            if hasattr(invoke, "same_frame") and invoke.same_frame:
-                namespace = self.namespace
+            # Deal with object invocations by using __call__ methods.
+
+            elif isinstance(callable, Instance):
+                subprograms = get_attributes(callable, "__call__")
+
+            # Normal functions or methods are more straightforward.
+
             else:
-                items = self.make_items(invoke, subprogram)
-                namespace = self.make_namespace(items)
-
-            annotator = Annotator()
-            annotator.process(subprogram, namespace, self.global_namespace)
+                subprograms = [callable]
 
-            # NOTE: Annotate the node with invocation details.
-            # NOTE: This should really be as part of a table of alternatives.
-
-            if hasattr(subprogram, "returns_value") and subprogram.returns_value:
-                self.types = annotator.returns
-                self.annotate(invoke)
+            for subprogram in subprograms:
+                if subprogram is not None:
+                    invocations.append(self.invoke_subprogram(invoke, subprogram))
 
         return invoke
 
     # Utility methods.
 
-    def make_items(self, invocation, subprogram):
-        # NOTE: Support star and dstar.
-        args = invocation.args
+    def invoke_subprogram(self, invoke, subprogram):
+
+        """
+        Invoke using the given 'invoke' node the given 'subprogram'.
+        """
+
+        # Test for context information.
+
+        if hasattr(subprogram, "context"):
+            context = subprogram.context
+            target = subprogram.type
+        else:
+            context = None
+            target = subprogram
+
+        # Provide the correct namespace for the invocation.
+
+        if getattr(invoke, "same_frame", 0):
+            namespace = Namespace()
+            namespace.merge_namespace(self.namespace)
+        else:
+            items = self.make_items(invoke, target, context)
+            namespace = self.make_namespace(items)
+
+        # Process the subprogram.
+
+        annotator = Annotator()
+        annotator.process(subprogram, namespace, self.global_namespace, self.builtins_namespace)
+
+        # NOTE: Annotate the node with invocation details.
+        # NOTE: This should really be as part of a table of alternatives.
+
+        if getattr(subprogram, "returns_value", 0):
+            self.types = annotator.returns
+            self.annotate(invoke)
+
+        if getattr(invoke, "same_frame", 0):
+            for locals in annotator.return_locals:
+                self.namespace.merge_namespace(locals)
+
+    def make_items(self, invocation, subprogram, context):
+
+        """
+        Make an items mapping for the 'invocation' of the 'subprogram' using the
+        given 'context' (which may be None).
+        """
+
+        if context is not None:
+            args = [context] + invocation.args
+        else:
+            args = invocation.args
+
         params = subprogram.params
         items = []
         keywords = {}
diff -r f60941bcfca1 -r adf464d8a188 fixnames.py
--- a/fixnames.py	Sun Jul 30 16:15:57 2006 +0200
+++ b/fixnames.py	Mon Jul 31 01:01:34 2006 +0200
@@ -62,7 +62,7 @@
 
         # Obtain a namespace either based on locals or on a structure.
 
-        self.namespace = Namespace(structure=getattr(node, "structure", None))
+        self.namespace = NameOrganiser(structure=getattr(node, "structure", None))
 
         # Add namespace details to any structure involved.
 
@@ -133,7 +133,7 @@
             self.namespace.store(storename.name)
             return storename
 
-class Namespace:
+class NameOrganiser:
 
     """
     A local namespace which may either relate to a genuine set of function
diff -r f60941bcfca1 -r adf464d8a188 simplified.py
--- a/simplified.py	Sun Jul 30 16:15:57 2006 +0200
+++ b/simplified.py	Mon Jul 31 01:01:34 2006 +0200
@@ -159,7 +159,7 @@
 class LoadName(Node): "Load a named object."
 class LoadGlobal(Node): "Load a named global object."
 class LoadAttr(Node): "Load an object attribute."
-class LoadRef(Node): "Load a reference, typically a subprogram."
+class LoadRef(Node): "Load a reference, typically a subprogram or a constant."
 class LoadExc(Node): "Load a handled exception."
 class StoreTemp(Node): "Store a temporary value."
 class StoreName(Node): "Associate a name with an object."
@@ -188,8 +188,24 @@
         else:
             return "%s (at %x)" % (self.__class__, id(self))
 
-class Class(Structure): "A Python class."
-class Instance(Structure): "An instance."
-class Constant(Instance): "A constant."
+class Class(Structure):
+
+    "A Python class."
+
+class Instance(Structure):
+
+    "An instance."
+
+    def __init__(self, **kw):
+        Structure.__init__(self, **kw)
+        self.types = [self]
+
+class Constant(Instance):
+
+    "A constant initialised with a type name for future processing."
+
+    def __init__(self, **kw):
+        Instance.__init__(self, **kw)
+        self.typename = self.value.__class__.__name__
 
 # vim: tabstop=4 expandtab shiftwidth=4
diff -r f60941bcfca1 -r adf464d8a188 simplify.py
--- a/simplify.py	Sun Jul 30 16:15:57 2006 +0200
+++ b/simplify.py	Mon Jul 31 01:01:34 2006 +0200
@@ -44,13 +44,14 @@
              Sub, Yield.
     """
 
-    def __init__(self):
+    def __init__(self, builtins=0):
         Visitor.__init__(self)
         self.result = None              # The resulting tree.
         self.subprograms = []           # Subprograms outside the tree.
         self.structures = []            # Structures/classes.
         self.constants = {}             # Constants.
         self.current_subprograms = []   # Current subprograms being processed.
+        self.builtins = builtins        # Whether the builtins are being processed.
 
     def process(self, node):
         self.visitor = self
@@ -77,7 +78,21 @@
 
     def visitModule(self, module):
         self.result = Module(module)
-        self.result.code = self.dispatch(module.node)
+        module_code = self.dispatch(module.node)
+
+        # NOTE: Constant initialisation necessary for annotation but perhaps
+        # NOTE: redundant in the program.
+
+        init_code = []
+        for value, constant in self.constants.items():
+            init_code.append(StoreAttr(lvalue=LoadRef(ref=constant), name="__class__", expr=LoadName(name=constant.typename)))
+
+        # NOTE: Hack to ensure correct initialisation of constants.
+
+        if self.builtins:
+            self.result.code = module_code + init_code
+        else:
+            self.result.code = init_code + module_code 
         return self.result
 
     def visitGetattr(self, getattr):
@@ -205,6 +220,8 @@
             self.current_subprograms.pop()
             self.subprograms.append(body_subprogram)
 
+            # Always return from conditional sections.
+
             test.body = [Invoke(stmt, expr=LoadRef(ref=body_subprogram), same_frame=1, star=None, dstar=None, args=[]), Return()]
             nodes.append(test)
 
@@ -219,7 +236,10 @@
             self.current_subprograms.pop()
             self.subprograms.append(else_subprogram)
 
+            # Always return from conditional subprograms.
+
             nodes.append(Invoke(stmt, expr=LoadRef(ref=else_subprogram), same_frame=1, star=None, dstar=None, args=[]))
+            nodes.append(Return())
 
         subprogram.code = nodes
 
@@ -258,6 +278,9 @@
                 test = Conditional(body=[], else_=[], test=Invoke(expr=LoadName(name="isinstance"), args=[LoadExc(), new_spec], star=None, dstar=None))
                 if assign is not None:
                     test.body.append(Assign(code=[StoreTemp(expr=LoadExc()), self.dispatch(assign), ReleaseTemp()]))
+
+                # Always return from conditional sections.
+
                 test.body += self.dispatch(stmt) + [Return()]
                 nodes.append(test)
 
@@ -316,6 +339,9 @@
             else:
                 raise NotImplementedError, op_name
             nodes.append(StoreTemp(expr=invocation))
+
+            # Always return from conditional sections/subprograms.
+
             if op is not last:
                 nodes.append(Conditional(test=Not(expr=LoadTemp()), body=[Return(expr=LoadTemp())]))
                 nodes.append(ReleaseTemp())
@@ -348,6 +374,9 @@
         last = and_.nodes[-1]
         for node in and_.nodes:
             expr = self.dispatch(node)
+
+            # Always return from conditional sections/subprograms.
+
             if node is not last:
                 nodes.append(StoreTemp(expr=expr))
                 invocation = Invoke(expr=LoadAttr(expr=LoadTemp(), name="__true__"), args=[], star=None, dstar=None)
@@ -381,6 +410,9 @@
         last = or_.nodes[-1]
         for node in or_.nodes:
             expr = self.dispatch(node)
+
+            # Always return from conditional sections/subprograms.
+
             if node is not last:
                 nodes.append(StoreTemp(expr=expr))
                 invocation = Invoke(expr=LoadAttr(expr=LoadTemp(), name="__true__"), args=[], star=None, dstar=None)
@@ -608,12 +640,12 @@
 
         # Make a subprogram which initialises the class structure.
 
-        subprogram = Subprogram(name=None, structure=structure, params=[], star=None, dstar=None)
+        subprogram = Subprogram(name=None, acquire_locals=1, structure=structure, params=[], star=None, dstar=None)
         self.current_subprograms.append(subprogram)
 
         # The class is initialised using the code found inside.
 
-        subprogram.code = self.dispatch(class_.code)
+        subprogram.code = self.dispatch(class_.code) + [Return()]
 
         self.current_subprograms.pop()
         self.subprograms.append(subprogram)
@@ -668,9 +700,9 @@
         # Make a subprogram for the function and record it outside the main
         # tree.
 
-        subprogram = Subprogram(name=function.name, returns_value=1, star=None, dstar=None)
+        subprogram = Subprogram(name=function.name, acquire_locals=0, returns_value=1, star=None, dstar=None)
         self.current_subprograms.append(subprogram)
-        subprogram.code = self.dispatch(function.code)
+        subprogram.code = self.dispatch(function.code) + [Return()]
         self.current_subprograms.pop()
         self._visitFunction(function, subprogram)
 
@@ -684,7 +716,7 @@
         # Make a subprogram for the function and record it outside the main
         # tree.
 
-        subprogram = Subprogram(name=None, returns_value=1, star=None, dstar=None)
+        subprogram = Subprogram(name=None, acquire_locals=0, returns_value=1, star=None, dstar=None)
         self.current_subprograms.append(subprogram)
         subprogram.code = [Return(expr=self.dispatch(lambda_.code))]
         self.current_subprograms.pop()
@@ -722,10 +754,13 @@
 
         continuation = Invoke(same_frame=1, star=None, dstar=None, args=[])
         continuation.expr = LoadRef(ref=subprogram)
-        test.body = self.dispatch(while_.body) + [continuation]
+
+        # Always return from conditional sections/subprograms.
+
+        test.body = self.dispatch(while_.body) + [continuation, Return()]
         if while_.else_ is not None:
             test.else_ = self.dispatch(while_.else_)
-        subprogram.code = [test]
+        subprogram.code = [test, Return()]
 
         self.current_subprograms.pop()
         self.subprograms.append(subprogram)
@@ -743,10 +778,12 @@
         subprogram = Subprogram(name=None, acquire_locals=1, returns_value=0, params=[], star=None, dstar=None)
         self.current_subprograms.append(subprogram)
 
+        # Always return from conditional sections/subprograms.
+
         if for_.else_ is not None:
-            else_stmt = self.dispatch(for_.else_)
+            else_stmt = self.dispatch(for_.else_) + [Return()]
         else:
-            else_stmt = []
+            else_stmt = [Return()]
 
         # Wrap the assignment in a try...except statement.
 
@@ -768,7 +805,7 @@
         continuation = Invoke(same_frame=1, produces_result=0, star=None, dstar=None, args=[])
         continuation.expr = LoadRef(ref=subprogram)
         try_except.body = [assign] + self.dispatch(for_.body) + [continuation]
-        subprogram.code = [try_except]
+        subprogram.code = [try_except, Return()]
 
         self.subprograms.append(subprogram)
         self.current_subprograms.pop()
diff -r f60941bcfca1 -r adf464d8a188 test.py
--- a/test.py	Sun Jul 30 16:15:57 2006 +0200
+++ b/test.py	Mon Jul 31 01:01:34 2006 +0200
@@ -5,9 +5,9 @@
 a = annotate.Annotator()
 b = compiler.parseFile(os.path.join("lib", "builtins.py"))
 m = compiler.parseFile(sys.argv[1])
-builtins_simplifier = simplify.Simplifier()
+vb = builtins_simplifier = simplify.Simplifier(1)
 builtins_simplifier.process(b)
-module_simplifier = simplify.Simplifier()
+v = module_simplifier = simplify.Simplifier()
 module_simplifier.process(m)
 builtins_fixer = fixnames.Fixer()
 builtins_fixer.process_all(builtins_simplifier)
diff -r f60941bcfca1 -r adf464d8a188 tests/dynamic_subclass.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/dynamic_subclass.py	Mon Jul 31 01:01:34 2006 +0200
@@ -0,0 +1,9 @@
+if 2:
+    class A:
+        def f(self):
+            pass
+else:
+    class A:
+        pass
+class B(A):
+    pass
diff -r f60941bcfca1 -r adf464d8a188 tests/subclass.py
--- a/tests/subclass.py	Sun Jul 30 16:15:57 2006 +0200
+++ b/tests/subclass.py	Mon Jul 31 01:01:34 2006 +0200
@@ -3,3 +3,4 @@
 class B(A):
     def b(self): pass
 class C(A, B): pass
+class D(B): pass