# HG changeset patch # User paulb@localhost.localdomain # Date 1180283125 -7200 # Node ID efe3cfb9196659f0a88cab3576516c0cfec9f318 # Parent dfd0634cfdb6f7e258b758ff75bfe83c986800fb Reorganised the package hierarchy. diff -r dfd0634cfdb6 -r efe3cfb91966 annotate.py --- a/annotate.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1789 +0,0 @@ -#!/usr/bin/env python - -""" -Annotate program node structures. The code in this module operates upon nodes -which are produced when simplifying AST node trees originating from the compiler -module. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - --------- - -To use this module, the easiest approach is to use the load function: - -load(filename, builtins) - -To control module importing, an importer should be constructed and employed. -Here, the standard path for module searching is used: - -importer = Importer(sys.path) -load(filename, builtins, importer) - -Underneath the load function, the annotate function provides support for -annotating modules already processed by simplify and fixnames: - -annotate(module, builtins) - -And at the most basic level, the most intricate approach involves obtaining an -Annotator object: - -annotator = Annotator() - -Then, processing an existing module with it: - -annotator.process(module) - -If a module containing built-in classes and functions has already been -annotated, such a module should be passed in as an additional argument: - -annotator.process(module, builtins) -""" - -from simplified import * -import simplify, fixnames # for the load function -import compiler -import os - -class System: - - """ - A class maintaining the state of the annotation system. When the system - counter can no longer be incremented by any annotation operation, the - system may be considered stable and fully annotated. - """ - - def __init__(self): - self.count = 0 - - def init(self, node, attr="types"): - - "Initialise a 'node' for annotation." - - if not hasattr(node, attr): - setattr(node, attr, set()) - - def annotate(self, node, types, attr="types"): - - "Annotate the given 'node' with the given 'types'." - - self.init(node, attr) - self.combine(getattr(node, attr), types) - - def combine(self, target, types): - - """ - Combine the 'target' list with the given 'types', counting new members. - """ - - for type in types: - if type not in target: - target.add(type) - self.count += 1 - -system = System() - -# Exceptions. - -class AnnotationError(SimplifiedError): - - "An error in the annotation process." - - pass - -class AnnotationMessage(Exception): - - "A lesser annotation error." - - pass - -# Annotation. - -class Annotator(Visitor): - - """ - The type annotator which traverses the program nodes, typically depth-first, - and maintains a record of the current set of types applying to the currently - considered operation. Such types are also recorded on the nodes, and a - special "system" record is maintained to monitor the level of annotation - activity with a view to recognising when no more annotations are possible. - - Throughout the annotation activity, type information consists of lists of - Attribute objects where such objects retain information about the context of - the type (since a value in the program may be associated with an object or - class) and the actual type of the value being manipulated. Upon accessing - attribute information on namespaces, additional accessor information is also - exchanged - this provides a means of distinguishing between the different - types possible when the means of constructing the namespace may depend on - run-time behaviour. - - Covered: Assign, CheckType, Conditional, Global, Import, InvokeRef, - InvokeFunction, LoadAttr, LoadExc, LoadName, LoadRef, LoadTemp, - Module, Not, Pass, Raise, ReleaseTemp, ReturnFromBlock, - ReturnFromFunction, StoreAttr, StoreName, StoreTemp, Subprogram, - Try. - """ - - def __init__(self, importer=None): - - "Initialise the visitor with an optional 'importer'." - - Visitor.__init__(self) - self.system = system - self.importer = importer or Importer() - - # Satisfy visitor issues. - - self.visitor = self - - def process(self, module, builtins=None): - - """ - Process the given 'module', using the optional 'builtins' to access - built-in classes and functions. - """ - - self.subprograms = [] - self.current_subprograms = [] - self.current_namespaces = [] - self.rerun_subprograms = {} - self.namespace = None - self.module = module - - # Process the module, supplying builtins if possible. - - self.builtins = builtins - self.global_namespace = Namespace() - - if builtins is not None: - self.builtins_namespace = builtins.namespace - else: - self.builtins_namespace = self.global_namespace - - # NOTE: Not declaring module namespace usage, even though it is used. - - self.process_node(module, self.global_namespace, 0) - - def process_node(self, node, locals, using_module_namespace): - - """ - Process a subprogram or module 'node', indicating the initial 'locals'. - Note that this method may mutate nodes in the original program. - """ - - # Recursion test. - - if node in self.current_subprograms: - if not self.rerun_subprograms.has_key(node): - self.rerun_subprograms[node] = [] - self.rerun_subprograms[node].append(locals) - return - - # Record the current subprogram and namespace. - - self.current_subprograms.append(node) - - # Determine the namespace. - - self.current_namespaces.append(self.namespace) - self.namespace = locals - - # Add namespace details to any structure involved. - - if getattr(node, "structure", None) is not None: - node.structure.namespace = Namespace() - - # Initialise bases where appropriate. - - if hasattr(node.structure, "bases"): - base_refs = [] - for base in node.structure.bases: - self.dispatch(base) - base_refs.append(self.namespace.types) - node.structure.base_refs = base_refs - - # Dispatch to the code itself. - - node.namespace = self.namespace - self.set_module_namespace(using_module_namespace) - - self.dispatch(node) - self.extract_results(node) - - while self.rerun_subprograms.has_key(node): - all_rerun_locals = self.rerun_subprograms[node] - del self.rerun_subprograms[node] - for rerun_locals in all_rerun_locals: - #print "Re-running", node, "with", rerun_locals - - self.namespace = rerun_locals - node.namespace = rerun_locals - self.set_module_namespace(using_module_namespace) - - self.dispatch(node) - self.extract_results(node) - - # Restore the previous subprogram and namespace. - - self.namespace = self.current_namespaces.pop() - self.current_subprograms.pop() - self.reset_module_namespace(using_module_namespace) - - def set_module_namespace(self, using_module_namespace): - - """ - In order to keep global accesses working, the module namespace must be - adjusted. - """ - - if using_module_namespace: - self.module.namespace = self.namespace - - def reset_module_namespace(self, using_module_namespace): - - """ - In order to keep global accesses working, the module namespace must be - reset. - """ - - if using_module_namespace: - self.module.namespace = self.namespace - - def extract_results(self, node): - - "Extract results from the namespace." - - node.namespace = self.namespace - self.system.annotate(node, self.namespace.raises, "raises") - self.system.annotate(node, self.namespace.returns, "returns") - if hasattr(node, "return_locals"): - node.return_locals.update(self.namespace.return_locals) - - def annotate(self, node, types=None): - - """ - Annotate the given 'node' in the system, using either the optional - 'types' or the namespace's current type information. - """ - - if types is None: - self.system.annotate(node, self.namespace.types) - else: - self.system.annotate(node, types) - - def annotate_parameters(self, node, items): - - """ - Annotate the given 'node' using the given 'items' and updating the - system's annotation counter. - """ - - if not hasattr(node, "paramtypes"): - node.paramtypes = {} - - for param, types in items: - if not node.paramtypes.has_key(param): - node.paramtypes[param] = set() - self.system.combine(node.paramtypes[param], types) - - # Visitor methods. - - def default(self, node): - - """ - Process the given 'node', given that it does not have a specific - handler. - """ - - raise AnnotationMessage, "Node '%s' not supported." % node - - def dispatch(self, node, *args): - try: - Visitor.dispatch(self, node, *args) - except AnnotationError, exc: - exc.add(node) - raise - except AnnotationMessage, exc: - raise AnnotationError(exc, node) - - # Specific node methods. - - def visitAssign(self, assign): - - """ - Process the 'assign' node and its contents. - """ - - self.dispatches(assign.code) - - def visitCheckType(self, checktype): - - """ - Process the 'checktype' node, finding the possible types of the - exception, and processing each choice to build a list of checked types - for the exception. - """ - - inverted = getattr(checktype, "inverted", 0) - self.dispatch(checktype.expr) - - expr_types = self.namespace.types - choice_types = set() - choices = [] - - for choice in checktype.choices: - choices.append(self.dispatch(choice)) - choice_types.update(self.namespace.types) - - for expr_type in expr_types: - in_choices = expr_type.type.get_class() in choice_types - - # Filter out types not in the choices list unless the operation is - # inverted; in which case, filter out types in the choices list. - - if not inverted and not in_choices or inverted and in_choices: - self._prune_non_accesses(checktype.expr, expr_type) - - def visitConditional(self, conditional): - - """ - Process the 'conditional' node, processing the test, body and else - clauses and recording their processed forms. The body and else clauses - are processed within their own namespaces, and the test is also - processed in its own namespace if 'isolate_test' is set on the - 'conditional' node. - """ - - # Conditionals keep local namespace changes isolated. - # With Return nodes inside the body/else sections, the changes are - # communicated to the caller. - - is_module = self.namespace is self.module.namespace - - # Where the test is closely associated with the body, save the namespace - # before entering the test. - - if conditional.isolate_test: - saved_namespace = self.namespace - self.namespace = Namespace() - if is_module: - self.module.namespace = self.namespace - self.namespace.merge_namespace(saved_namespace) - - self.dispatch(conditional.test) - - # Where the test may affect the body and the else clause, save the - # namespace after processing the test. - - if not conditional.isolate_test: - saved_namespace = self.namespace - self.namespace = Namespace() - if is_module: - self.module.namespace = self.namespace - self.namespace.merge_namespace(saved_namespace) - - # NOTE: Exception recording. - - else: - test_raises = set() - test_raises.update(self.namespace.raises) - - # Process the body clause. - - self.dispatches(conditional.body) - body_namespace = self.namespace - - # Use the saved namespace as a template for the else clause. - - self.namespace = Namespace() - if is_module: - self.module.namespace = self.namespace - self.namespace.merge_namespace(saved_namespace) - - # Process the else clause. - - self.dispatches(conditional.else_) - else_namespace = self.namespace - - # Merge the body and else namespaces. - - self.namespace = Namespace() - if is_module: - self.module.namespace = self.namespace - self.namespace.merge_namespace(body_namespace) - self.namespace.merge_namespace(else_namespace) - - # NOTE: Test of exception type pruning based on the test/body. - # Note that the checked exceptions are tested for re-raising. - - if conditional.isolate_test: - for exc_type in test_raises: - if exc_type not in body_namespace.raises: - self.namespace.revoke_exception_type(exc_type) - - def visitGlobal(self, global_): - - """ - Leave the 'global_' node unprocessed since namespaces should have - already been altered to take global names into consideration. - """ - - pass - - def visitImport(self, import_): - - """ - Process the 'import_' node, importing the module with the stated name - and storing details on the node. - """ - - module = self.importer.load(import_.name, self.builtins, getattr(import_, "alias", None)) - if module is not None: - self.namespace.set_types(set([module])) - else: - self.namespace.set_types(set()) - self.annotate(import_) # mainly for viewing purposes - - def _visitInvoke(self, invoke, invocation_types, have_args): - - """ - Process the 'invoke' node, using the given 'invocation_types' as the - list of callables to be investigated for instantiation or for the - invocation of functions or blocks. If 'have_args' is a true value, any - invocation or instantiation will involve arguments. - """ - - # Now locate and invoke the subprogram. This can be complicated because - # the target may be a class or object, and there may be many different - # related subprograms. - - invocations = [] - - # Visit each callable in turn, finding subprograms. - - for attr in invocation_types: - - # Deal with class invocations by providing instance objects. - # Here, each class is queried for the __init__ method, which may - # exist for some combinations of classes in a hierarchy but not for - # others. - - if isinstance(attr.type, Class): - attributes = get_attributes(attr.type, "__init__") - - # Deal with object invocations by using __call__ methods. - - elif isinstance(attr.type, Instance): - attributes = get_attributes(attr.type, "__call__") - - # Normal functions or methods are more straightforward. - # Here, we model them using an attribute with no context and with - # no associated accessor. - - else: - attributes = [(attr, None)] - - # Inspect each attribute and extract the subprogram. - - for attribute, accessor in attributes: - - # If a class is involved, presume that it must create a new - # object. - - if isinstance(attr.type, Class): - - # Instantiate the class. - - instance = self.new_instance(invoke, attr.type) - - # For instantiations, switch the context. - - if attribute is not None: - attribute = Attribute(instance, attribute.type) - - # Request an instance-specific initialiser. - - attribute = attr.type.get_attribute_for_instance(attribute, instance) - - # Skip cases where no callable is found. - - if attribute is not None: - - # If a subprogram is defined, invoke it. - - self.invoke_subprogram(invoke, attribute) - if attribute.type not in invocations: - invocations.append(attribute.type) - - elif not isinstance(attr.type, Class): - print "Invocation type is None for", accessor - - else: - - # Test to see if no arguments were supplied in cases where no - # initialiser was found. - - if have_args: - raise AnnotationMessage, "No initialiser found for '%s' with arguments." % attr.type - - # Special case: initialisation. - - if isinstance(attr.type, Class): - - # Associate the instance with the result of this invocation. - - self.namespace.set_types(set([Attribute(None, instance)])) - self.annotate(invoke) - - # Remember the invocations that were found, along with the return type - # information. - - invoke.invocations = invocations - self.namespace.set_types(getattr(invoke, "types", set())) - - def visitInvokeRef(self, invoke): - - """ - Process the 'invoke' node, first finding the callables indicated by the - reference. - """ - - # Where the invocation belongs to an instance but the invoked subprogram - # does not, request a special copy. - - instance = getattr(invoke, "instance", None) - if instance is not None and getattr(invoke.ref, "instance", None) is None: - if invoke.ref.copies.has_key(instance): - invoke.ref = invoke.ref.copies[instance] - else: - invoke.ref = invoke.ref.copy(instance) - #print "Created", invoke.ref, "for", getattr(invoke.ref, "instance", None) - invoke.ref.module.simplifier.subnames[invoke.ref.full_name()] = invoke.ref - invocation_types = [Attribute(None, invoke.ref)] - self._visitInvoke(invoke, invocation_types, have_args=0) - - def visitInvokeFunction(self, invoke): - - """ - Process the 'invoke' node, first finding the callables indicated by the - expression. - """ - - self.dispatch(invoke.expr) - invocation_types = self.namespace.types - - # Invocation processing starts with making sure that the arguments have - # been processed. - - self._visitInvoke(invoke, invocation_types, have_args=self.process_args(invoke)) - - def visitLoadAttr(self, loadattr): - - """ - Process the 'loadattr' node, processing and storing the expression, and - using the expression's types to construct records of accesses and - non-accesses using the stated attribute name. - """ - - self.dispatch(loadattr.expr) - types = set() - raises = set() - non_accesses = [] - accesses = {} - - # For each expression type... - - for attr in self.namespace.types: - - # Find types for the named attribute. - - attributes = get_attributes(attr.type, loadattr.name) - - # Where no attributes exist... - - if not attributes: - - # Register new invalid accesses and mark a possible exception. - - if not attr in non_accesses: - non_accesses.append(attr) - exc = self.get_builtin_instances(loadattr, "AttributeError") - raises.update(exc) - self.namespace.raises.update(exc) - - # Revoke this type from any name involved. - - self._prune_non_accesses(loadattr.expr, attr) - - # For each type found... - - for attribute, accessor in attributes: - - # For actual attributes, register the type and remember the - # access. - - if attribute is not None: - types.add(attribute) - if not accesses.has_key(attr.type): - accesses[attr.type] = [] - if not (attribute, accessor) in accesses[attr.type]: - accesses[attr.type].append((attribute, accessor)) - - # Otherwise, register new invalid accesses and note a possible - # exception. - - else: - if not attr in non_accesses: - non_accesses.append(attr) - exc = self.get_builtin_instances(loadattr, "AttributeError") - raises.update(exc) - self.namespace.raises.update(exc) - - # Revoke this type from any name involved. - - self._prune_non_accesses(loadattr.expr, attr) - - if not types: - print "No attribute found for", loadattr.name, "given", self.namespace.types - - # Remember the result types. - - self.namespace.set_types(types) - loadattr.non_accesses = non_accesses - loadattr.accesses = accesses - loadattr.raises = raises - self.annotate(loadattr) - - def _prune_non_accesses(self, expr, attr): - - """ - Prune type information from 'expr' where the given 'attr' has been - shown to be a non-access. - """ - - if isinstance(expr, LoadName): - self.namespace.revoke(expr.name, attr) - elif isinstance(expr, LoadExc): - self.namespace.revoke_exception_type(attr) - elif isinstance(expr, LoadTemp): - self.namespace.revoke_temp_type(getattr(expr, "index", None), attr) - - # LoadAttr cannot be pruned since this might unintentionally prune - # legitimate types from other applications of the referenced type, it - # almost certainly doesn't take "concurrent" mutation into - # consideration (where in a running program, the pruned type is actually - # reintroduced, making the pruning invalid), and there is no easy way of - # preserving the meaning of a namespace without either creating lots of - # specialised instances, and even then... - - #elif isinstance(expr, LoadAttr): - # for expr_attr in expr.expr.types: - # if hasattr(expr_attr.type, "namespace"): - # expr_attr.type.namespace.revoke(expr.name, attr) - - def visitLoadExc(self, loadexc): - - """ - Process the 'loadexc' node, discovering the possible exception types - raised. - """ - - self.namespace.set_types(self.namespace.raises) - self.annotate(loadexc) - - def visitLoadName(self, loadname): - - """ - Process the 'loadname' node, processing the name information on the node - to determine which types are involved with the name. - """ - - self.namespace.set_types(self.namespace.load(loadname.name)) - self.annotate(loadname) - - def visitLoadRef(self, loadref): - - """ - Process the 'loadref' node, obtaining type information about the - reference stated on the node. - """ - - self.namespace.set_types(set([Attribute(None, loadref.ref)])) - self.annotate(loadref) - - def visitLoadTemp(self, loadtemp): - - """ - Process the 'loadtemp' node, obtaining type information about the - temporary variable accessed, and removing variable information where the - 'release' attribute has been set on the node. - """ - - index = getattr(loadtemp, "index", None) - try: - if getattr(loadtemp, "release", 0): - self.namespace.set_types(self.namespace.temp[index].pop()) - else: - self.namespace.set_types(self.namespace.temp[index][-1]) - except KeyError: - raise AnnotationMessage, "Temporary store index '%s' not defined." % index - self.annotate(loadtemp) - - def visitMakeTuple(self, maketuple): - - """ - Process the 'maketuple' node and its contents. - """ - - # Get a tuple and populate it with type information for the contents. - - tuples = self.get_builtin_instances(maketuple, "tuple") - - # NOTE: This is dependent on the tuple definition in the builtins. - - for node in maketuple.nodes: - self.dispatch(node) - for t in tuples: - t.type.namespace.add("value", self.namespace.types) - - self.namespace.set_types(tuples) - self.annotate(maketuple) - - def visitModule(self, module): - - """ - Process the 'module' and its contents. - """ - - self.dispatches(module.code) - - def visitNot(self, not_): - - "Process the 'not_' node and its expression." - - self.dispatch(not_.expr) - - def visitPass(self, pass_): - - "Leave the 'pass_' node unprocessed." - - pass - - def visitRaise(self, raise_): - - """ - Process the 'raise_' node, processing any traceback information along - with the raised exception expression, converting the node into a kind of - invocation where the expression is found not to be an invocation itself. - This node affects the namespace, adding exception types to the list of - those raised in the namespace. - """ - - if getattr(raise_, "traceback", None) is not None: - self.dispatch(raise_.traceback) - self.dispatch(raise_.expr) - - # Handle bare name exceptions by converting any classes to instances. - - if not isinstance(raise_.expr, InvokeFunction): - raise_.pos_args = [] - raise_.kw_args = {} - raise_.star = None - raise_.dstar = None - types = set() - for attr in self.namespace.types: - if isinstance(attr.type, Class): - self._visitInvoke(raise_, [attr], have_args=0) - types.update(self.namespace.types) - else: - types = self.namespace.types - - self.namespace.raises.update(types) - - def visitReleaseTemp(self, releasetemp): - - """ - Process the 'releasetemp' node, removing temporary variable information - from the current namespace. - """ - - index = getattr(releasetemp, "index", None) - try: - self.namespace.temp[index].pop() - except KeyError: - raise AnnotationMessage, "Temporary store index '%s' not defined." % index - except IndexError: - pass #raise AnnotationMessage, "Temporary store index '%s' is empty." % index - - def visitResetExc(self, resetexc): - self.namespace.raises = set() - - def visitReturn(self, return_): - - """ - Process the 'return_' node, processing any expression and obtaining type - information to be accumulated in the current namespace's list of return - types. A snapshot of the namespace is taken for the purposes of - reconciling or merging namespaces where subprograms actually share - locals with their callers. - """ - - if hasattr(return_, "expr"): - self.dispatch(return_.expr) - self.namespace.returns.update(self.namespace.types) - self.annotate(return_) - self.namespace.snapshot() - - visitReturnFromBlock = visitReturn - visitReturnFromFunction = visitReturn - - def visitStoreAttr(self, storeattr): - - """ - Process the 'storeattr' node, processing the expression and target, and - using the type information obtained to build records of legitimate - writes to the stated attribute, along with "impossible" non-writes to - the attribute. - """ - - self.dispatch(storeattr.expr) - expr = self.namespace.types - self.dispatch(storeattr.lvalue) - writes = {} - non_writes = [] - for attr in self.namespace.types: - # NOTE: Impose "atomic" constraints on certain types. - if attr is None: - if not attr in non_writes: - non_writes.append(attr) - continue - attr.type.namespace.add(storeattr.name, expr) - writes[attr.type] = attr.type.namespace.load(storeattr.name) - if not writes: - print "Unable to store attribute", storeattr.name, "given", self.namespace.types - storeattr.writes = writes - storeattr.non_writes = non_writes - - def visitStoreName(self, storename): - - """ - Process the 'storename' node, processing the expression on the node and - associating the type information obtained with the stated name in the - current namespace. - """ - - self.dispatch(storename.expr) - self.namespace.store(storename.name, self.namespace.types) - self.annotate(storename) - - def visitStoreTemp(self, storetemp): - - """ - Process the 'storetemp' node, processing the expression on the node and - associating the type information obtained with a temporary variable in - the current namespace. - """ - - self.dispatch(storetemp.expr) - index = getattr(storetemp, "index", None) - if not self.namespace.temp.has_key(index): - self.namespace.temp[index] = [] - self.namespace.temp[index].append(self.namespace.types) - - def visitSubprogram(self, subprogram): - - """ - Process the 'subprogram' node, processing its contents (a group of nodes - comprising the subprogram). - """ - - self.dispatches(subprogram.code) - - def visitTry(self, try_): - - """ - Process the 'try_' node, processing the body clause in its own namespace - derived from the current namespace, processing any handler clause using - the namespace information accumulated in the body, and processing any - else and finally clauses, attempting to supply each with appropriate - namespace information. - """ - - is_module = self.namespace is self.module.namespace - - self.dispatches(try_.body) - - # Save the namespace from the body. - - body_namespace = Namespace() - body_namespace.merge_namespace(self.namespace) - - # Process the handler. - - if hasattr(try_, "handler"): - self.dispatches(try_.handler) - - # Save the namespace from the handler. - - handler_namespace = Namespace() - handler_namespace.merge_namespace(self.namespace) - - # Remember the raised exceptions encountered so far. - - raises = self.namespace.raises - - # Process the else clause. - - if hasattr(try_, "else_"): - - # Restore the body namespace for the else clause. - - self.namespace = body_namespace - if is_module: - self.module.namespace = self.namespace - - # Empty the raised exceptions for the else clause. - - self.namespace.raises = set() - self.dispatches(try_.else_) - self.namespace.raises = raises - - # Merge the namespaces. - - self.namespace = Namespace() - if is_module: - self.module.namespace = self.namespace - self.namespace.merge_namespace(body_namespace) - self.namespace.merge_namespace(handler_namespace) - - # Process the finally clause, if any. - - self.dispatches(try_.finally_) - - def visitYield(self, yield_): - raise NotImplementedError, "The yield statement is not currently supported." - - # Utility methods. - - def get_builtin_instances(self, node, name): - return set([Attribute(None, self.new_instance(node, attr.type)) for attr in self.builtins.namespace[name]]) - - def new_instance(self, node, type): - - "For the given 'node', obtain an instance from the given 'type'." - - if not type.has_instance(node): - instance = Instance() - instance.namespace = Namespace() - instance.namespace.store("__class__", set([Attribute(None, type)])) - type.add_instance(node, instance) - else: - instance = type.get_instance(node) - - return instance - - def invoke_subprogram(self, invoke, attribute): - - """ - Invoke using the given 'invoke' node the subprogram represented by the - given 'attribute'. - """ - - # Test for context information, making it into a real attribute. - - if attribute.context is not None: - context = Attribute(None, attribute.context) - target = attribute.type - else: - context = None - target = attribute.type - - # Test to see if anything has changed. - - if hasattr(invoke, "syscount") and invoke.syscount.has_key(target) and invoke.syscount[target] == self.system.count: - return - - # Remember the state of the system. - - else: - if not hasattr(invoke, "syscount"): - invoke.syscount = {} - invoke.syscount[target] = self.system.count - - # Provide the correct namespace for the invocation. - # This may be a "shared" namespace... - - if getattr(invoke, "share_locals", 0): - namespace = Namespace() - namespace.merge_namespace(self.namespace, everything=0) - using_module_namespace = self.namespace is self.module.namespace - - # Or it may be a structure... - - elif getattr(target, "structure", None): - namespace = Namespace() - using_module_namespace = 0 - - # Or it may be a new namespace populated with the supplied parameters. - - else: - items = self.make_items(invoke, target, context) - namespace = Namespace() - namespace.merge_items(items) - using_module_namespace = 0 - - # NOTE: Avoid PEP 227 (nested scopes) whilst permitting references to a - # NOTE: subprogram within itself. Do not define the name of the function - # NOTE: within a method definition. - - if getattr(target, "name", None) is not None and not getattr(target, "is_method", 0): - namespace.store(target.name, set([Attribute(None, target)])) - - # Process the subprogram. - - self.process_node(target, namespace, using_module_namespace) - - # NOTE: Improve and verify this. - # If the invocation returns a value, acquire the return types. - - if getattr(target, "returns_value", 0): - self.namespace.set_types(target.returns) - self.annotate(invoke) - - # If it is a normal block, merge the locals. - # This can happen in addition to the above because for things like - # logical expressions, the namespace can be modified whilst values are - # returned as results. - - if getattr(invoke, "share_locals", 0): - self.namespace.reset() - - # Merge the locals snapshots. - - for locals in target.return_locals: - - # For blocks returning values (such as operations), do not merge - # snapshots or results. - - if getattr(target, "returns_value", 0): - self.namespace.merge_namespace(locals, everything=0) - - # For blocks not returning values (such as loops), merge - # snapshots and results since they contain details of genuine - # returns. - - else: - self.namespace.merge_namespace(locals) - - # Incorporate any raised exceptions. - - if not hasattr(invoke, "raises"): - invoke.raises = set() - invoke.raises.update(target.raises) - self.namespace.raises.update(target.raises) - - def process_args(self, invocation): - - """ - Process the arguments associated with an 'invocation'. Return whether - any arguments were processed. - """ - - self.dispatches(invocation.pos_args) - self.dispatch_dict(invocation.kw_args) - - # Get type information for star and dstar arguments. - - if invocation.star is not None: - param, default = invocation.star - self.dispatch(default) - invocation.star = param, default - - if invocation.dstar is not None: - param, default = invocation.dstar - self.dispatch(default) - invocation.dstar = param, default - - if invocation.pos_args or invocation.kw_args or invocation.star or invocation.dstar: - return 1 - else: - return 0 - - def make_items(self, invocation, subprogram, context): - - """ - Make an items mapping for the 'invocation' of the 'subprogram' using the - given 'context' (which may be None). - """ - - # NOTE: Support class methods! - - if context is not None and isinstance(context.type, Instance): - pos_args = [Self(context)] + invocation.pos_args - else: - pos_args = invocation.pos_args - - # Duplicate the keyword arguments - we remove them in processing below. - - kw_args = {} - kw_args.update(invocation.kw_args) - - # Sort the arguments into positional and keyword arguments. - - params = subprogram.params - items = [] - star_args = [] - - # Match each positional argument, taking excess arguments as star args. - - for arg in pos_args: - if params: - param, default = params[0] - if arg is None: - arg = default - if hasattr(arg, "types"): - items.append((param, arg.types)) - else: - items.append((param, set())) # Annotation has not succeeded. - params = params[1:] - else: - star_args.append(arg) - - # Collect the remaining defaults. - - while params: - param, default = params[0] - if kw_args.has_key(param): - arg = kw_args[param] - del kw_args[param] - elif default is not None: - self.dispatch(default) - arg = default - else: - raise AnnotationMessage, "No argument supplied in '%s' for parameter '%s'." % (subprogram, param) - if hasattr(arg, "types"): - items.append((param, arg.types)) - else: - items.append((param, set())) # Annotation has not succeeded. - params = params[1:] - - dstar_args = kw_args.items() - - # Construct temporary objects. - - if star_args: - star_invocation = self.make_star_args(invocation, subprogram, star_args) - self.dispatch(star_invocation) - star_types = star_invocation.types - else: - star_types = None - - if dstar_args: - dstar_invocation = self.make_dstar_args(invocation, subprogram, dstar_args) - self.dispatch(dstar_invocation) - dstar_types = dstar_invocation.types - else: - dstar_types = None - - # NOTE: Merge the objects properly. - - star_types = star_types or invocation.star and invocation.star.types - dstar_types = dstar_types or invocation.dstar and invocation.dstar.types - - # Add star and dstar. - - if star_types is not None: - if subprogram.star is not None: - param, default = subprogram.star - items.append((param, star_types)) - else: - raise AnnotationMessage, "Invocation provides unwanted *args." - elif subprogram.star is not None: - param, default = subprogram.star - if not hasattr(default, "types"): - subprogram.star = param, self.dispatch(default) # NOTE: Review reprocessing. - items.append((param, default.types)) - - if dstar_types is not None: - if subprogram.dstar is not None: - param, default = subprogram.dstar - items.append((param, dstar_types)) - else: - raise AnnotationMessage, "Invocation provides unwanted **args." - elif subprogram.dstar is not None: - param, default = subprogram.dstar - if not hasattr(default, "types"): - subprogram.dstar = param, self.dispatch(default) # NOTE: Review reprocessing. - items.append((param, default.types)) - - # Record the parameter types. - - self.annotate_parameters(subprogram, items) - return subprogram.paramtypes.items() - - def make_star_args(self, invocation, subprogram, star_args): - - "Make a subprogram which initialises a list containing 'star_args'." - - if not hasattr(invocation, "stars"): - invocation.stars = {} - - if not invocation.stars.has_key(subprogram.full_name()): - instance = getattr(invocation, "instance", None) - - code = [ - Return( - instance=instance, - expr=MakeTuple( - instance=instance, - nodes=star_args - ) - ) - ] - - new_subprogram = Subprogram( - instance=instance, - name=None, - returns_value=1, - params=[], - star=None, - dstar=None, - code=code - ) - - subprogram.module.simplifier.subnames[new_subprogram.full_name()] = new_subprogram - - invocation.stars[subprogram.full_name()] = InvokeRef( - invocation.original, - instance=instance, - produces_result=1, - ref=new_subprogram - ) - - return invocation.stars[subprogram.full_name()] - - def make_dstar_args(self, invocation, subprogram, dstar_args): - - """ - Make a subprogram which initialises a dictionary built from the given - 'dstar_args'. - """ - - if not hasattr(invocation, "dstars"): - invocation.dstars = {} - - if not invocation.dstars.has_key(subprogram.full_name()): - instance = getattr(invocation, "instance", None) - - code=[ - StoreTemp( - instance=instance, - expr=InvokeFunction( - invocation.original, - instance=instance, - expr=LoadAttr( - instance=instance, - expr=LoadRef( - instance=instance, - ref=self.builtins - ), - name="dict", - nstype="module", - ) - ) - ) - ] - - for arg, value in dstar_args: - - # NOTE: Constant not added to table. - - constant = Constant(name=repr(arg), value=arg) - code += [ - StoreTemp( - instance=instance, - expr=InvokeFunction( - instance=instance, - expr=LoadName( - instance=instance, - name=constant.typename - ) - ), - index="const" - ), - InvokeFunction( - invocation.original, - instance=instance, - expr=LoadAttr( - instance=instance, - expr=LoadTemp( - instance=instance - ), - name="__setitem__" - ), - args=[ - LoadTemp( - instance=instance, - index="const", - release=1 - ), - value - ] - ) - ] - - code += [ - Return( - instance=instance, - expr=LoadTemp( - instance=instance, - release=1 - ) - ) - ] - - new_subprogram = Subprogram( - instance=instance, - name=None, - returns_value=1, - params=[], - star=None, - dstar=None, - code=code - ) - subprogram.module.simplifier.subnames[new_subprogram.full_name()] = new_subprogram - - invocation.dstars[subprogram.full_name()] = InvokeRef( - invocation.original, - instance=instance, - produces_result=1, - ref=new_subprogram - ) - - return invocation.dstars[subprogram.full_name()] - -# Namespace-related abstractions. - -class Namespace: - - """ - A local namespace which may either relate to a genuine set of function - locals or the initialisation of a structure or module. - """ - - def __init__(self): - - """ - Initialise the namespace with a mapping of local names to possible - types, a list of return values and of possible returned local - namespaces. The namespace also tracks the "current" types and a mapping - of temporary value names to types. - """ - - self.names = {} - self.returns = set() - self.return_locals = set() - self.raises = set() - self.temp = {} - self.types = set() - - def set_types(self, types): - - "Set the current collection of 'types'." - - self.types = types.copy() - - def add(self, name, types): - - "Add to the entry with the given 'name' the specified 'types'." - - if self.names.has_key(name): - self.names[name].update(types) - else: - self.store(name, types) - - def store(self, name, types): - - "Store in (or associate with) the given 'name' the specified 'types'." - - self.names[name] = types.copy() - - __setitem__ = store - - def load(self, name): - - "Load the types associated with the given 'name'." - - return self.names[name] - - __getitem__ = load - - def has_key(self, name): - return self.names.has_key(name) - - def revoke(self, name, type): - - "Revoke from the entry for the given 'name' the specified 'type'." - - new_types = self.names[name].copy() - new_types.remove(type) - self.names[name] = new_types - - def revoke_exception_type(self, type): - - "Revoke the given 'type' from the collection of exception types." - - self.raises.remove(type) - - def revoke_temp_type(self, index, type): - - "Revoke from the temporary variable 'index' the given 'type'." - - new_types = self.temp[index][-1].copy() - new_types.remove(type) - self.temp[index][-1] = new_types - - def merge_namespace(self, namespace, everything=1): - - """ - Merge items from the given 'namespace' with this namespace. When the - optional 'everything' parameter is set to a false value (unlike the - default), return values and locals snapshots will not be copied to this - namespace. - """ - - self.merge_items(namespace.names.items()) - self.raises.update(namespace.raises) - if everything: - self.returns.update(namespace.returns) - self.return_locals.update(namespace.return_locals) - for name, values in namespace.temp.items(): - if values: - if not self.temp.has_key(name) or not self.temp[name]: - self.temp[name] = [set()] - self.temp[name][-1].update(values[-1]) - - def merge_items(self, items): - - "Merge the given 'items' with this namespace." - - for name, types in items: - self.merge(name, types) - - def merge(self, name, types): - - "Merge the entry for the given 'name' and 'types' with this namespace." - - if not self.names.has_key(name): - self.names[name] = types.copy() - else: - existing = self.names[name] - existing.update(types) - - def snapshot(self): - - "Make a snapshot of the locals and remember them." - - namespace = Namespace() - namespace.merge_namespace(self) - self.return_locals.add(namespace) - - def reset(self): - - "Reset a namespace in preparation for merging with returned locals." - - self.names = {} - - def __repr__(self): - return repr(self.names) + " (temp) " + repr(self.temp) - -class Importer: - - "An import machine, searching for and loading modules." - - def __init__(self, path=None): - - """ - Initialise the importer with the given search 'path' - a list of - directories to search for Python modules. - """ - - self.path = path or [os.getcwd()] - self.path.append(libdir) - self.modules = {} - - def find_in_path(self, name): - - """ - Find the given module 'name' in the search path, returning None where no - such module could be found, or a 2-tuple from the 'find' method - otherwise. - """ - - for d in self.path: - m = self.find(d, name) - if m: return m - return None - - def find(self, d, name): - - """ - In the directory 'd', find the given module 'name', where 'name' can - either refer to a single file module or to a package. Return None if the - 'name' cannot be associated with either a file or a package directory, - or a 2-tuple from '_find_package' or '_find_module' otherwise. - """ - - m = self._find_package(d, name) - if m: return m - m = self._find_module(d, name) - if m: return m - return None - - def _find_module(self, d, name): - - """ - In the directory 'd', find the given module 'name', returning None where - no suitable file exists in the directory, or a 2-tuple consisting of - None (indicating that no package directory is involved) and a filename - indicating the location of the module. - """ - - name_py = name + os.extsep + "py" - filename = self._find_file(d, name_py) - if filename: - return None, filename - return None - - def _find_package(self, d, name): - - """ - In the directory 'd', find the given package 'name', returning None - where no suitable package directory exists, or a 2-tuple consisting of - a directory (indicating the location of the package directory itself) - and a filename indicating the location of the __init__.py module which - declares the package's top-level contents. - """ - - filename = self._find_file(d, name) - if filename: - init_py = "__init__" + os.path.extsep + "py" - init_py_filename = self._find_file(filename, init_py) - if init_py_filename: - return filename, init_py_filename - return None - - def _find_file(self, d, filename): - - """ - Return the filename obtained when searching the directory 'd' for the - given 'filename', or None if no actual file exists for the filename. - """ - - filename = os.path.join(d, filename) - if os.path.exists(filename): - return filename - else: - return None - - def load(self, name, builtins, alias=None): - - """ - Load the module or package with the given 'name' and using the specified - 'builtins'. Return an Attribute object referencing the loaded module or - package, or None if no such module or package exists. - """ - - if self.modules.has_key(name): - return Attribute(None, self.modules[name]) - - path = name.split(".") - m = self.find_in_path(path[0]) - if not m: - return None # NOTE: Import error. - d, filename = m - - if self.modules.has_key(path[0]): - top = module = self.modules[path[0]] - else: - top = module = self.modules[path[0]] = load(filename, builtins, path[0], self, no_annotate=1) - annotate(module, builtins, self) - - if len(path) > 1: - path_so_far = path[:1] - for p in path[1:]: - path_so_far.append(p) - m = self.find(d, p) - if not m: - return None # NOTE: Import error. - d, filename = m - module_name = ".".join(path_so_far) - - if self.modules.has_key(module_name): - submodule = self.modules[module_name] - else: - submodule = self.modules[module_name] = load(filename, builtins, module_name, self, no_annotate=1) - annotate(submodule, builtins, self) - - # Store the submodule within its parent module. - - module.namespace[p] = [Attribute(None, submodule)] - module = submodule - - if alias: - return Attribute(None, module) - else: - return Attribute(None, top) - -def combine(target, additions): - - """ - Merge into the 'target' sequence the given 'additions', preventing duplicate - items. - """ - - for addition in additions: - if addition not in target: - target.append(addition) - -def find_attributes(structure, name): - - """ - Find for the given 'structure' all attributes for the given 'name', visiting - base classes where appropriate and returning the attributes in order of - descending precedence for all possible base classes. - - The elements in the result list are 2-tuples which contain the attribute and - the structure involved in accessing the attribute. - """ - - # First attempt to search the instance/class namespace. - - try: - l = structure.namespace.load(name) - attributes = [] - for attribute in l: - attributes.append((attribute, structure)) - - # If that does not work, attempt to investigate any class or base classes. - - except KeyError: - attributes = [] - - # Investigate any instance's implementing class. - - if isinstance(structure, Instance): - for attr in structure.namespace.load("__class__"): - cls = attr.type - l = get_attributes(cls, name) - combine(attributes, l) - - # Investigate any class's base classes. - - elif isinstance(structure, Class): - - # If no base classes exist, return an indicator that no attribute - # exists. - - if not structure.base_refs: - return [(None, structure)] - - # Otherwise, find all possible base classes. - - for base_refs in structure.base_refs: - base_attributes = [] - - # For each base class, find attributes either in the base - # class or its own base classes. - - for base_ref in base_refs: - l = get_attributes(base_ref, name) - combine(base_attributes, l) - - combine(attributes, base_attributes) - - return attributes - -def get_attributes(structure, name): - - """ - Return all possible attributes for the given 'structure' having the given - 'name', wrapping each attribute in an Attribute object which includes - context information for the attribute access. - - The elements in the result list are 2-tuples which contain the attribute and - the structure involved in accessing the attribute. - """ - - if isinstance(structure, Attribute): - structure = structure.type - results = [] - for attribute, accessor in find_attributes(structure, name): - - # Detect class attribute access via instances. - - if attribute is not None and isinstance(structure, Instance) and isinstance(accessor, Class): - attribute = accessor.get_attribute_for_instance(attribute, structure) - - # Produce an attribute with the appropriate context. - - if attribute is not None and isinstance(structure, Structure): - results.append((Attribute(structure, attribute.type), accessor)) - else: - results.append((attribute, accessor)) - - return results - -def prompt(vars): - try: - while 1: - s = raw_input("> ") - print eval(s, vars) - except EOFError: - pass - -# Convenience functions. - -def load(name, builtins=None, module_name=None, importer=None, no_annotate=0): - - """ - Load the module with the given 'name' (which may be a full module path), - using the optional 'builtins' to resolve built-in names, and using the - optional 'importer' to provide a means of finding and loading modules. - """ - - module = simplify.simplify(name, builtins is None, module_name) - fixnames.fix(module, builtins) - if not no_annotate: - annotate(module, builtins, importer) - return module - -def annotate(module, builtins=None, importer=None): - - """ - Annotate the given 'module', also employing the optional 'builtins' module, - if specified. If the optional 'importer' is given, use that to find and load - modules. - """ - - annotator = Annotator(importer) - if builtins is not None: - annotator.process(module, builtins) - else: - annotator.process(module) - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 fixnames.py --- a/fixnames.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,489 +0,0 @@ -#!/usr/bin/env python - -""" -Fix name-related operations. The code in this module operates upon simplified -program node trees. - -Copyright (C) 2006 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - --------- - -To use this module, the easiest approach is to use the fix function: - -fix(module) - -The more complicated approach involves instantiating a Fixer object: - -fixer = Fixer() - -Then, applying the fixer to an existing module: - -fixer.process(module) - -If a module containing built-in classes and functions exists, apply the fixer as -follows: - -fixer.process(module, builtins) -""" - -from simplified import * - -# Fixing of name-related operations. - -class Fixer(Visitor): - - """ - The name fixer which traverses the program nodes in a module, typically - depth-first, and maintains a record of name usage in the different - namespaces. As a consequence of various observations, some parts of the - program node tree are modified with different operations employed to those - originally defined. - - There are two kinds of subprograms in modules: functions/methods and - internal subprograms which support things like loops. The latter kind of - subprogram may acquire the locals from their callers and must therefore be - traversed with information from such callers. Thus, we choose the top-level - code and all functions/methods as roots for processing, following - invocations of internal subprograms in order to reach all subprograms that - are defined in each module. - - top-level - ... - invoke function - ... - invoke loop -> subprogram (internal) - ... - - subprogram (function) - ... - invoke loop -> subprogram (internal) - ... - - ... - - The above approach should guarantee that all subprograms are traversed and - that all name lookups are correctly categorised. - """ - - def __init__(self): - - "Initialise the name fixer." - - Visitor.__init__(self) - - # Satisfy visitor issues. - - self.visitor = self - - def process(self, module, builtins=None): - - """ - Process the given 'module' optionally using some 'builtins' to reference - built-in objects. - """ - - # The fixer maintains a list of transformed subprograms (added for each - # of the processing "roots" and also for each invoked internal - # subprogram), along with a list of current subprograms (used to avoid - # recursion issues) and a list of current namespaces (used to recall - # namespaces upon invoking internal subprograms). - - self.subprograms = [] - self.current_subprograms = [] - self.current_namespaces = [] - - # First, process the top-level code, finding out which names are - # defined at that level. - - self.global_namespace = None - self.module = module - self.builtins = builtins or module - - self.process_node(self.module) - - # Then, process all functions and methods, providing a global namespace. - # By setting a global namespace, we influence the resolution of names: - # those which are global to the top-level module (processed above) are - # considered as built-in names, whereas those which are global to a - # function or method are searched for in the global namespace. - - self.global_namespace = self.namespace - - for subprogram in self.module.simplifier.subprograms: - - # Internal subprograms are skipped here and processed specially via - # Invoke nodes. - - if not getattr(subprogram, "internal", 0): - self.subprograms.append(self.process_node(subprogram)) - - # Ultimately, we redefine the list of subprograms on the visitor. - - self.module.simplifier.subprograms = self.subprograms - return self.module - - def process_node(self, node, namespace=None): - - """ - Process a subprogram or module 'node', discovering from attributes on - 'node' any initial locals. Return a modified subprogram or module. - """ - - # Do not process subprograms already being processed. - - if node in self.current_subprograms: - return None - - # Obtain a namespace either based on locals or on a structure. - - structure = structure=getattr(node, "structure", None) - - # If passed some namespace, use that as the current namespace. - - if namespace is not None: - self.namespace.merge_namespace(namespace) - else: - self.namespace = NameOrganiser(structure) - - # Record the current subprogram and namespace. - - self.current_subprograms.append(node) - self.current_namespaces.append(self.namespace) - - # NOTE: Avoid PEP 227 (nested scopes) whilst permitting references to a - # NOTE: subprogram within itself. Do not define the name of the function - # NOTE: within a method definition. - - if isinstance(node, Subprogram) and getattr(node, "name", None) is not None and not getattr(node, "is_method", 0): - self.namespace.store(node.name) - - # Register the names of parameters in the namespace. - - if hasattr(node, "params"): - new_params = [] - for param, default in node.params: - new_params.append((param, self.dispatch(default))) - self.namespace.store(param) - node.params = new_params - if getattr(node, "star", None): - param, default = node.star - self.namespace.store(param) - node.star = param, self.dispatch(default) - if getattr(node, "dstar", None): - param, default = node.dstar - self.namespace.store(param) - node.dstar = param, self.dispatch(default) - - # Add namespace details to any structure involved. - - if hasattr(node, "structure") and node.structure is not None: - - # Initialise bases where appropriate. - - if hasattr(node.structure, "bases"): - bases = [] - for base in node.structure.bases: - bases.append(self.dispatch(base)) - node.structure.bases = bases - - # Dispatch to the code itself. - - result = self.dispatch(node) - result.organiser = self.namespace - - # Restore the previous subprogram and namespace. - - self.current_namespaces.pop() - if self.current_namespaces: - self.namespace = self.current_namespaces[-1] - self.current_subprograms.pop() - - return result - - # Visitor methods. - - def default(self, node): - - """ - Process the given 'node', given that it does not have a specific - handler. - """ - - for attr in ("pos_args",): - value = getattr(node, attr, None) - if value is not None: - setattr(node, attr, self.dispatches(value)) - for attr in ("kw_args",): - value = getattr(node, attr, None) - if value is not None: - setattr(node, attr, self.dispatch_dict(value)) - for attr in ("expr", "lvalue", "test", "star", "dstar"): - value = getattr(node, attr, None) - if value is not None: - setattr(node, attr, self.dispatch(value)) - for attr in ("body", "else_", "handler", "finally_", "code", "choices", "nodes"): - value = getattr(node, attr, None) - if value is not None: - setattr(node, attr, self.dispatches(value)) - return node - - def dispatch(self, node, *args): - return Visitor.dispatch(self, node, *args) - - def visitGlobal(self, global_): - for name in global_.names: - self.namespace.make_global(name) - return global_ - - def visitLoadName(self, loadname): - - "Transform the 'loadname' node to a specific, scope-sensitive node." - - scope = self.namespace.find_for_load(loadname.name) - - # For structure namespaces, load an attribute. - - if scope == "structure": - result = self.dispatch( - LoadAttr(loadname.original, loadname.defining, - expr=LoadRef(loadname.original, - ref=self.namespace.structure), - name=loadname.name, - nstype="structure") - ) - - # For global accesses (ie. those outside the local namespace)... - - elif scope == "global": - - # Where a distinct global namespace exists, examine it. - - if self.global_namespace is not None: - scope = self.global_namespace.find_for_load(loadname.name) - - # Where the name is outside the global namespace, it must be a - # built-in. - - if scope == "global": - result = self.dispatch( - LoadAttr(loadname.original, loadname.defining, - expr=LoadRef(loadname.original, - ref=self.builtins), - name=loadname.name, - nstype="module") - ) - - # Otherwise, it is within the global namespace and must be a - # global. - - else: - result = self.dispatch( - LoadAttr(loadname.original, loadname.defining, - expr=LoadRef(loadname.original, - ref=self.module), - name=loadname.name, - nstype="module") - ) - - # Where no global namespace exists, we are at the module level and - # must be accessing a built-in. - - else: - result = self.dispatch( - LoadAttr(loadname.original, loadname.defining, - expr=LoadRef(loadname.original, - ref=self.builtins), - name=loadname.name, - nstype="module") - ) - - # For local accesses... - - else: - - # Where a distinct global namespace exists, it must be a local. - - if self.global_namespace is not None: - result = loadname - - # Otherwise, we must be accessing a global (which is local at the - # module level). - - else: - result = self.dispatch( - LoadAttr(loadname.original, loadname.defining, - expr=LoadRef(loadname.original, - ref=self.module), - name=loadname.name, - nstype="module") - ) - - return result - - def visitStoreName(self, storename): - - "Transform the 'storename' node to a specific, scope-sensitive node." - - scope = self.namespace.find_for_store(storename.name) - - # For structure namespaces, store an attribute. - - if scope == "structure": - self.namespace.store(storename.name) - - return self.dispatch( - StoreAttr(storename.original, storename.defining, - lvalue=LoadRef(storename.original, - ref=self.namespace.structure), - name=storename.name, - expr=storename.expr, - nstype="structure") - ) - - # Where the name is outside the local namespace, disallow any built-in - # assignment and store the name globally. - - elif scope == "global": - return self.dispatch( - StoreAttr(storename.original, storename.defining, - lvalue=LoadRef(storename.original, - ref=self.module), - name=storename.name, - expr=storename.expr, - nstype="module") - ) - - # For local namespace accesses... - - else: - self.namespace.store(storename.name) - - # If a distinct global namespace exists, it must be a local access. - - if self.global_namespace is not None: - return storename - - # Otherwise, the name is being set at the module level and is - # considered global. - - else: - return self.dispatch( - StoreAttr(storename.original, storename.defining, - lvalue=LoadRef(storename.original, - ref=self.module), - name=storename.name, - expr=storename.expr, - nstype="module") - ) - - def visitInvokeFunction(self, invoke): - - "Transform the 'invoke' node, performing processing on subprograms." - - return self.default(invoke) - - def visitInvokeRef(self, invoke): - - "Transform the 'invoke' node, performing processing on subprograms." - - # The special case of internal subprogram invocation is addressed by - # propagating namespace information to the subprogram and processing it. - - if invoke.share_locals: - subprogram = self.process_node(invoke.ref, self.namespace) - else: - subprogram = self.process_node(invoke.ref) - - if subprogram is not None: - self.subprograms.append(subprogram) - return invoke - -class ScopeMismatch(Exception): - pass - -class NameOrganiser: - - """ - A local namespace which may either relate to a genuine set of function - locals or the initialisation of a structure. - """ - - def __init__(self, structure=None): - - "Initialise the namespace with an optional 'structure'." - - self.structure = structure - if structure is not None: - self.local = "structure" - else: - self.local = "local" - - # Names may be self.local or "global". - - self.names = {} - - def make_global(self, name): - if not self.names.has_key(name): - self.names[name] = "global" - elif self.names[name] == self.local: - raise ScopeMismatch, "Name '%s' already considered as %s." % (name, self.local) - - def find_for_load(self, name): - return self.names.get(name, "global") - - def find_for_store(self, name): - return self.names.get(name, self.local) - - def store(self, name): - if self.names.get(name) != "global": - self.names[name] = self.local - else: - raise ScopeMismatch, "Name '%s' already considered as global." % name - - def merge(self, name, scope): - if self.names.get(name) in (None, scope): - self.names[name] = scope - else: - raise ScopeMismatch, "Name '%s' already considered as %s." % (name, self.names[name]) - - def merge_namespace(self, namespace): - self.merge_items(namespace.names.items()) - - def merge_items(self, items): - for name, scope in items: - self.merge(name, scope) - - def __repr__(self): - return repr(self.names) - -# Convenience functions. - -def fix(module, builtins=None): - - """ - Fix the names in the given 'module', also employing the optional 'builtins' - module, if specified. - """ - - fixer = Fixer() - if builtins is not None: - fixer.process(module, builtins) - else: - fixer.process(module) - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplified/__init__.py --- a/simplified/__init__.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -""" -Simplified program representation. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -""" - -__version__ = "0.1" - -from simplified.ast import * -from simplified.data import * -from simplified.program import * -from simplified.utils import * -import os - -# Location of the built-in libraries. -# NOTE: Change this if the package structure changes. - -libdir = os.path.join(os.path.split(os.path.split(__file__)[0])[0], "lib") - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplified/ast.py --- a/simplified/ast.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -""" -Additional program AST nodes. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -""" - -class Self: - - """ - A program node encapsulating object/context information in an argument list. - This is not particularly like Attribute, Class, Instance or other such - things, since it actually appears in the program representation. - """ - - def __init__(self, attribute): - self.types = set() - self.types.add(attribute) - -class Op: - - "A replacement AST node representing an operation in a Compare construct." - - def __init__(self, name, expr): - self.name = name - self.expr = expr - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplified/data.py --- a/simplified/data.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,237 +0,0 @@ -#!/usr/bin/env python - -""" -Simplified program data. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -""" - -from simplified.utils import Structure, WithName, name - -# Special non-program nodes. - -class _Class(Structure, WithName): - - "A Python class." - - def __init__(self, *args, **kw): - Structure.__init__(self, *args, **kw) - WithName.__init__(self) - - def full_name(self): - return "class %s" % self._full_name - -class SingleInstanceClass(_Class): - - "A Python class." - - def __init__(self, *args, **kw): - _Class.__init__(self, *args, **kw) - self.instance = None - - def has_instance(self, node): - return self.instance is not None - - def add_instance(self, node, instance): - self.instance = instance - - def get_instance(self, node): - return self.instance - - def get_instance_name(self, instance): - return self._full_name - - # Attribute propagation. - - def get_attribute_for_instance(self, attribute, instance): - return attribute - -class MultipleInstanceClass(_Class): - - "A Python class." - - def __init__(self, *args, **kw): - _Class.__init__(self, *args, **kw) - self.instances = {} - self.attributes_for_instances = {} - - def _get_key(self, node): - return id(getattr(node, "original", None)) # self.module.original - - def has_instance(self, node): - return self.instances.has_key(self._get_key(node)) - - def add_instance(self, node, instance): - self.instances[self._get_key(node)] = instance - - def get_instance(self, node): - return self.instances[self._get_key(node)] - - def get_instance_name(self, instance): - return name(instance, self._full_name) - - # Attribute propagation. - - def get_attribute_for_instance(self, attribute, instance): - - # Create specialised methods. - - if isinstance(attribute.type, Subprogram): - subprogram = attribute.type - - # Each instance may have its own version of the subprogram. - - key = (subprogram, instance) - if not self.attributes_for_instances.has_key(key): - new_subprogram = subprogram.copy(instance, subprogram.full_name()) - subprogram.module.simplifier.subnames[new_subprogram.full_name()] = new_subprogram - self.attributes_for_instances[key] = Attribute(attribute.context, new_subprogram) - print "New subprogram", new_subprogram, "for", key - - return self.attributes_for_instances[key] - - # The original nodes are returned for other attributes. - - else: - return attribute - -class SelectiveMultipleInstanceClass(MultipleInstanceClass): - - "A Python class which provides multiple instances depending on the class." - - def _get_key(self, node): - if self.namespace.has_key("__atomic__"): - return id(self) - else: - return MultipleInstanceClass._get_key(self, node) - -class ProlificMultipleInstanceClass(MultipleInstanceClass): - - """ - A Python class which provides multiple instances for different versions of - methods. In order to avoid unbounded instance production (since new - instances cause new copies of methods which in turn would cause new - instances), a relations dictionary is maintained which attempts to map - "requesting instances" to existing instances, suggesting such instances in - preference to new ones. - """ - - def __init__(self, *args, **kw): - MultipleInstanceClass.__init__(self, *args, **kw) - self.instance_relations = {} - - def _get_key(self, node): - if self.namespace.has_key("__atomic__"): - return id(self) - else: - return id(node) - - def has_instance(self, node): - requesting_instance = getattr(node, "instance", None) - #return requesting_instance is not None and requesting_instance.get_class() is self or \ - return self.instance_relations.has_key(requesting_instance) or self.instances.has_key(self._get_key(node)) - - def add_instance(self, node, instance): - requesting_instance = getattr(node, "instance", None) - print "New instance", instance, "for", id(node), requesting_instance - self.instances[self._get_key(node)] = instance - if requesting_instance is not None: - self.instance_relations[requesting_instance] = instance - requesting_instance.get_class().instance_relations[instance] = requesting_instance - - def get_instance(self, node): - requesting_instance = getattr(node, "instance", None) - #if requesting_instance is not None and requesting_instance.get_class() is self: - # return requesting_instance - return self.instance_relations.get(requesting_instance) or self.instances[self._get_key(node)] - -class Instance(Structure): - - "An instance." - - def full_name(self): - return self.get_class().get_instance_name(self) - - def get_class(self): - for n in self.namespace.load("__class__"): - return n.type - else: - raise ValueError, "__class__" - - def __repr__(self): - return "Instance of type '%s'" % self.full_name() - - def __eq__(self, other): - # NOTE: Single instance: all instances are the same - # NOTE: Multiple instances: all instances are different - return self.full_name() == other.full_name() - - def __hash__(self): - return id(self) - -class Constant: - - "A constant initialised with a type name for future processing." - - def __init__(self, name, value): - self.name = name - self.value = value - self.typename = self.value.__class__.__name__ - -class Attribute: - - """ - An attribute abstraction, indicating the type of the attribute along with - its context or origin. - """ - - def __init__(self, context, type): - self.context = context - self.type = type - - def __eq__(self, other): - return hasattr(other, "type") and other.type == self.type or other == self.type - - def __repr__(self): - return "Attribute(%s, %s)" % (repr(self.context), repr(self.type)) - - def __hash__(self): - return id(self.type) - -# Configuration setting. - -Class = SingleInstanceClass -#Class = MultipleInstanceClass - -def set_single_instance_mode(): - global Class - Class = SingleInstanceClass - -def set_multiple_instance_mode(): - global Class - Class = MultipleInstanceClass - -def set_selective_multiple_instance_mode(): - global Class - Class = SelectiveMultipleInstanceClass - -def set_prolific_multiple_instance_mode(): - global Class - Class = ProlificMultipleInstanceClass - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplified/program.py --- a/simplified/program.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,412 +0,0 @@ -#!/usr/bin/env python - -""" -Simplified program nodes for easier type propagation and analysis. This module -contains nodes representing program instructions or operations, program -structure or organisation, and abstract program data. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -""" - -from simplified.utils import Structure, WithName, name -import sys - -# Simplified program nodes. - -class Node: - - """ - A result node with common attributes: - - original The original node from which this node was created. - defining Whether the node defines something in the original program. - name Any name involved (variable or attribute). - index Any index involved (temporary variable name). - value Any constant value. - ref Any reference to (for example) subprograms. - nstype Any indication of the namespace type involved in a name access. - - Expression-related attributes: - - expr Any contributing expression. - lvalue Any target expression. - test Any test expression in a conditional instruction. - - Invocation and subprogram attributes: - - args Any collection of argument nodes. - params Any collection of parameter nodes and defaults. - - Tuple construction attributes: - - nodes Any expressions used to initialise a tuple - - Statement-grouping attributes: - - body Any conditional code depending on the success of a test. - else_ Any conditional code depending on the failure of a test. - handler Any exception handler code. - finally_ Any code which will be executed regardless. - code Any unconditional code. - choices Any choices which may be included in the final program. - """ - - common_attributes = "name", "index", "value", "nstype", "internal", "returns_value", "is_method", "ref", "module", "structures", "original" - expression_attributes = "expr", "lvalue", "test" - argument_attributes = "star", "dstar" - invocation_attributes = "params", # not "args" - see "pos_args", "kw_args" - grouping_attributes = "code", "body", "else_", "handler", "finally_", "choices" - - def __init__(self, original=None, defining=0, **kw): - - """ - Initialise a program node with a link to an optional 'original' AST - node. An optional 'defining' parameter (if set to a true value), sets - this node as the defining node in the original. - """ - - self.original = original - self.defining = defining - self.copies = {} - - if self.original is not None and defining: - self.original._node = self - for name, value in kw.items(): - setattr(self, name, value) - - # Annotations. - - self.types = set() - - def __repr__(self): - - "Return a readable representation." - - if hasattr(self, "full_name"): - s = "%s '%s'" % (self.__class__.__name__, self.full_name()) - elif hasattr(self, "name"): - s = "%s '%s'" % (self.__class__.__name__, self.name) - elif hasattr(self, "index"): - s = "%s (%s)" % (self.__class__.__name__, self.index) - elif hasattr(self, "value"): - s = "%s %s" % (self.__class__.__name__, repr(self.value)) - elif hasattr(self, "ref"): - s = "%s '%s'" % (self.__class__.__name__, name(self.ref, self.ref.name)) - else: - s = "%s" % (self.__class__.__name__,) - - # Annotations. - - if self.types: - return "%s -> %s" % (s, self.types) - else: - return s - - def _pprint(self, indent, continuation, s, stream=None): - - """ - Print, at the given 'indent' level, with the given 'continuation' text, - the string 's', either to the given, optional 'stream' or to standard - output, this node's "pretty" representation. - """ - - stream = stream or sys.stdout - if continuation: - print >>stream, (" " * max(0, indent - len(continuation))) + continuation + s - else: - print >>stream, (" " * indent) + s - - def pprint(self, indent=0, continuation=None, stream=None): - - """ - Print, at the given, optional 'indent', with the given optional - 'continuation' text, either to the given, optional 'stream' or to - standard output, this node's "pretty" representation along with its - children and their "pretty" representation (and so on). - """ - - stream = stream or sys.stdout - self._pprint(indent, continuation, repr(self), stream) - - # Subprogram-related details. - - if hasattr(self, "params"): - for name, default in self.params: - self._pprint(indent + 2, "( ", "%s default %s" % (name, default), stream=stream) - if hasattr(self, "star") and self.star: - name, default = self.star - self._pprint(indent + 2, "( ", "%s default %s" % (name, default), stream=stream) - if hasattr(self, "dstar") and self.dstar: - name, default = self.dstar - self._pprint(indent + 2, "( ", "%s default %s" % (name, default), stream=stream) - if getattr(self, "internal", 0): - self._pprint(indent + 2, "( ", "internal", stream=stream) - if getattr(self, "structure", 0): - self._pprint(indent + 2, "( ", "structure '%s'" % self.structure.name, stream=stream) - - # Expression-related details. - - if hasattr(self, "expr"): - self.expr.pprint(indent + 2, "- ", stream=stream) - if hasattr(self, "nodes"): - for node in self.nodes: - node.pprint(indent + 2, "- ", stream=stream) - if hasattr(self, "lvalue"): - self.lvalue.pprint(indent + 2, "->", stream=stream) - if hasattr(self, "nstype"): - self._pprint(indent + 2, "", self.nstype, stream=stream) - if hasattr(self, "args"): - for arg in self.pos_args: - arg.pprint(indent + 2, "( ", stream=stream) - for name, arg in self.kw_args.items(): - arg.pprint(indent + 2, "( ", stream=stream) - if hasattr(self, "star") and self.star: - self.star.pprint(indent + 2, "( ", stream=stream) - if hasattr(self, "dstar") and self.dstar: - self.dstar.pprint(indent + 2, "( ", stream=stream) - - # Statement-related details. - - if hasattr(self, "test"): - self.test.pprint(indent + 2, "? ", stream=stream) - for attr in self.grouping_attributes: - if hasattr(self, attr) and getattr(self, attr): - self._pprint(indent, "", "%s {" % attr, stream=stream) - for node in getattr(self, attr): - node.pprint(indent + 2, stream=stream) - self._pprint(indent, "", "}", stream=stream) - - # Annotations. - - if hasattr(self, "accesses"): - self._pprint(indent, "", "--------", stream=stream) - for ref, attributes in self.accesses.items(): - self._pprint(indent + 2, "| ", "when %s: %s" % (ref, ", ".join([("%s via %s" % attr_acc) for attr_acc in attributes])), stream=stream) - self._pprint(indent, "", "--------", stream=stream) - if hasattr(self, "writes"): - self._pprint(indent, "", "--------", stream=stream) - for ref, attribute in self.writes.items(): - self._pprint(indent + 2, "| ", "when %s: %s" % (ref, attribute), stream=stream) - self._pprint(indent, "", "--------", stream=stream) - - # Node discovery functions. - - def active(self): - - "Return the active copies of this node or a list containing this node." - - return self.copies.values() or [self] - - # Node manipulation functions. - - def copy(self, instance=None, new_name=None): - - """ - Perform a deep copy of the node, optionally specifying the 'instance' - for whom the copy has been requested and a 'new_name' for the copied - node. Return new unannotated copies of the node and its descendants. - """ - - # Copy the common attributes of this node. - - common = {} - for attr in self.common_attributes: - if hasattr(self, attr): - common[attr] = getattr(self, attr) - - # Add new attributes specially for copies. - - common["instance"] = instance - - if new_name is not None: - common["copy_of"] = self - common["name"] = new_name - - # Instantiate the copy, avoiding side-effects with original and defining. - - node = self.__class__(**common) - node.defining = self.defining - - # Add links to copies from originals. - - self.copies[instance] = node - - # Copy attributes of different types. - - for attr in self.expression_attributes: - if hasattr(self, attr): - n = getattr(self, attr) - if n is None: - n2 = n - else: - n2 = n.copy(instance) - setattr(node, attr, n2) - - for attr in self.argument_attributes: - if hasattr(self, attr): - t = getattr(self, attr) - if t is None: - t2 = t - else: - name, n = t - n2 = n.copy(instance) - t2 = name, n2 - setattr(node, attr, t2) - - for attr in self.invocation_attributes: - if hasattr(self, attr): - l = getattr(self, attr) - l2 = [] - for name, n in l: - if n is None: - l2.append((name, n)) - else: - l2.append((name, n.copy(instance))) - setattr(node, attr, l2) - - for attr in self.grouping_attributes: - if hasattr(self, attr): - l = getattr(self, attr) - setattr(node, attr, [n.copy(instance) for n in l]) - - # Arguments are usually processed further - "args" is useless. - - if hasattr(self, "pos_args"): - node.pos_args = [n.copy(instance) for n in self.pos_args] - - if hasattr(self, "kw_args"): - node.kw_args = {} - for name, n in self.kw_args.items(): - node.kw_args[name] = n.copy(instance) - - return node - -# These are the supported "operations" described by simplified program nodes. - -class Pass(Node): "A placeholder node corresponding to pass." -class Assign(Node): "A grouping node for assignment-related operations." -class Keyword(Node): "A grouping node for keyword arguments." -class Global(Node): "A global name designator." -class Import(Node): "A module import operation." -class LoadTemp(Node): "Load a previously-stored temporary value." -class LoadName(Node): "Load a named object." -class LoadAttr(Node): "Load an object attribute." -class LoadRef(Node): "Load a reference, typically a subprogram or a constant." -class LoadExc(Node): "Load a handled exception." -class ResetExc(Node): "Reset the exception state." -class StoreTemp(Node): "Store a temporary value." -class StoreName(Node): "Associate a name with an object." -class StoreAttr(Node): "Associate an object's attribute with a value." -class ReleaseTemp(Node): "Release a temporary value." -class Try(Node): "A try...except...else...finally grouping node." -class Raise(Node): "An exception raising node." -class Not(Node): "A negation of an expression." -class CheckType(Node): "Check a value's type from a list of choices." -class Return(Node): "Return an evaluated expression." -class Invoke(Node): "An invocation." -class MakeTuple(Node): "Make a tuple object." - -# There are two types of return node: return from function and return from -# block. - -class ReturnFromFunction(Return): - pass - -class ReturnFromBlock(Return): - pass - -# NOTE: Not actually supported. -# Additionally, yield statements act like return statements for the purposes -# of this system. - -class Yield(ReturnFromFunction): - pass - -# Some behaviour is set as the default in conditional nodes but may be -# overridden. - -class Conditional(Node): - - "A conditional node consisting of a test and outcomes." - - def __init__(self, *args, **kw): - self.isolate_test = 0 - Node.__init__(self, *args, **kw) - -# Invocations involve some more work to process calculated attributes. - -class InvokeFunction(Invoke): - - "A function or method invocation." - - def __init__(self, *args, **kw): - self.args = [] - self.star = None - self.dstar = None - Invoke.__init__(self, *args, **kw) - self.set_args(self.args) - self.share_locals = 0 - - def set_args(self, args): - - "Sort the 'args' into positional and keyword arguments." - - self.pos_args = [] - self.kw_args = {} - add_kw = 0 - for arg in args: - if not add_kw: - if not isinstance(arg, Keyword): - self.pos_args.append(arg) - else: - add_kw = 1 - if add_kw: - if isinstance(arg, Keyword): - self.kw_args[arg.name] = arg.expr - else: - raise TypeError, "Positional argument appears after keyword arguments in '%s'." % self - -class InvokeRef(Invoke): - - "A block or loop invocation." - - def __init__(self, *args, **kw): - self.share_locals = 1 - Invoke.__init__(self, *args, **kw) - -# Program structure nodes. - -class Module(Node, Structure): - - "A Python module." - - def full_name(self): - return "module %s" % self.name - -class Subprogram(Node, WithName): - - "A subprogram: functions, methods and loops." - - def __init__(self, *args, **kw): - Node.__init__(self, *args, **kw) - WithName.__init__(self) - self.raises = set() - self.returns = set() - self.return_locals = set() - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplified/utils.py --- a/simplified/utils.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,167 +0,0 @@ -#!/usr/bin/env python - -""" -Simplified program utilities. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -""" - -from compiler.visitor import ASTVisitor - -# Exceptions. - -class SimplifiedError(Exception): - - "An error in the annotation process." - - def __init__(self, exc, node, *args): - - """ - Initialise the error with an existing exception 'exc', the 'node' at - which this error occurs, along with additional optional arguments. - """ - - Exception.__init__(self, *args) - self.nodes = [node] - self.exc = exc - - def add(self, node): - - "Add the given 'node' to the path of nodes leading from the exception." - - self.nodes.append(node) - - def __str__(self): - - "Return a string showing the principal exception details." - - return "%s, %s" % (self.exc, self.nodes) - -# Elementary visitor support. - -class Visitor(ASTVisitor): - - "A visitor base class." - - def __init__(self): - ASTVisitor.__init__(self) - - def default(self, node, *args): - raise SimplifiedError, (None, node) - - def dispatch(self, node, *args): - return ASTVisitor.dispatch(self, node, *args) - - def dispatches(self, nodes, *args): - results = [] - for node in nodes: - results.append(self.dispatch(node, *args)) - return results - - def dispatch_dict(self, d, *args): - results = {} - for name, node in d.items(): - results[name] = self.dispatch(node, *args) - return results - -# Unique name registration. - -class Naming: - - "Maintain records of unique names for each simple name." - - index_separator = "-" - - def __init__(self): - self.names = {} - - def get(self, obj): - return obj._unique_name - - def set(self, obj, name): - if hasattr(obj, "_unique_name"): - return - if not self.names.has_key(name): - self.names[name] = 0 - n = self.names[name] + 1 - self.names[name] = n - obj._unique_name = "%s%s%d" % (name, self.index_separator, n) - -def name(obj, name): - - "Return a unique name for the given 'obj', indicating the base 'name'." - - naming.set(obj, name) - return naming.get(obj) - -# Naming singleton. - -naming = Naming() - -# Named nodes are those which can be referenced in some way. - -class WithName: - - "Node naming." - - def __init__(self): - - "Initialise the object's full name." - - self._full_name = name(self, self.name or "$untitled") - - def full_name(self): - - "Return the object's full name." - - return self._full_name - -# Comparable nodes based on naming. - -class Comparable: - - "Comparable nodes implementing the 'full_name' method." - - def __eq__(self, other): - - "This object is equal to 'other' if the full names are the same." - - # NOTE: Single instance: all instances are the same - # NOTE: Multiple instances: all instances are different - if hasattr(other, "full_name"): - return self.full_name() == other.full_name() - else: - return NotImplemented - - def __hash__(self): - - "The hash of this object is based on its full name." - - return hash(self.full_name()) - -# Structure nodes indicating namespace-bearing objects. - -class Structure(Comparable): - - "A non-program node containing some kind of namespace." - - def __init__(self, **kw): - for name, value in kw.items(): - setattr(self, name, value) - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify.py --- a/simplify.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1941 +0,0 @@ -#!/usr/bin/env python - -""" -Simplify AST structures for easier type propagation and analysis. The code in -this module processes AST trees originating from the compiler module and -produces a result tree consisting of instruction-oriented program nodes. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - --------- - -To use this module, the easiest approach is to use the simplify function: - -simplify(filename) - -The more complicated approach involves first instantiating a Simplifier object: - -simplifier = Simplifier() - -Then, applying the simplifier to an AST tree: - -module = compiler.parseFile(filename) -simplifier.process(module) -""" - -from simplified import * -import compiler.ast -import os - -class Simplifier(Visitor): - - """ - A simplifying visitor for AST nodes. - - Covered: Add, And, Assert, AssAttr, AssList, AssName, AssTuple, Assign, - AugAssign, Bitand, Break, CallFunc, Class, Compare, Const, - Continue, Dict, Discard, Div, FloorDiv, For, From, Function, - Getattr, Global, If, Import, Invert, Keyword, Lambda, List, - ListComp, ListCompFor, ListCompIf, Mod, Module, Mul, Name, Not, Or, - Pass, Power, Print, Printnl, Raise, Return, Slice, Sliceobj, Stmt, - Sub, Subscript, TryExcept, TryFinally, Tuple, While, UnaryAdd, - UnarySub. - - Missing: Backquote, Bitor, Bitxor, Decorators, Ellipsis, Exec, LeftShift, - RightShift, Yield. - """ - - def __init__(self, builtins=0): - - """ - Initialise the simplifier with the optional 'builtins' parameter - indicating whether the module contains the built-in classes and - functions. - """ - - Visitor.__init__(self) - self.subprograms = [] # Subprograms outside the tree. - self.structures = [] # Structures/classes. - self.constants = {} # Constants. - self.current_subprograms = [] # Current subprograms being processed. - self.current_structures = [] # Current structures being processed. - self.within_class = 0 # Whether a class is being defined. - self.builtins = builtins # Whether the builtins are being processed. - - # Convenience attributes. - - self.subnames = {} - - # For compiler package mechanisms. - - self.visitor = self - - def process(self, node, name): - result = self.dispatch(node, name) - result.simplifier = self - return result - - def dispatch_or_none(self, node, *args): - if node is not None: - return self.dispatch(node, *args) - else: - return LoadName(node, name="None") - - # Top-level transformation. - - def visitModule(self, module, name=None): - - """ - Process the given 'module', producing a Module object which contains the - resulting program nodes. If the optional 'name' is provided, the 'name' - attribute is set on the Module object using a value other than None. - """ - - result = self.module = Module(module, 1, name=name) - result.code = self.dispatch(module.node) - return result - - # Node transformations. - - def visitAdd(self, add): - return self._visitBinary(add, "__add__", "__radd__") - - def visitAnd(self, and_): - - """ - Make a subprogram for the 'and_' node and record its contents inside the - subprogram. Convert... - - And (test) - (test) - ... - - ...to: - - Subprogram -> Conditional (test) -> ReturnFromBlock ... - (else) -> Conditional (test) -> ReturnFromBlock ... - (else) -> ... - """ - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - - # In the subprogram, make instructions which store each operand, test - # for each operand's truth status, and if appropriate return from the - # subprogram with the value of the operand. - - last = and_.nodes[-1] - results = nodes = [] - - for node in and_.nodes: - expr = self.dispatch(node) - - # Return from the subprogram where the test is not satisfied. - - if node is not last: - nodes += [ - StoreTemp(expr=expr), - Conditional( - test=self._visitNot(LoadTemp()), - body=[ - ReturnFromBlock( - expr=LoadTemp() - ) - ], - else_=[ - ReleaseTemp() - # Subsequent operations go here! - ] - ) - ] - - # Put subsequent operations in the else section of this conditional. - - nodes = nodes[-1].else_ - - # For the last operation, return the result. - - else: - nodes.append(ReturnFromBlock(expr=expr)) - - # Finish the subprogram definition. - - subprogram.code = results - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make an invocation of the subprogram. - - result = InvokeRef(and_, 1, produces_result=1, ref=subprogram) - return result - - def visitAssert(self, assert_): - if assert_.fail: - fail_args = [self.dispatch(assert_.fail)] - else: - fail_args = [] - - result = Conditional(assert_, 1, - test=self.dispatch(assert_.test), - body=[], - else_=[ - Raise(assert_, - expr=InvokeFunction(assert_, - expr=LoadName(name="AssertionError"), - args=fail_args, - star=None, - dstar=None - ) - ) - ] - ) - - # Make nice annotations for the viewer. - - assert_._raises = result.else_[0].expr - return result - - # Assignments. - - def visitAssAttr(self, assattr, in_sequence=0): - expr = self._visitAssNameOrAttr(assattr, in_sequence) - lvalue = self.dispatch(assattr.expr) - result = StoreAttr(assattr, 1, name=assattr.attrname, lvalue=lvalue, expr=expr) - return result - - def visitAssign(self, assign): - result = Assign(assign, 1) - store = StoreTemp(expr=self.dispatch(assign.expr)) - release = ReleaseTemp() - result.code = [store] + self.dispatches(assign.nodes, 0) + [release] - return result - - def visitAssList(self, asslist, in_sequence=0): - if not in_sequence: - expr = LoadTemp() - else: - expr = InvokeFunction(asslist, expr=LoadAttr(expr=LoadTemp(), name="next")) - result = Assign(asslist, 1) - store = StoreTemp(expr=InvokeFunction(asslist, expr=LoadAttr(name="__iter__", expr=expr))) - release = ReleaseTemp() - result.code = [store] + self.dispatches(asslist.nodes, 1) + [release] - return result - - visitAssTuple = visitAssList - - def _visitAssNameOrAttr(self, node, in_sequence): - if not in_sequence: - return LoadTemp() - else: - return InvokeFunction(node, expr=LoadAttr(expr=LoadTemp(), name="next")) - - def visitAssName(self, assname, in_sequence=0): - expr = self._visitAssNameOrAttr(assname, in_sequence) - result = StoreName(assname, 1, name=assname.name, expr=expr) - return result - - augassign_methods = { - "+=" : "__iadd__", "-=" : "__isub__", "*=" : "__imul__", "/=" : "__idiv__", - "%=" : "__imod__", "**=" : "__ipow__", "<<=" : "__ilshift__", ">>=" : "__irshift__", - "&=" : "__iand__", "^=" : "__ixor__", "|=" : "__ior__" - } - - def visitAugAssign(self, augassign): - - """ - Convert the augmented assignment... - - AugAssign (node) -> Name | Getattr | Slice | Subscript - (op) - (expr) - - ...to: - - Assign (code) -> StoreTemp (expr) -> InvokeFunction (expr) -> LoadAttr (expr) -> - (name) -> - StoreName (name) -> - (expr) -> LoadTemp - ReleaseTemp - """ - - result = Assign(augassign, 1) - expr = self.dispatch(augassign.expr) - - # Simple augmented assignment: name += expr - - if isinstance(augassign.node, compiler.ast.Name): - result.code = [ - StoreTemp( - expr=InvokeFunction( # referenced below - augassign, - args=[expr], - star=None, - dstar=None, - expr=LoadAttr( - expr=self.dispatch(augassign.node), - name=self.augassign_methods[augassign.op] - ) - ) - ), - StoreName( - expr=LoadTemp(), - name=augassign.node.name), - ReleaseTemp() - ] - - # Make nice annotations for the viewer. - - augassign._op_call = result.code[0].expr - - # Complicated augmented assignment: lvalue.attr += expr - - elif isinstance(augassign.node, compiler.ast.Getattr): - - # -> setattr(, getattr(, "attr").__xxx__(expr)) - - result.code = [ - StoreTemp( - index="expr", - expr=self.dispatch(augassign.node.expr) - ), - StoreTemp( - expr=InvokeFunction( # referenced below - augassign, - args=[expr], star=None, dstar=None, - expr=LoadAttr( - expr=LoadAttr(augassign.node, 1, - expr=LoadTemp(index="expr"), - name=augassign.node.attrname - ), - name=self.augassign_methods[augassign.op] - ) - ) - ), - StoreAttr( - expr=LoadTemp(), - lvalue=LoadTemp(index="expr"), - name=augassign.node.attrname - ), - ReleaseTemp(index="expr"), - ReleaseTemp() - ] - - # Make nice annotations for the viewer. - - augassign._op_call = result.code[1].expr - - # Complicated augassign using slices: lvalue[lower:upper] += expr - - elif isinstance(augassign.node, compiler.ast.Slice): - - # , , -> .__setslice__(, , .__getslice__(, ).__xxx__(expr)) - - result.code = [ - StoreTemp( - index="expr", - expr=self.dispatch(augassign.node.expr) - ), - StoreTemp( - index="lower", - expr=self.dispatch_or_none(augassign.node.lower) - ), - StoreTemp( - index="upper", - expr=self.dispatch_or_none(augassign.node.upper) - ), - StoreTemp( - expr=InvokeFunction( # referenced below - augassign, - args=[expr], star=None, dstar=None, - expr=LoadAttr( - expr=self._visitSlice( - augassign.node, - LoadTemp(index="expr"), - LoadTemp(index="lower"), - LoadTemp(index="upper"), - "OP_APPLY"), - name=self.augassign_methods[augassign.op] - ) - ) - ), - self._visitSlice( - augassign.node, - LoadTemp(index="expr"), - LoadTemp(index="lower"), - LoadTemp(index="upper"), - "OP_ASSIGN", - LoadTemp() - ), - ReleaseTemp(index="expr"), - ReleaseTemp(index="lower"), - ReleaseTemp(index="upper"), - ReleaseTemp() - ] - - # Make nice annotations for the viewer. - - augassign._op_call = result.code[3].expr - - # Complicated augassign using subscripts: lvalue[subs] += expr - - elif isinstance(augassign.node, compiler.ast.Subscript): - - # , -> .__setitem__(, .__getitem__().__xxx__(expr)) - - result.code = [ - StoreTemp(index="expr", expr=self.dispatch(augassign.node.expr)), - StoreTemp(index="subs", expr=self._visitSubscriptSubs(augassign.node, augassign.node.subs)), - StoreTemp( - expr=InvokeFunction( # referenced below - augassign, - args=[expr], star=None, dstar=None, - expr=LoadAttr( - expr=self._visitSubscript( - augassign.node, - LoadTemp(index="expr"), - LoadTemp(index="subs"), - "OP_APPLY" - ), - name=self.augassign_methods[augassign.op] - ) - ) - ), - self._visitSubscript( - augassign.node, - LoadTemp(index="expr"), - LoadTemp(index="subs"), - "OP_ASSIGN", - LoadTemp() - ), - ReleaseTemp(index="expr"), - ReleaseTemp(index="subs"), - ReleaseTemp() - ] - - # Make nice annotations for the viewer. - - augassign._op_call = result.code[2].expr - - else: - raise NotImplementedError, augassign.node.__class__ - - return result - - def visitBitand(self, bitand): - - """ - Make a subprogram for the 'bitand' node and record its contents inside the - subprogram. Convert... - - Bitand (node) - (node) - ... - - ...to: - - Subprogram -> Conditional (test) -> ReturnFromBlock ... - (else) -> Conditional (test) -> ReturnFromBlock ... - (else) -> ... - """ - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - - # In the subprogram, make instructions which store each operand, test - # for each operand's truth status, and if appropriate return from the - # subprogram with the value of the operand. - - last = bitand.nodes[-1] - results = nodes = [] - - # Start by storing the first operand. - - nodes += [ - StoreTemp(expr=self.dispatch(bitand.nodes[0])) - ] - - # For viewing purposes, record invocations on the AST node. - - bitand._ops = [] - - for node in bitand.nodes[1:]: - - # Make a new AST-style node to wrap the operation program nodes. - - new_op = Op("&", node) - bitand._ops.append(new_op) - - # Generate the operation involving the previous result and the - # current operand. - - expr = self._visitBinaryOp(new_op, LoadTemp(), self.dispatch(node), "__and__", "__rand__") - - # Return from the subprogram where the test is not satisfied. - - if node is not last: - nodes += [ - StoreTemp(expr=expr), - Conditional( - test=self._visitNot(LoadTemp()), - body=[ - ReturnFromBlock( - expr=LoadTemp() - ) - ], - else_=[ - # Subsequent operations go here! - ] - ) - ] - - # Put subsequent operations in the else section of this conditional. - - nodes = nodes[-1].else_ - - # For the last operation, return the result. - - else: - nodes.append(ReturnFromBlock(expr=expr)) - - # Finish the subprogram definition. - - subprogram.code = results - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make an invocation of the subprogram. - - result = InvokeRef(bitand, 1, produces_result=1, ref=subprogram) - return result - - def visitBreak(self, break_): - result = ReturnFromBlock(break_, 1) - return result - - def visitCallFunc(self, callfunc): - result = InvokeFunction(callfunc, 1, star=None, dstar=None, args=self.dispatches(callfunc.args)) - if callfunc.star_args is not None: - result.star = self.dispatch(callfunc.star_args) - if callfunc.dstar_args is not None: - result.dstar = self.dispatch(callfunc.dstar_args) - result.expr = self.dispatch(callfunc.node) - return result - - def visitClass(self, class_): - - # Add "object" if the class is not "object" and has an empty bases list. - - if class_.name != "object" and not class_.bases: - bases = [compiler.ast.Name("object")] - else: - bases = class_.bases - - structure = Class(name=class_.name, module=self.module, bases=self.dispatches(bases)) - self.structures.append(structure) - within_class = self.within_class - self.within_class = 1 - - # Make a subprogram which initialises the class structure. - - subprogram = Subprogram(name=None, module=self.module, structure=structure, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - self.current_structures.append(structure) # mostly for name construction - - # The class is initialised using the code found inside. - - subprogram.code = self.dispatch(class_.code) + [ReturnFromBlock()] - - self.within_class = within_class - self.current_structures.pop() - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make a definition of the class associating it with a name. - - result = Assign( - code=[ - StoreName(class_, 1, # defines the class - name=class_.name, - expr=LoadRef(ref=structure) - ), - InvokeRef( - class_, - share_locals=0, # override the local sharing usually in InvokeRef - ref=subprogram - ) - ] - ) - return result - - comparison_methods = { - "==" : ("__eq__", "__ne__"), - "!=" : ("__ne__", "__eq__"), - "<" : ("__lt__", "__gt__"), - "<=" : ("__le__", "__ge__"), - ">=" : ("__ge__", "__le__"), - ">" : ("__gt__", "__lt__"), - "is" : None, - "is not" : None, - "in" : None, - "not in" : None - } - - def visitCompare(self, compare): - - """ - Make a subprogram for the 'compare' node and record its contents inside - the subprogram. Convert... - - Compare (expr) - (name/node) - ... - - ...to: - - InvokeRef -> Subprogram -> Conditional (test) -> (body) - (else) -> Conditional (test) -> (body) - (else) -> ... - """ - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - - # In the subprogram, make instructions which invoke a method on the - # first operand of each operand pair and, if appropriate, return with - # the value from that method. - - last = compare.ops[-1] - previous = self.dispatch(compare.expr) - results = nodes = [] - - # For viewing purposes, record invocations on the AST node. - - compare._ops = [] - - for op in compare.ops: - op_name, node = op - - # Make a new AST-style node to wrap the operation program nodes. - - new_op = Op(op_name, node) - compare._ops.append(new_op) - - expr = self.dispatch(node) - - # Identify the operation and produce the appropriate method call. - - method_names = self.comparison_methods[op_name] - if method_names: - first_name, alternative_name = method_names - invocation = self._visitBinaryCompareOp(new_op, previous, expr, first_name, alternative_name) - - elif op_name == "is": - invocation = InvokeFunction( - new_op, 1, - expr=LoadName(name="__is__"), - args=[previous, expr], - star=None, - dstar=None) - - elif op_name == "is not": - invocation = Not( - new_op, 1, - expr=InvokeFunction( - new_op, - expr=LoadName(name="__is__"), - args=[previous, expr], - star=None, - dstar=None) - ) - - elif op_name == "in": - invocation = InvokeFunction( - new_op, 1, - expr=LoadAttr( - expr=previous, - name="__contains__" - ), - args=[expr], - star=None, - dstar=None) - - elif op_name == "not in": - invocation = Not( - new_op, 1, - expr=InvokeFunction( - new_op, - expr=LoadAttr( - expr=previous, - name="__contains__" - ), - args=[expr], - star=None, - dstar=None) - ) - - else: - raise NotImplementedError, op_name - - nodes.append(StoreTemp(expr=invocation)) - - # Return from the subprogram where the test is not satisfied. - - if op is not last: - nodes.append( - Conditional( - test=self._visitNot(LoadTemp()), - body=[ - ReturnFromBlock(expr=LoadTemp()) - ], - else_=[ - ReleaseTemp() - # Subsequent operations go here! - ] - ) - ) - - # Put subsequent operations in the else section of this conditional. - - nodes = nodes[-1].else_ - - # For the last operation, return the result. - - else: - nodes.append( - ReturnFromBlock(expr=LoadTemp(release=1)) - ) - - previous = expr - - # Finish the subprogram definition. - - subprogram.code = results - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make an invocation of the subprogram. - - result = InvokeRef(compare, 1, produces_result=1, ref=subprogram) - return result - - def visitConst(self, const): - key = "%s-%s" % (const.value.__class__.__name__, const.value) - if not self.constants.has_key(key): - self.constants[key] = Constant(name=repr(const.value), value=const.value) - result = InvokeFunction(const, 1, expr=LoadName(name=self.constants[key].typename)) - return result - - def visitContinue(self, continue_): - result = InvokeRef(continue_, 1, ref=self.current_subprograms[-1]) - return result - - def visitDict(self, dict): - result = InvokeFunction(dict, 1, expr=LoadName(name="dict")) - args = [] - for key, value in dict.items: - tuple = InvokeFunction(dict, expr=LoadName(name="tuple")) - tuple.set_args([self.dispatch(key), self.dispatch(value)]) - args.append(tuple) - result.set_args(args) - return result - - def visitDiscard(self, discard): - return self.dispatch(discard.expr) - - def visitDiv(self, div): - return self._visitBinary(div, "__div__", "__rdiv__") - - def visitFloorDiv(self, floordiv): - return self._visitBinary(floordiv, "__floordiv__", "__rfloordiv__") - - def visitFor(self, for_): - - """ - Make a subprogram for the 'for_' node and record its contents inside the - subprogram. Convert... - - For (assign) - (body) - (else) - - ...to: - - Assign (assign #1) - Invoke -> Subprogram -> Try (body) -> (assign #2) - (body) - Invoke subprogram - (handler) -> ... - (else) -> ... - """ - - return self._visitFor(for_, self.dispatches(for_.body), for_.else_) - - def _visitFor(self, node, body_stmt, else_=None): - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[]) - self.current_subprograms.append(subprogram) - - # Always return from conditional sections/subprograms. - - if else_ is not None: - else_stmt = self.dispatch(else_) + [ReturnFromBlock()] - else: - else_stmt = [ReturnFromBlock()] - - # Wrap the assignment in a try...except statement. - # Inside the body, add a recursive invocation to the subprogram. - - subprogram.code = [ - Try( - body=[ - Assign( - code=[ - StoreTemp( - expr=InvokeFunction(node, - expr=LoadAttr( - expr=LoadTemp(), - name="next" - ) - ) - ), - self.dispatch(node.assign), - ReleaseTemp() - ]) - ] + body_stmt + [ - InvokeRef( - node, - ref=subprogram - ) - ], - handler=[ - Conditional( - test=InvokeFunction( - node, - expr=LoadName(name="isinstance"), - args=[LoadExc(), LoadName(name="StopIteration")], - star=None, - dstar=None), - body=else_stmt, - else_=[ - Raise( - expr=LoadExc() - ) - ] - ) - ], - else_=[], - finally_=[] - ), - ReturnFromBlock() - ] - - # Finish the subprogram definition. - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Obtain an iterator for the sequence involved. - # Then, make an invocation of the subprogram. - - result = Assign(node, 1, - code=[ - StoreTemp( - expr=InvokeFunction( - node, - expr=LoadAttr( - name="__iter__", - expr=self.dispatch(node.list) - ) - ) - ), - InvokeRef(node, ref=subprogram), - ReleaseTemp() - ] - ) - - # Make nice annotations for the viewer. - - node._iter_call = result.code[0].expr - node._next_call = subprogram.code[0].body[0].code[0].expr - - return result - - def visitFrom(self, from_): - result = Assign(from_, 1) - code = [] - _names = [] - code.append( - StoreTemp( - expr=Import(name=from_.modname, alias=1) - ) - ) - from_._modname = code[-1].expr - for name, alias in from_.names: - code.append( - StoreName( - expr=LoadAttr( - expr=LoadTemp(), - name=name), - name=(alias or name) - ) - ) - _names.append(code[-1].expr) - code.append(ReleaseTemp()) - result.code = code - from_._names = _names - return result - - def _visitFunction(self, function, subprogram): - - """ - A common function generator which transforms the given 'function' node - and initialises the given 'subprogram' appropriately. - """ - - # Discover star and dstar parameters. - - if function.flags & 4 != 0: - has_star = 1 - else: - has_star = 0 - if function.flags & 8 != 0: - has_dstar = 1 - else: - has_dstar = 0 - - # Discover the number of defaults and positional parameters. - - ndefaults = len(function.defaults) - npositional = len(function.argnames) - has_star - has_dstar - - # Produce star and dstar parameters with appropriate defaults. - - if has_star: - star = ( - function.argnames[npositional], - self.dispatch(compiler.ast.List([])) - ) - else: - star = None - if has_dstar: - dstar = ( - function.argnames[npositional + has_star], - self.dispatch(compiler.ast.Dict([])) - ) - else: - dstar = None - - params = [] - for i in range(0, npositional - ndefaults): - params.append((function.argnames[i], None)) - - # Process defaults. - - for i in range(0, ndefaults): - default = function.defaults[i] - if default is not None: - params.append((function.argnames[npositional - ndefaults + i], self.dispatch(default))) - else: - params.append((function.argnames[npositional - ndefaults + i], None)) - - subprogram.params = params - subprogram.star = star - subprogram.dstar = dstar - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - def visitFunction(self, function): - - """ - Make a subprogram for the 'function' and record it outside the main - tree. Produce something like the following: - - StoreName (name) - (expr) -> LoadRef (ref) -> Subprogram (params) - (star) - (dstar) - """ - - subprogram = Subprogram(function, name=function.name, module=self.module, structures=self.current_structures[:], - internal=0, returns_value=1, star=None, dstar=None, is_method=self.within_class, original_def=function) - - # Make nice annotations for the viewer. - - function._subprogram = subprogram - - self.current_subprograms.append(subprogram) - within_class = self.within_class - self.within_class = 0 - - subprogram.code = self.dispatch(function.code) + [ReturnFromFunction()] - - self.within_class = within_class - self.current_subprograms.pop() - self._visitFunction(function, subprogram) - - # Make a definition of the function associating it with a name. - - result = StoreName(function, 1, name=function.name, expr=LoadRef(ref=subprogram)) - return result - - def visitGetattr(self, getattr): - result = LoadAttr(getattr, 1, - name=getattr.attrname, - expr=self.dispatch(getattr.expr) - ) - return result - - def visitGlobal(self, global_): - result = Global(global_, 1, - names=global_.names - ) - return result - - def visitIf(self, if_): - - """ - Make conditionals for each test from an 'if_' AST node, adding the body - and putting each subsequent test as part of the conditional's else - section. - - Convert... - - If (test/body) - (test/body) - ... - (else/body) - - ...to: - - Conditional (test) -> (body) - (else) -> Conditional (test) -> (body) - (else) -> ... - """ - - - results = nodes = [] - - # Produce something like... - # expr.__bool__() ? body - - first = 1 - for compare, stmt in if_.tests: - - # Set the first as the defining node. - - test = Conditional(if_, first, - test=InvokeFunction( - if_, - expr=LoadAttr( - expr=self.dispatch(compare), - name="__bool__" - ), - ) - ) - test.body = self.dispatch(stmt) - nodes.append(test) - nodes = test.else_ = [] - first = 0 - - # Add the compound statement from any else clause to the end. - - if if_.else_ is not None: - nodes += self.dispatch(if_.else_) - - result = results[0] - return result - - def visitImport(self, import_): - result = Assign(import_, 1) - code = [] - _names = [] - for path, alias in import_.names: - importer = Import(name=path, alias=alias) - top = alias or path.split(".")[0] - code.append(StoreName(expr=importer, name=top)) - _names.append(code[-1].expr) - result.code = code - import_._names = _names - return result - - def visitInvert(self, invert): - return self._visitUnary(invert, "__invert__") - - def visitKeyword(self, keyword): - result = Keyword(keyword, 1, - name=keyword.name, - expr=self.dispatch(keyword.expr) - ) - return result - - def visitLambda(self, lambda_): - - # Make a subprogram for the function and record it outside the main - # tree. - - subprogram = Subprogram(lambda_, name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=lambda_) - - # Make nice annotations for the viewer. - - lambda_._subprogram = subprogram - - # Process the lambda contents. - - self.current_subprograms.append(subprogram) - subprogram.code = [ReturnFromFunction(expr=self.dispatch(lambda_.code))] - self.current_subprograms.pop() - self._visitFunction(lambda_, subprogram) - - # Get the subprogram reference to the lambda. - - return LoadRef(lambda_, 1, ref=subprogram) - - def visitList(self, list): - - # Make a subprogram for the list construction and record it outside the - # main tree. - - subprogram = Subprogram(list, name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=list) - self.current_subprograms.append(subprogram) - - # Make nice annotations for the viewer. - - list._subprogram = subprogram - - subprogram.code=[ - StoreTemp( - expr=InvokeFunction( - list, - expr=LoadName( - name="list" - ), - args=[], - star=None, - dstar=None - ) - ) - ] - - for node in list.nodes: - subprogram.code.append( - InvokeFunction( - list, - expr=LoadAttr( - expr=LoadTemp(), - name="append" - ), - args=[self.dispatch(node)], - star=None, - dstar=None - ) - ) - - subprogram.code.append( - ReturnFromBlock( - expr=LoadTemp(release=1) - ) - ) - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make an invocation of the subprogram. - - result = InvokeRef(list, 1, - produces_result=1, - ref=subprogram - ) - return result - - def visitListComp(self, listcomp): - - # Make a subprogram for the list comprehension and record it outside the - # main tree. - - subprogram = Subprogram(listcomp, name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=listcomp) - self.current_subprograms.append(subprogram) - - # Make nice annotations for the viewer. - - listcomp._subprogram = subprogram - - # Add a temporary variable. - # Produce for loops within the subprogram. - # Return the result. - - subprogram.code = [ - StoreTemp( - index="listcomp", - expr=InvokeFunction( - expr=LoadName(name="list"), - args=[], - star=None, - dstar=None - ) - ) - ] + self._visitListCompFor(listcomp, listcomp.quals) + [ - ReturnFromBlock( - expr=LoadTemp( - index="listcomp", - release=1 - ) - ) - ] - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make an invocation of the subprogram. - - result = InvokeRef(listcomp, 1, - produces_result=1, - ref=subprogram - ) - return result - - def _visitListCompFor(self, node, quals): - qual = quals[0] - if len(quals) > 1: - body = self._visitListCompFor(node, quals[1:]) - if qual.ifs: - body = self._visitListCompIf(node, qual.ifs, body) - elif qual.ifs: - body = self._visitListCompIf(node, qual.ifs) - else: - body = self._visitListCompBody(node) - return [self._visitFor(qual, body)] - - def _visitListCompIf(self, node, ifs, expr=None): - if_ = ifs[0] - if len(ifs) > 1: - body = self._visitListCompIf(node, ifs[1:], expr) - elif expr is None: - body = self._visitListCompBody(node) - else: - body = expr - return [ - Conditional(if_, 1, - test=InvokeFunction( - if_, - expr=LoadAttr( - expr=self.dispatch(if_.test), - name="__bool__" - ), - ), - body=body, - else_=[] - ) - ] - - def _visitListCompBody(self, node): - return [ - InvokeFunction( - expr=LoadAttr( - expr=LoadTemp(index="listcomp"), - name="append" - ), - args=[self.dispatch(node.expr)], - star=None, - dstar=None - ) - ] - - def visitMod(self, mod): - return self._visitBinary(mod, "__mod__", "__rmod__") - - def visitMul(self, mul): - return self._visitBinary(mul, "__mul__", "__rmul__") - - def visitName(self, name): - result = LoadName(name, 1, name=name.name) - return result - - def _visitNot(self, expr, not_=None): - invocation = InvokeFunction( - not_, # NOTE: May need a real original node. - expr=LoadAttr( - expr=expr, - name="__bool__" - ), - ) - if not_ is not None: - result = Not(not_, 1, expr=invocation) - else: - result = Not(expr=invocation) - return result - - def visitNot(self, not_): - return self._visitNot(self.dispatch(not_.expr), not_) - - def visitOr(self, or_): - - """ - Make a subprogram for the 'or_' node and record its contents inside the - subprogram. Convert... - - Or (test) - (test) - ... - - ...to: - - Subprogram -> Conditional (test) -> ReturnFromBlock ... - (else) -> Conditional (test) -> ReturnFromBlock ... - (else) -> ... - """ - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - - # In the subprogram, make instructions which store each operand, test - # for each operand's truth status, and if appropriate return from the - # subprogram with the value of the operand. - - last = or_.nodes[-1] - results = nodes = [] - - for node in or_.nodes: - expr = self.dispatch(node) - - # Return from the subprogram where the test is satisfied. - - if node is not last: - nodes.append(StoreTemp(expr=expr)) - invocation = InvokeFunction(or_, expr=LoadAttr(expr=LoadTemp(), name="__bool__")) - test = Conditional(test=invocation, body=[ReturnFromBlock(expr=LoadTemp())]) - nodes.append(test) - - # Put subsequent operations in the else section of this conditional. - - nodes = test.else_ = [ReleaseTemp()] - - # For the last operation, return the result. - - else: - nodes.append( - ReturnFromBlock(expr=expr) - ) - - # Finish the subprogram definition. - - subprogram.code = results - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make an invocation of the subprogram. - - result = InvokeRef(or_, 1, - produces_result=1, - ref=subprogram - ) - return result - - def visitPass(self, pass_): - return Pass(pass_, 1) - - def visitPower(self, power): - return self._visitBinary(power, "__pow__", "__rpow__") - - def visitPrint(self, print_): - - """ - Convert... - - Print (dest) -> - (nodes) - - ...to: - - StoreTemp (index) -> "print" - (expr) -> LoadAttr (expr) -> (dest) - (name) -> "write" - InvokeFunction (expr) -> LoadTemp (index) -> "print" - (args) -> [(node)] - ReleaseTemp (index) -> "print" - """ - - if print_.dest is not None: - dest = self.dispatch(print_.dest) - else: - dest = self.dispatch(compiler.ast.Name("stdout")) - - result = Assign(print_, 1, - code=[ - StoreTemp( - index="print", - expr=LoadAttr( - expr=dest, - name="write" - ) - ) - ] - ) - - for node in print_.nodes: - result.code.append( - InvokeFunction( - print_, - expr=LoadTemp(index="print"), - args=[self.dispatch(node)], - star=None, - dstar=None - ) - ) - - result.code.append( - ReleaseTemp(index="print") - ) - - return result - - def visitPrintnl(self, printnl): - result = self.visitPrint(printnl) - result.code.insert( - len(result.code) - 1, - InvokeFunction( - printnl, - expr=LoadTemp(index="print"), - args=[self.dispatch(compiler.ast.Const("\n"))], - star=None, - dstar=None - ) - ) - return result - - def visitRaise(self, raise_): - result = Raise(raise_, 1) - if raise_.expr2 is None: - result.expr = self.dispatch(raise_.expr1) - else: - result.expr = InvokeFunction( - raise_, - expr=self.dispatch(raise_.expr1), - args=[self.dispatch(raise_.expr2)], - star=None, - dstar=None - ) - if raise_.expr3 is not None: - result.traceback = self.dispatch(raise_.expr3) - else: - result.traceback = None - return result - - def visitReturn(self, return_): - result = ReturnFromFunction(return_, 1, - expr=self.dispatch(return_.value) - ) - return result - - def _visitSlice(self, slice, expr, lower, upper, flags, value=None): - if flags == "OP_ASSIGN": - result = InvokeFunction(slice, 1, - expr=LoadAttr( - expr=expr, - name="__setslice__" - ), - star=None, - dstar=None, - args=[lower, upper, value] - ) - elif flags == "OP_APPLY": - args = [] - result = InvokeFunction(slice, 1, - expr=LoadAttr( - expr=expr, - name="__getslice__" - ), - star=None, - dstar=None, - args=[lower, upper] - ) - elif flags == "OP_DELETE": - args = [] - result = InvokeFunction(slice, 1, - expr=LoadAttr( - expr=expr, - name="__delslice__" - ), - star=None, - dstar=None, - args=[lower, upper] - ) - else: - raise NotImplementedError, flags - - return result - - def visitSlice(self, slice, in_sequence=0): - return self._visitSlice(slice, self.dispatch(slice.expr), self.dispatch_or_none(slice.lower), - self.dispatch_or_none(slice.upper), slice.flags, self._visitAssNameOrAttr(slice, in_sequence)) - - def visitSliceobj(self, sliceobj): - return InvokeFunction(sliceobj, 1, - expr=LoadName(name="slice"), - args=self.dispatches(sliceobj.nodes), - star=None, - dstar=None - ) - - def visitStmt(self, stmt): - return self.dispatches(stmt.nodes) - - def visitSub(self, sub): - return self._visitBinary(sub, "__sub__", "__rsub__") - - def _visitSubscript(self, subscript, expr, subs, flags, value=None): - if flags == "OP_ASSIGN": - result = InvokeFunction(subscript, 1, - expr=LoadAttr( - expr=expr, - name="__setitem__" - ), - star=None, - dstar=None, - args=[subs, value] - ) - elif flags == "OP_APPLY": - args = [] - result = InvokeFunction(subscript, 1, - expr=LoadAttr( - expr=expr, - name="__getitem__" - ), - star=None, - dstar=None, - args=[subs] - ) - elif flags == "OP_DELETE": - args = [] - result = InvokeFunction(subscript, 1, - expr=LoadAttr( - expr=expr, - name="__delitem__" - ), - star=None, - dstar=None, - args=[subs] - ) - else: - raise NotImplementedError, flags - - return result - - def _visitSubscriptSubs(self, node, subs): - if len(subs) == 1: - return self.dispatch(subs[0]) - else: - return InvokeFunction(node, 1, - expr=LoadName(name="tuple"), - args=self.dispatches(subs), - star=None, - dstar=None - ) - - def visitSubscript(self, subscript, in_sequence=0): - return self._visitSubscript( - subscript, self.dispatch(subscript.expr), self._visitSubscriptSubs(subscript, subscript.subs), subscript.flags, - self._visitAssNameOrAttr(subscript, in_sequence) - ) - - def visitTryExcept(self, tryexcept): - - """ - Make conditionals for each handler associated with a 'tryexcept' node. - - Convert... - - TryExcept (body) - (else) - (spec/assign/stmt) - ... - - ...to: - - Try (body) - (else) - (handler) -> Conditional (test) -> (stmt) - (body) -> ResetExc ... - (else) -> Conditional (test) -> (stmt) - (body) -> ResetExc ... - (else) -> ... - """ - - result = Try(tryexcept, 1, body=[], else_=[], finally_=[]) - - if tryexcept.body is not None: - result.body = self.dispatch(tryexcept.body) - if tryexcept.else_ is not None: - result.else_ = self.dispatch(tryexcept.else_) - - results = nodes = [] - catch_all = 0 - - for spec, assign, stmt in tryexcept.handlers: - - # If no specification exists, produce an unconditional block. - - if spec is None: - nodes += self.dispatch(stmt) - catch_all = 1 - - # Produce an exception value check. - - else: - test = Conditional( - isolate_test=1, - test=CheckType(expr=LoadExc(), choices=self._visitTryExcept(spec)) - ) - test.body = [] - - if assign is not None: - test.body.append( - Assign( - code=[ - StoreTemp(expr=LoadExc()), - self.dispatch(assign), - ReleaseTemp() - ] - ) - ) - - test.body += [ResetExc()] + self.dispatch(stmt) - nodes.append(test) - nodes = test.else_ = [] - - # Add a raise operation to deal with unhandled exceptions. - - if not catch_all: - nodes.append( - Raise( - expr=LoadExc()) - ) - - result.handler = results - return result - - def _visitTryExcept(self, spec): - - "Return a list of nodes for the given exception type 'spec'." - - if isinstance(spec, compiler.ast.Tuple): - nodes = [] - for node in spec.nodes: - nodes += self._visitTryExcept(node) - else: - nodes = [self.dispatch(spec)] - return nodes - - def visitTryFinally(self, tryfinally): - result = Try(tryfinally, 1, body=[], else_=[], finally_=[]) - if tryfinally.body is not None: - result.body = self.dispatch(tryfinally.body) - if tryfinally.final is not None: - result.finally_ = self.dispatch(tryfinally.final) - return result - - def visitTuple(self, tuple): - - "Make a MakeTuple node containing the original 'tuple' contents." - - result = MakeTuple(tuple, 1, - nodes=self.dispatches(tuple.nodes) - ) - return result - - def visitUnaryAdd(self, unaryadd): - return self._visitUnary(unaryadd, "__pos__") - - def visitUnarySub(self, unarysub): - return self._visitUnary(unarysub, "__neg__") - - def visitWhile(self, while_): - - """ - Make a subprogram for the 'while' node and record its contents inside the - subprogram. Convert... - - While (test) -> (body) - (else) - - ...to: - - Subprogram -> Conditional (test) -> (body) -> Invoke subprogram - (else) -> Conditional (test) -> ReturnFromBlock ... - (else) -> ... - """ - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - - # Include a conditional statement in the subprogram. - # Inside the conditional, add a recursive invocation to the subprogram - # if the test condition was satisfied. - # Return within the main section of the loop. - - test = Conditional( - test=InvokeFunction( - while_, - expr=LoadAttr( - expr=self.dispatch(while_.test), - name="__bool__"), - ), - body=self.dispatch(while_.body) + [ - InvokeRef( - while_, - ref=subprogram - ), - ReturnFromBlock() - ], - else_=[] - ) - - # Provide the else section, if present, along with an explicit return. - - if while_.else_ is not None: - test.else_ = self.dispatch(while_.else_) + [ReturnFromBlock()] - - # Finish the subprogram definition. - - subprogram.code = [test] - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - # Make an invocation of the subprogram. - - result = InvokeRef(while_, 1, ref=subprogram) - - # Make nice annotations for the viewer. - - while_._test_call = subprogram.code[0].test - - return result - - # NOTE: Not actually supported. - # NOTE: Virtually the same as visitReturn... - - def visitYield(self, yield_): - result = Yield(yield_, 1, - expr=self.dispatch(yield_.value) - ) - return result - - # Convenience methods. - - def _visitBinary(self, binary, left_name, right_name): - return self._visitBinaryOp(binary, self.dispatch(binary.left), self.dispatch(binary.right), left_name, right_name) - - def _visitBinaryCompareOp(self, binary, left, right, left_name, right_name): - - """ - Emulate the current mechanisms by producing nodes as follows: - - InvokeRef -> Subprogram -> StoreTemp (expr) -> x.__lt__(y) - Conditional (test) -> __is__(LoadTemp, NotImplemented) - (body) -> ReleaseTemp - StoreTemp (expr) -> y.__gt__(x) - Conditional (test) -> __is__(LoadTemp, NotImplemented) - (body) -> ReturnFromBlock (expr) -> False - (else) -> ReturnFromBlock (expr) -> LoadTemp - (else) -> ReturnFromBlock (expr) -> LoadTemp - """ - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - - subprogram.code = [ - StoreTemp( - expr=InvokeFunction( - binary, - expr=LoadAttr(expr=left, name=left_name), - args=[right], - star=None, - dstar=None) - ), - Conditional( - isolate_test=1, - test=CheckType( - expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] - ), - body=[ - ReleaseTemp(), - StoreTemp( - expr=InvokeFunction( - binary, - expr=LoadAttr(expr=right, name=right_name), - args=[left], - star=None, - dstar=None) - ), - Conditional( - isolate_test=1, - test=CheckType( - expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] - ), - body=[ - ReturnFromBlock( - expr=LoadName(name="False") - ) - ], - else_=[ - CheckType( - inverted=1, expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] - ), - ReturnFromBlock( - expr=LoadTemp() - ) - ] - ) - ], - else_=[ - CheckType( - inverted=1, expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] - ), - ReturnFromBlock( - expr=LoadTemp() - ) - ] - ) - ] - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - result = InvokeRef( - binary, - produces_result=1, - ref=subprogram - ) - - # Make nice annotations for the viewer. - - binary._left_call = subprogram.code[0].expr - binary._right_call = subprogram.code[1].body[1].expr - - return result - - def _visitBinaryOp(self, binary, left, right, left_name, right_name): - - """ - Emulate the current mechanisms by producing nodes as follows: - - InvokeRef -> Subprogram -> Try (body) -> ReturnFromBlock (expr) -> x.__add__(y) - (else) - (handler) -> Conditional (test) -> CheckType (expr) -> LoadExc - (choices) -> LoadName TypeError - (body) -> ReturnFromBlock (expr) -> y.__radd__(x) - (else) - """ - - subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) - self.current_subprograms.append(subprogram) - - subprogram.code = [ - Try(binary, 1, - body=[ - ReturnFromBlock( - expr=InvokeFunction( - binary, - expr=LoadAttr(expr=left, name=left_name), - args=[right], - star=None, - dstar=None) - ) - ], - else_=[], - finally_=[], - handler=[ - Conditional( - test=CheckType(expr=LoadExc(), choices=[LoadName(name="TypeError")]), - body=[ - ReturnFromBlock( - expr=InvokeFunction( - binary, - expr=LoadAttr(expr=right, name=right_name), - args=[left], - star=None, - dstar=None) - ) - ], - else_=[] - ) - ] - ) - ] - - self.current_subprograms.pop() - self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram - - result = InvokeRef( - binary, - produces_result=1, - ref=subprogram - ) - - # Make nice annotations for the viewer. - - binary._left_call = subprogram.code[0].body[0].expr - binary._right_call = subprogram.code[0].handler[0].body[0].expr - - return result - - def _visitBuiltin(self, builtin, name): - result = InvokeFunction(builtin, 1, expr=LoadName(name=name), args=self.dispatches(builtin.nodes)) - return result - - def _visitUnary(self, unary, name): - result = InvokeFunction(unary, 1, - expr=LoadAttr( - expr=self.dispatch(unary.expr), - name=name - ) - ) - - # Make nice annotations for the viewer. - - unary._unary_call = result - - return result - -# Convenience functions. - -def simplify(filename, builtins=0, module_name=None): - - """ - Simplify the module stored in the file with the given 'filename'. - - If the optional 'builtins' parameter is set to a true value (the default - being a false value), then the module is considered as the builtins module. - """ - - simplifier = Simplifier(builtins) - module = compiler.parseFile(filename) - compiler.misc.set_filename(filename, module) - if builtins: - name = module_name or "__builtins__" - else: - path, ext = os.path.splitext(filename) - path, name = os.path.split(path) - name = module_name or name - simplified = simplifier.process(module, name) - return simplified - -# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/__init__.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,1941 @@ +#!/usr/bin/env python + +""" +Simplify AST structures for easier type propagation and analysis. The code in +this module processes AST trees originating from the compiler module and +produces a result tree consisting of instruction-oriented program nodes. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +-------- + +To use this module, the easiest approach is to use the simplify function: + +simplify(filename) + +The more complicated approach involves first instantiating a Simplifier object: + +simplifier = Simplifier() + +Then, applying the simplifier to an AST tree: + +module = compiler.parseFile(filename) +simplifier.process(module) +""" + +from simplify.simplified import * +import compiler.ast +import os + +class Simplifier(Visitor): + + """ + A simplifying visitor for AST nodes. + + Covered: Add, And, Assert, AssAttr, AssList, AssName, AssTuple, Assign, + AugAssign, Bitand, Break, CallFunc, Class, Compare, Const, + Continue, Dict, Discard, Div, FloorDiv, For, From, Function, + Getattr, Global, If, Import, Invert, Keyword, Lambda, List, + ListComp, ListCompFor, ListCompIf, Mod, Module, Mul, Name, Not, Or, + Pass, Power, Print, Printnl, Raise, Return, Slice, Sliceobj, Stmt, + Sub, Subscript, TryExcept, TryFinally, Tuple, While, UnaryAdd, + UnarySub. + + Missing: Backquote, Bitor, Bitxor, Decorators, Ellipsis, Exec, LeftShift, + RightShift, Yield. + """ + + def __init__(self, builtins=0): + + """ + Initialise the simplifier with the optional 'builtins' parameter + indicating whether the module contains the built-in classes and + functions. + """ + + Visitor.__init__(self) + self.subprograms = [] # Subprograms outside the tree. + self.structures = [] # Structures/classes. + self.constants = {} # Constants. + self.current_subprograms = [] # Current subprograms being processed. + self.current_structures = [] # Current structures being processed. + self.within_class = 0 # Whether a class is being defined. + self.builtins = builtins # Whether the builtins are being processed. + + # Convenience attributes. + + self.subnames = {} + + # For compiler package mechanisms. + + self.visitor = self + + def process(self, node, name): + result = self.dispatch(node, name) + result.simplifier = self + return result + + def dispatch_or_none(self, node, *args): + if node is not None: + return self.dispatch(node, *args) + else: + return LoadName(node, name="None") + + # Top-level transformation. + + def visitModule(self, module, name=None): + + """ + Process the given 'module', producing a Module object which contains the + resulting program nodes. If the optional 'name' is provided, the 'name' + attribute is set on the Module object using a value other than None. + """ + + result = self.module = Module(module, 1, name=name) + result.code = self.dispatch(module.node) + return result + + # Node transformations. + + def visitAdd(self, add): + return self._visitBinary(add, "__add__", "__radd__") + + def visitAnd(self, and_): + + """ + Make a subprogram for the 'and_' node and record its contents inside the + subprogram. Convert... + + And (test) + (test) + ... + + ...to: + + Subprogram -> Conditional (test) -> ReturnFromBlock ... + (else) -> Conditional (test) -> ReturnFromBlock ... + (else) -> ... + """ + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + + # In the subprogram, make instructions which store each operand, test + # for each operand's truth status, and if appropriate return from the + # subprogram with the value of the operand. + + last = and_.nodes[-1] + results = nodes = [] + + for node in and_.nodes: + expr = self.dispatch(node) + + # Return from the subprogram where the test is not satisfied. + + if node is not last: + nodes += [ + StoreTemp(expr=expr), + Conditional( + test=self._visitNot(LoadTemp()), + body=[ + ReturnFromBlock( + expr=LoadTemp() + ) + ], + else_=[ + ReleaseTemp() + # Subsequent operations go here! + ] + ) + ] + + # Put subsequent operations in the else section of this conditional. + + nodes = nodes[-1].else_ + + # For the last operation, return the result. + + else: + nodes.append(ReturnFromBlock(expr=expr)) + + # Finish the subprogram definition. + + subprogram.code = results + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make an invocation of the subprogram. + + result = InvokeRef(and_, 1, produces_result=1, ref=subprogram) + return result + + def visitAssert(self, assert_): + if assert_.fail: + fail_args = [self.dispatch(assert_.fail)] + else: + fail_args = [] + + result = Conditional(assert_, 1, + test=self.dispatch(assert_.test), + body=[], + else_=[ + Raise(assert_, + expr=InvokeFunction(assert_, + expr=LoadName(name="AssertionError"), + args=fail_args, + star=None, + dstar=None + ) + ) + ] + ) + + # Make nice annotations for the viewer. + + assert_._raises = result.else_[0].expr + return result + + # Assignments. + + def visitAssAttr(self, assattr, in_sequence=0): + expr = self._visitAssNameOrAttr(assattr, in_sequence) + lvalue = self.dispatch(assattr.expr) + result = StoreAttr(assattr, 1, name=assattr.attrname, lvalue=lvalue, expr=expr) + return result + + def visitAssign(self, assign): + result = Assign(assign, 1) + store = StoreTemp(expr=self.dispatch(assign.expr)) + release = ReleaseTemp() + result.code = [store] + self.dispatches(assign.nodes, 0) + [release] + return result + + def visitAssList(self, asslist, in_sequence=0): + if not in_sequence: + expr = LoadTemp() + else: + expr = InvokeFunction(asslist, expr=LoadAttr(expr=LoadTemp(), name="next")) + result = Assign(asslist, 1) + store = StoreTemp(expr=InvokeFunction(asslist, expr=LoadAttr(name="__iter__", expr=expr))) + release = ReleaseTemp() + result.code = [store] + self.dispatches(asslist.nodes, 1) + [release] + return result + + visitAssTuple = visitAssList + + def _visitAssNameOrAttr(self, node, in_sequence): + if not in_sequence: + return LoadTemp() + else: + return InvokeFunction(node, expr=LoadAttr(expr=LoadTemp(), name="next")) + + def visitAssName(self, assname, in_sequence=0): + expr = self._visitAssNameOrAttr(assname, in_sequence) + result = StoreName(assname, 1, name=assname.name, expr=expr) + return result + + augassign_methods = { + "+=" : "__iadd__", "-=" : "__isub__", "*=" : "__imul__", "/=" : "__idiv__", + "%=" : "__imod__", "**=" : "__ipow__", "<<=" : "__ilshift__", ">>=" : "__irshift__", + "&=" : "__iand__", "^=" : "__ixor__", "|=" : "__ior__" + } + + def visitAugAssign(self, augassign): + + """ + Convert the augmented assignment... + + AugAssign (node) -> Name | Getattr | Slice | Subscript + (op) + (expr) + + ...to: + + Assign (code) -> StoreTemp (expr) -> InvokeFunction (expr) -> LoadAttr (expr) -> + (name) -> + StoreName (name) -> + (expr) -> LoadTemp + ReleaseTemp + """ + + result = Assign(augassign, 1) + expr = self.dispatch(augassign.expr) + + # Simple augmented assignment: name += expr + + if isinstance(augassign.node, compiler.ast.Name): + result.code = [ + StoreTemp( + expr=InvokeFunction( # referenced below + augassign, + args=[expr], + star=None, + dstar=None, + expr=LoadAttr( + expr=self.dispatch(augassign.node), + name=self.augassign_methods[augassign.op] + ) + ) + ), + StoreName( + expr=LoadTemp(), + name=augassign.node.name), + ReleaseTemp() + ] + + # Make nice annotations for the viewer. + + augassign._op_call = result.code[0].expr + + # Complicated augmented assignment: lvalue.attr += expr + + elif isinstance(augassign.node, compiler.ast.Getattr): + + # -> setattr(, getattr(, "attr").__xxx__(expr)) + + result.code = [ + StoreTemp( + index="expr", + expr=self.dispatch(augassign.node.expr) + ), + StoreTemp( + expr=InvokeFunction( # referenced below + augassign, + args=[expr], star=None, dstar=None, + expr=LoadAttr( + expr=LoadAttr(augassign.node, 1, + expr=LoadTemp(index="expr"), + name=augassign.node.attrname + ), + name=self.augassign_methods[augassign.op] + ) + ) + ), + StoreAttr( + expr=LoadTemp(), + lvalue=LoadTemp(index="expr"), + name=augassign.node.attrname + ), + ReleaseTemp(index="expr"), + ReleaseTemp() + ] + + # Make nice annotations for the viewer. + + augassign._op_call = result.code[1].expr + + # Complicated augassign using slices: lvalue[lower:upper] += expr + + elif isinstance(augassign.node, compiler.ast.Slice): + + # , , -> .__setslice__(, , .__getslice__(, ).__xxx__(expr)) + + result.code = [ + StoreTemp( + index="expr", + expr=self.dispatch(augassign.node.expr) + ), + StoreTemp( + index="lower", + expr=self.dispatch_or_none(augassign.node.lower) + ), + StoreTemp( + index="upper", + expr=self.dispatch_or_none(augassign.node.upper) + ), + StoreTemp( + expr=InvokeFunction( # referenced below + augassign, + args=[expr], star=None, dstar=None, + expr=LoadAttr( + expr=self._visitSlice( + augassign.node, + LoadTemp(index="expr"), + LoadTemp(index="lower"), + LoadTemp(index="upper"), + "OP_APPLY"), + name=self.augassign_methods[augassign.op] + ) + ) + ), + self._visitSlice( + augassign.node, + LoadTemp(index="expr"), + LoadTemp(index="lower"), + LoadTemp(index="upper"), + "OP_ASSIGN", + LoadTemp() + ), + ReleaseTemp(index="expr"), + ReleaseTemp(index="lower"), + ReleaseTemp(index="upper"), + ReleaseTemp() + ] + + # Make nice annotations for the viewer. + + augassign._op_call = result.code[3].expr + + # Complicated augassign using subscripts: lvalue[subs] += expr + + elif isinstance(augassign.node, compiler.ast.Subscript): + + # , -> .__setitem__(, .__getitem__().__xxx__(expr)) + + result.code = [ + StoreTemp(index="expr", expr=self.dispatch(augassign.node.expr)), + StoreTemp(index="subs", expr=self._visitSubscriptSubs(augassign.node, augassign.node.subs)), + StoreTemp( + expr=InvokeFunction( # referenced below + augassign, + args=[expr], star=None, dstar=None, + expr=LoadAttr( + expr=self._visitSubscript( + augassign.node, + LoadTemp(index="expr"), + LoadTemp(index="subs"), + "OP_APPLY" + ), + name=self.augassign_methods[augassign.op] + ) + ) + ), + self._visitSubscript( + augassign.node, + LoadTemp(index="expr"), + LoadTemp(index="subs"), + "OP_ASSIGN", + LoadTemp() + ), + ReleaseTemp(index="expr"), + ReleaseTemp(index="subs"), + ReleaseTemp() + ] + + # Make nice annotations for the viewer. + + augassign._op_call = result.code[2].expr + + else: + raise NotImplementedError, augassign.node.__class__ + + return result + + def visitBitand(self, bitand): + + """ + Make a subprogram for the 'bitand' node and record its contents inside the + subprogram. Convert... + + Bitand (node) + (node) + ... + + ...to: + + Subprogram -> Conditional (test) -> ReturnFromBlock ... + (else) -> Conditional (test) -> ReturnFromBlock ... + (else) -> ... + """ + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + + # In the subprogram, make instructions which store each operand, test + # for each operand's truth status, and if appropriate return from the + # subprogram with the value of the operand. + + last = bitand.nodes[-1] + results = nodes = [] + + # Start by storing the first operand. + + nodes += [ + StoreTemp(expr=self.dispatch(bitand.nodes[0])) + ] + + # For viewing purposes, record invocations on the AST node. + + bitand._ops = [] + + for node in bitand.nodes[1:]: + + # Make a new AST-style node to wrap the operation program nodes. + + new_op = Op("&", node) + bitand._ops.append(new_op) + + # Generate the operation involving the previous result and the + # current operand. + + expr = self._visitBinaryOp(new_op, LoadTemp(), self.dispatch(node), "__and__", "__rand__") + + # Return from the subprogram where the test is not satisfied. + + if node is not last: + nodes += [ + StoreTemp(expr=expr), + Conditional( + test=self._visitNot(LoadTemp()), + body=[ + ReturnFromBlock( + expr=LoadTemp() + ) + ], + else_=[ + # Subsequent operations go here! + ] + ) + ] + + # Put subsequent operations in the else section of this conditional. + + nodes = nodes[-1].else_ + + # For the last operation, return the result. + + else: + nodes.append(ReturnFromBlock(expr=expr)) + + # Finish the subprogram definition. + + subprogram.code = results + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make an invocation of the subprogram. + + result = InvokeRef(bitand, 1, produces_result=1, ref=subprogram) + return result + + def visitBreak(self, break_): + result = ReturnFromBlock(break_, 1) + return result + + def visitCallFunc(self, callfunc): + result = InvokeFunction(callfunc, 1, star=None, dstar=None, args=self.dispatches(callfunc.args)) + if callfunc.star_args is not None: + result.star = self.dispatch(callfunc.star_args) + if callfunc.dstar_args is not None: + result.dstar = self.dispatch(callfunc.dstar_args) + result.expr = self.dispatch(callfunc.node) + return result + + def visitClass(self, class_): + + # Add "object" if the class is not "object" and has an empty bases list. + + if class_.name != "object" and not class_.bases: + bases = [compiler.ast.Name("object")] + else: + bases = class_.bases + + structure = Class(name=class_.name, module=self.module, bases=self.dispatches(bases)) + self.structures.append(structure) + within_class = self.within_class + self.within_class = 1 + + # Make a subprogram which initialises the class structure. + + subprogram = Subprogram(name=None, module=self.module, structure=structure, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + self.current_structures.append(structure) # mostly for name construction + + # The class is initialised using the code found inside. + + subprogram.code = self.dispatch(class_.code) + [ReturnFromBlock()] + + self.within_class = within_class + self.current_structures.pop() + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make a definition of the class associating it with a name. + + result = Assign( + code=[ + StoreName(class_, 1, # defines the class + name=class_.name, + expr=LoadRef(ref=structure) + ), + InvokeRef( + class_, + share_locals=0, # override the local sharing usually in InvokeRef + ref=subprogram + ) + ] + ) + return result + + comparison_methods = { + "==" : ("__eq__", "__ne__"), + "!=" : ("__ne__", "__eq__"), + "<" : ("__lt__", "__gt__"), + "<=" : ("__le__", "__ge__"), + ">=" : ("__ge__", "__le__"), + ">" : ("__gt__", "__lt__"), + "is" : None, + "is not" : None, + "in" : None, + "not in" : None + } + + def visitCompare(self, compare): + + """ + Make a subprogram for the 'compare' node and record its contents inside + the subprogram. Convert... + + Compare (expr) + (name/node) + ... + + ...to: + + InvokeRef -> Subprogram -> Conditional (test) -> (body) + (else) -> Conditional (test) -> (body) + (else) -> ... + """ + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + + # In the subprogram, make instructions which invoke a method on the + # first operand of each operand pair and, if appropriate, return with + # the value from that method. + + last = compare.ops[-1] + previous = self.dispatch(compare.expr) + results = nodes = [] + + # For viewing purposes, record invocations on the AST node. + + compare._ops = [] + + for op in compare.ops: + op_name, node = op + + # Make a new AST-style node to wrap the operation program nodes. + + new_op = Op(op_name, node) + compare._ops.append(new_op) + + expr = self.dispatch(node) + + # Identify the operation and produce the appropriate method call. + + method_names = self.comparison_methods[op_name] + if method_names: + first_name, alternative_name = method_names + invocation = self._visitBinaryCompareOp(new_op, previous, expr, first_name, alternative_name) + + elif op_name == "is": + invocation = InvokeFunction( + new_op, 1, + expr=LoadName(name="__is__"), + args=[previous, expr], + star=None, + dstar=None) + + elif op_name == "is not": + invocation = Not( + new_op, 1, + expr=InvokeFunction( + new_op, + expr=LoadName(name="__is__"), + args=[previous, expr], + star=None, + dstar=None) + ) + + elif op_name == "in": + invocation = InvokeFunction( + new_op, 1, + expr=LoadAttr( + expr=previous, + name="__contains__" + ), + args=[expr], + star=None, + dstar=None) + + elif op_name == "not in": + invocation = Not( + new_op, 1, + expr=InvokeFunction( + new_op, + expr=LoadAttr( + expr=previous, + name="__contains__" + ), + args=[expr], + star=None, + dstar=None) + ) + + else: + raise NotImplementedError, op_name + + nodes.append(StoreTemp(expr=invocation)) + + # Return from the subprogram where the test is not satisfied. + + if op is not last: + nodes.append( + Conditional( + test=self._visitNot(LoadTemp()), + body=[ + ReturnFromBlock(expr=LoadTemp()) + ], + else_=[ + ReleaseTemp() + # Subsequent operations go here! + ] + ) + ) + + # Put subsequent operations in the else section of this conditional. + + nodes = nodes[-1].else_ + + # For the last operation, return the result. + + else: + nodes.append( + ReturnFromBlock(expr=LoadTemp(release=1)) + ) + + previous = expr + + # Finish the subprogram definition. + + subprogram.code = results + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make an invocation of the subprogram. + + result = InvokeRef(compare, 1, produces_result=1, ref=subprogram) + return result + + def visitConst(self, const): + key = "%s-%s" % (const.value.__class__.__name__, const.value) + if not self.constants.has_key(key): + self.constants[key] = Constant(name=repr(const.value), value=const.value) + result = InvokeFunction(const, 1, expr=LoadName(name=self.constants[key].typename)) + return result + + def visitContinue(self, continue_): + result = InvokeRef(continue_, 1, ref=self.current_subprograms[-1]) + return result + + def visitDict(self, dict): + result = InvokeFunction(dict, 1, expr=LoadName(name="dict")) + args = [] + for key, value in dict.items: + tuple = InvokeFunction(dict, expr=LoadName(name="tuple")) + tuple.set_args([self.dispatch(key), self.dispatch(value)]) + args.append(tuple) + result.set_args(args) + return result + + def visitDiscard(self, discard): + return self.dispatch(discard.expr) + + def visitDiv(self, div): + return self._visitBinary(div, "__div__", "__rdiv__") + + def visitFloorDiv(self, floordiv): + return self._visitBinary(floordiv, "__floordiv__", "__rfloordiv__") + + def visitFor(self, for_): + + """ + Make a subprogram for the 'for_' node and record its contents inside the + subprogram. Convert... + + For (assign) + (body) + (else) + + ...to: + + Assign (assign #1) + Invoke -> Subprogram -> Try (body) -> (assign #2) + (body) + Invoke subprogram + (handler) -> ... + (else) -> ... + """ + + return self._visitFor(for_, self.dispatches(for_.body), for_.else_) + + def _visitFor(self, node, body_stmt, else_=None): + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[]) + self.current_subprograms.append(subprogram) + + # Always return from conditional sections/subprograms. + + if else_ is not None: + else_stmt = self.dispatch(else_) + [ReturnFromBlock()] + else: + else_stmt = [ReturnFromBlock()] + + # Wrap the assignment in a try...except statement. + # Inside the body, add a recursive invocation to the subprogram. + + subprogram.code = [ + Try( + body=[ + Assign( + code=[ + StoreTemp( + expr=InvokeFunction(node, + expr=LoadAttr( + expr=LoadTemp(), + name="next" + ) + ) + ), + self.dispatch(node.assign), + ReleaseTemp() + ]) + ] + body_stmt + [ + InvokeRef( + node, + ref=subprogram + ) + ], + handler=[ + Conditional( + test=InvokeFunction( + node, + expr=LoadName(name="isinstance"), + args=[LoadExc(), LoadName(name="StopIteration")], + star=None, + dstar=None), + body=else_stmt, + else_=[ + Raise( + expr=LoadExc() + ) + ] + ) + ], + else_=[], + finally_=[] + ), + ReturnFromBlock() + ] + + # Finish the subprogram definition. + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Obtain an iterator for the sequence involved. + # Then, make an invocation of the subprogram. + + result = Assign(node, 1, + code=[ + StoreTemp( + expr=InvokeFunction( + node, + expr=LoadAttr( + name="__iter__", + expr=self.dispatch(node.list) + ) + ) + ), + InvokeRef(node, ref=subprogram), + ReleaseTemp() + ] + ) + + # Make nice annotations for the viewer. + + node._iter_call = result.code[0].expr + node._next_call = subprogram.code[0].body[0].code[0].expr + + return result + + def visitFrom(self, from_): + result = Assign(from_, 1) + code = [] + _names = [] + code.append( + StoreTemp( + expr=Import(name=from_.modname, alias=1) + ) + ) + from_._modname = code[-1].expr + for name, alias in from_.names: + code.append( + StoreName( + expr=LoadAttr( + expr=LoadTemp(), + name=name), + name=(alias or name) + ) + ) + _names.append(code[-1].expr) + code.append(ReleaseTemp()) + result.code = code + from_._names = _names + return result + + def _visitFunction(self, function, subprogram): + + """ + A common function generator which transforms the given 'function' node + and initialises the given 'subprogram' appropriately. + """ + + # Discover star and dstar parameters. + + if function.flags & 4 != 0: + has_star = 1 + else: + has_star = 0 + if function.flags & 8 != 0: + has_dstar = 1 + else: + has_dstar = 0 + + # Discover the number of defaults and positional parameters. + + ndefaults = len(function.defaults) + npositional = len(function.argnames) - has_star - has_dstar + + # Produce star and dstar parameters with appropriate defaults. + + if has_star: + star = ( + function.argnames[npositional], + self.dispatch(compiler.ast.List([])) + ) + else: + star = None + if has_dstar: + dstar = ( + function.argnames[npositional + has_star], + self.dispatch(compiler.ast.Dict([])) + ) + else: + dstar = None + + params = [] + for i in range(0, npositional - ndefaults): + params.append((function.argnames[i], None)) + + # Process defaults. + + for i in range(0, ndefaults): + default = function.defaults[i] + if default is not None: + params.append((function.argnames[npositional - ndefaults + i], self.dispatch(default))) + else: + params.append((function.argnames[npositional - ndefaults + i], None)) + + subprogram.params = params + subprogram.star = star + subprogram.dstar = dstar + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + def visitFunction(self, function): + + """ + Make a subprogram for the 'function' and record it outside the main + tree. Produce something like the following: + + StoreName (name) + (expr) -> LoadRef (ref) -> Subprogram (params) + (star) + (dstar) + """ + + subprogram = Subprogram(function, name=function.name, module=self.module, structures=self.current_structures[:], + internal=0, returns_value=1, star=None, dstar=None, is_method=self.within_class, original_def=function) + + # Make nice annotations for the viewer. + + function._subprogram = subprogram + + self.current_subprograms.append(subprogram) + within_class = self.within_class + self.within_class = 0 + + subprogram.code = self.dispatch(function.code) + [ReturnFromFunction()] + + self.within_class = within_class + self.current_subprograms.pop() + self._visitFunction(function, subprogram) + + # Make a definition of the function associating it with a name. + + result = StoreName(function, 1, name=function.name, expr=LoadRef(ref=subprogram)) + return result + + def visitGetattr(self, getattr): + result = LoadAttr(getattr, 1, + name=getattr.attrname, + expr=self.dispatch(getattr.expr) + ) + return result + + def visitGlobal(self, global_): + result = Global(global_, 1, + names=global_.names + ) + return result + + def visitIf(self, if_): + + """ + Make conditionals for each test from an 'if_' AST node, adding the body + and putting each subsequent test as part of the conditional's else + section. + + Convert... + + If (test/body) + (test/body) + ... + (else/body) + + ...to: + + Conditional (test) -> (body) + (else) -> Conditional (test) -> (body) + (else) -> ... + """ + + + results = nodes = [] + + # Produce something like... + # expr.__bool__() ? body + + first = 1 + for compare, stmt in if_.tests: + + # Set the first as the defining node. + + test = Conditional(if_, first, + test=InvokeFunction( + if_, + expr=LoadAttr( + expr=self.dispatch(compare), + name="__bool__" + ), + ) + ) + test.body = self.dispatch(stmt) + nodes.append(test) + nodes = test.else_ = [] + first = 0 + + # Add the compound statement from any else clause to the end. + + if if_.else_ is not None: + nodes += self.dispatch(if_.else_) + + result = results[0] + return result + + def visitImport(self, import_): + result = Assign(import_, 1) + code = [] + _names = [] + for path, alias in import_.names: + importer = Import(name=path, alias=alias) + top = alias or path.split(".")[0] + code.append(StoreName(expr=importer, name=top)) + _names.append(code[-1].expr) + result.code = code + import_._names = _names + return result + + def visitInvert(self, invert): + return self._visitUnary(invert, "__invert__") + + def visitKeyword(self, keyword): + result = Keyword(keyword, 1, + name=keyword.name, + expr=self.dispatch(keyword.expr) + ) + return result + + def visitLambda(self, lambda_): + + # Make a subprogram for the function and record it outside the main + # tree. + + subprogram = Subprogram(lambda_, name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=lambda_) + + # Make nice annotations for the viewer. + + lambda_._subprogram = subprogram + + # Process the lambda contents. + + self.current_subprograms.append(subprogram) + subprogram.code = [ReturnFromFunction(expr=self.dispatch(lambda_.code))] + self.current_subprograms.pop() + self._visitFunction(lambda_, subprogram) + + # Get the subprogram reference to the lambda. + + return LoadRef(lambda_, 1, ref=subprogram) + + def visitList(self, list): + + # Make a subprogram for the list construction and record it outside the + # main tree. + + subprogram = Subprogram(list, name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=list) + self.current_subprograms.append(subprogram) + + # Make nice annotations for the viewer. + + list._subprogram = subprogram + + subprogram.code=[ + StoreTemp( + expr=InvokeFunction( + list, + expr=LoadName( + name="list" + ), + args=[], + star=None, + dstar=None + ) + ) + ] + + for node in list.nodes: + subprogram.code.append( + InvokeFunction( + list, + expr=LoadAttr( + expr=LoadTemp(), + name="append" + ), + args=[self.dispatch(node)], + star=None, + dstar=None + ) + ) + + subprogram.code.append( + ReturnFromBlock( + expr=LoadTemp(release=1) + ) + ) + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make an invocation of the subprogram. + + result = InvokeRef(list, 1, + produces_result=1, + ref=subprogram + ) + return result + + def visitListComp(self, listcomp): + + # Make a subprogram for the list comprehension and record it outside the + # main tree. + + subprogram = Subprogram(listcomp, name=None, module=self.module, internal=0, returns_value=1, star=None, dstar=None, original_def=listcomp) + self.current_subprograms.append(subprogram) + + # Make nice annotations for the viewer. + + listcomp._subprogram = subprogram + + # Add a temporary variable. + # Produce for loops within the subprogram. + # Return the result. + + subprogram.code = [ + StoreTemp( + index="listcomp", + expr=InvokeFunction( + expr=LoadName(name="list"), + args=[], + star=None, + dstar=None + ) + ) + ] + self._visitListCompFor(listcomp, listcomp.quals) + [ + ReturnFromBlock( + expr=LoadTemp( + index="listcomp", + release=1 + ) + ) + ] + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make an invocation of the subprogram. + + result = InvokeRef(listcomp, 1, + produces_result=1, + ref=subprogram + ) + return result + + def _visitListCompFor(self, node, quals): + qual = quals[0] + if len(quals) > 1: + body = self._visitListCompFor(node, quals[1:]) + if qual.ifs: + body = self._visitListCompIf(node, qual.ifs, body) + elif qual.ifs: + body = self._visitListCompIf(node, qual.ifs) + else: + body = self._visitListCompBody(node) + return [self._visitFor(qual, body)] + + def _visitListCompIf(self, node, ifs, expr=None): + if_ = ifs[0] + if len(ifs) > 1: + body = self._visitListCompIf(node, ifs[1:], expr) + elif expr is None: + body = self._visitListCompBody(node) + else: + body = expr + return [ + Conditional(if_, 1, + test=InvokeFunction( + if_, + expr=LoadAttr( + expr=self.dispatch(if_.test), + name="__bool__" + ), + ), + body=body, + else_=[] + ) + ] + + def _visitListCompBody(self, node): + return [ + InvokeFunction( + expr=LoadAttr( + expr=LoadTemp(index="listcomp"), + name="append" + ), + args=[self.dispatch(node.expr)], + star=None, + dstar=None + ) + ] + + def visitMod(self, mod): + return self._visitBinary(mod, "__mod__", "__rmod__") + + def visitMul(self, mul): + return self._visitBinary(mul, "__mul__", "__rmul__") + + def visitName(self, name): + result = LoadName(name, 1, name=name.name) + return result + + def _visitNot(self, expr, not_=None): + invocation = InvokeFunction( + not_, # NOTE: May need a real original node. + expr=LoadAttr( + expr=expr, + name="__bool__" + ), + ) + if not_ is not None: + result = Not(not_, 1, expr=invocation) + else: + result = Not(expr=invocation) + return result + + def visitNot(self, not_): + return self._visitNot(self.dispatch(not_.expr), not_) + + def visitOr(self, or_): + + """ + Make a subprogram for the 'or_' node and record its contents inside the + subprogram. Convert... + + Or (test) + (test) + ... + + ...to: + + Subprogram -> Conditional (test) -> ReturnFromBlock ... + (else) -> Conditional (test) -> ReturnFromBlock ... + (else) -> ... + """ + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + + # In the subprogram, make instructions which store each operand, test + # for each operand's truth status, and if appropriate return from the + # subprogram with the value of the operand. + + last = or_.nodes[-1] + results = nodes = [] + + for node in or_.nodes: + expr = self.dispatch(node) + + # Return from the subprogram where the test is satisfied. + + if node is not last: + nodes.append(StoreTemp(expr=expr)) + invocation = InvokeFunction(or_, expr=LoadAttr(expr=LoadTemp(), name="__bool__")) + test = Conditional(test=invocation, body=[ReturnFromBlock(expr=LoadTemp())]) + nodes.append(test) + + # Put subsequent operations in the else section of this conditional. + + nodes = test.else_ = [ReleaseTemp()] + + # For the last operation, return the result. + + else: + nodes.append( + ReturnFromBlock(expr=expr) + ) + + # Finish the subprogram definition. + + subprogram.code = results + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make an invocation of the subprogram. + + result = InvokeRef(or_, 1, + produces_result=1, + ref=subprogram + ) + return result + + def visitPass(self, pass_): + return Pass(pass_, 1) + + def visitPower(self, power): + return self._visitBinary(power, "__pow__", "__rpow__") + + def visitPrint(self, print_): + + """ + Convert... + + Print (dest) -> + (nodes) + + ...to: + + StoreTemp (index) -> "print" + (expr) -> LoadAttr (expr) -> (dest) + (name) -> "write" + InvokeFunction (expr) -> LoadTemp (index) -> "print" + (args) -> [(node)] + ReleaseTemp (index) -> "print" + """ + + if print_.dest is not None: + dest = self.dispatch(print_.dest) + else: + dest = self.dispatch(compiler.ast.Name("stdout")) + + result = Assign(print_, 1, + code=[ + StoreTemp( + index="print", + expr=LoadAttr( + expr=dest, + name="write" + ) + ) + ] + ) + + for node in print_.nodes: + result.code.append( + InvokeFunction( + print_, + expr=LoadTemp(index="print"), + args=[self.dispatch(node)], + star=None, + dstar=None + ) + ) + + result.code.append( + ReleaseTemp(index="print") + ) + + return result + + def visitPrintnl(self, printnl): + result = self.visitPrint(printnl) + result.code.insert( + len(result.code) - 1, + InvokeFunction( + printnl, + expr=LoadTemp(index="print"), + args=[self.dispatch(compiler.ast.Const("\n"))], + star=None, + dstar=None + ) + ) + return result + + def visitRaise(self, raise_): + result = Raise(raise_, 1) + if raise_.expr2 is None: + result.expr = self.dispatch(raise_.expr1) + else: + result.expr = InvokeFunction( + raise_, + expr=self.dispatch(raise_.expr1), + args=[self.dispatch(raise_.expr2)], + star=None, + dstar=None + ) + if raise_.expr3 is not None: + result.traceback = self.dispatch(raise_.expr3) + else: + result.traceback = None + return result + + def visitReturn(self, return_): + result = ReturnFromFunction(return_, 1, + expr=self.dispatch(return_.value) + ) + return result + + def _visitSlice(self, slice, expr, lower, upper, flags, value=None): + if flags == "OP_ASSIGN": + result = InvokeFunction(slice, 1, + expr=LoadAttr( + expr=expr, + name="__setslice__" + ), + star=None, + dstar=None, + args=[lower, upper, value] + ) + elif flags == "OP_APPLY": + args = [] + result = InvokeFunction(slice, 1, + expr=LoadAttr( + expr=expr, + name="__getslice__" + ), + star=None, + dstar=None, + args=[lower, upper] + ) + elif flags == "OP_DELETE": + args = [] + result = InvokeFunction(slice, 1, + expr=LoadAttr( + expr=expr, + name="__delslice__" + ), + star=None, + dstar=None, + args=[lower, upper] + ) + else: + raise NotImplementedError, flags + + return result + + def visitSlice(self, slice, in_sequence=0): + return self._visitSlice(slice, self.dispatch(slice.expr), self.dispatch_or_none(slice.lower), + self.dispatch_or_none(slice.upper), slice.flags, self._visitAssNameOrAttr(slice, in_sequence)) + + def visitSliceobj(self, sliceobj): + return InvokeFunction(sliceobj, 1, + expr=LoadName(name="slice"), + args=self.dispatches(sliceobj.nodes), + star=None, + dstar=None + ) + + def visitStmt(self, stmt): + return self.dispatches(stmt.nodes) + + def visitSub(self, sub): + return self._visitBinary(sub, "__sub__", "__rsub__") + + def _visitSubscript(self, subscript, expr, subs, flags, value=None): + if flags == "OP_ASSIGN": + result = InvokeFunction(subscript, 1, + expr=LoadAttr( + expr=expr, + name="__setitem__" + ), + star=None, + dstar=None, + args=[subs, value] + ) + elif flags == "OP_APPLY": + args = [] + result = InvokeFunction(subscript, 1, + expr=LoadAttr( + expr=expr, + name="__getitem__" + ), + star=None, + dstar=None, + args=[subs] + ) + elif flags == "OP_DELETE": + args = [] + result = InvokeFunction(subscript, 1, + expr=LoadAttr( + expr=expr, + name="__delitem__" + ), + star=None, + dstar=None, + args=[subs] + ) + else: + raise NotImplementedError, flags + + return result + + def _visitSubscriptSubs(self, node, subs): + if len(subs) == 1: + return self.dispatch(subs[0]) + else: + return InvokeFunction(node, 1, + expr=LoadName(name="tuple"), + args=self.dispatches(subs), + star=None, + dstar=None + ) + + def visitSubscript(self, subscript, in_sequence=0): + return self._visitSubscript( + subscript, self.dispatch(subscript.expr), self._visitSubscriptSubs(subscript, subscript.subs), subscript.flags, + self._visitAssNameOrAttr(subscript, in_sequence) + ) + + def visitTryExcept(self, tryexcept): + + """ + Make conditionals for each handler associated with a 'tryexcept' node. + + Convert... + + TryExcept (body) + (else) + (spec/assign/stmt) + ... + + ...to: + + Try (body) + (else) + (handler) -> Conditional (test) -> (stmt) + (body) -> ResetExc ... + (else) -> Conditional (test) -> (stmt) + (body) -> ResetExc ... + (else) -> ... + """ + + result = Try(tryexcept, 1, body=[], else_=[], finally_=[]) + + if tryexcept.body is not None: + result.body = self.dispatch(tryexcept.body) + if tryexcept.else_ is not None: + result.else_ = self.dispatch(tryexcept.else_) + + results = nodes = [] + catch_all = 0 + + for spec, assign, stmt in tryexcept.handlers: + + # If no specification exists, produce an unconditional block. + + if spec is None: + nodes += self.dispatch(stmt) + catch_all = 1 + + # Produce an exception value check. + + else: + test = Conditional( + isolate_test=1, + test=CheckType(expr=LoadExc(), choices=self._visitTryExcept(spec)) + ) + test.body = [] + + if assign is not None: + test.body.append( + Assign( + code=[ + StoreTemp(expr=LoadExc()), + self.dispatch(assign), + ReleaseTemp() + ] + ) + ) + + test.body += [ResetExc()] + self.dispatch(stmt) + nodes.append(test) + nodes = test.else_ = [] + + # Add a raise operation to deal with unhandled exceptions. + + if not catch_all: + nodes.append( + Raise( + expr=LoadExc()) + ) + + result.handler = results + return result + + def _visitTryExcept(self, spec): + + "Return a list of nodes for the given exception type 'spec'." + + if isinstance(spec, compiler.ast.Tuple): + nodes = [] + for node in spec.nodes: + nodes += self._visitTryExcept(node) + else: + nodes = [self.dispatch(spec)] + return nodes + + def visitTryFinally(self, tryfinally): + result = Try(tryfinally, 1, body=[], else_=[], finally_=[]) + if tryfinally.body is not None: + result.body = self.dispatch(tryfinally.body) + if tryfinally.final is not None: + result.finally_ = self.dispatch(tryfinally.final) + return result + + def visitTuple(self, tuple): + + "Make a MakeTuple node containing the original 'tuple' contents." + + result = MakeTuple(tuple, 1, + nodes=self.dispatches(tuple.nodes) + ) + return result + + def visitUnaryAdd(self, unaryadd): + return self._visitUnary(unaryadd, "__pos__") + + def visitUnarySub(self, unarysub): + return self._visitUnary(unarysub, "__neg__") + + def visitWhile(self, while_): + + """ + Make a subprogram for the 'while' node and record its contents inside the + subprogram. Convert... + + While (test) -> (body) + (else) + + ...to: + + Subprogram -> Conditional (test) -> (body) -> Invoke subprogram + (else) -> Conditional (test) -> ReturnFromBlock ... + (else) -> ... + """ + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=0, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + + # Include a conditional statement in the subprogram. + # Inside the conditional, add a recursive invocation to the subprogram + # if the test condition was satisfied. + # Return within the main section of the loop. + + test = Conditional( + test=InvokeFunction( + while_, + expr=LoadAttr( + expr=self.dispatch(while_.test), + name="__bool__"), + ), + body=self.dispatch(while_.body) + [ + InvokeRef( + while_, + ref=subprogram + ), + ReturnFromBlock() + ], + else_=[] + ) + + # Provide the else section, if present, along with an explicit return. + + if while_.else_ is not None: + test.else_ = self.dispatch(while_.else_) + [ReturnFromBlock()] + + # Finish the subprogram definition. + + subprogram.code = [test] + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + # Make an invocation of the subprogram. + + result = InvokeRef(while_, 1, ref=subprogram) + + # Make nice annotations for the viewer. + + while_._test_call = subprogram.code[0].test + + return result + + # NOTE: Not actually supported. + # NOTE: Virtually the same as visitReturn... + + def visitYield(self, yield_): + result = Yield(yield_, 1, + expr=self.dispatch(yield_.value) + ) + return result + + # Convenience methods. + + def _visitBinary(self, binary, left_name, right_name): + return self._visitBinaryOp(binary, self.dispatch(binary.left), self.dispatch(binary.right), left_name, right_name) + + def _visitBinaryCompareOp(self, binary, left, right, left_name, right_name): + + """ + Emulate the current mechanisms by producing nodes as follows: + + InvokeRef -> Subprogram -> StoreTemp (expr) -> x.__lt__(y) + Conditional (test) -> __is__(LoadTemp, NotImplemented) + (body) -> ReleaseTemp + StoreTemp (expr) -> y.__gt__(x) + Conditional (test) -> __is__(LoadTemp, NotImplemented) + (body) -> ReturnFromBlock (expr) -> False + (else) -> ReturnFromBlock (expr) -> LoadTemp + (else) -> ReturnFromBlock (expr) -> LoadTemp + """ + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + + subprogram.code = [ + StoreTemp( + expr=InvokeFunction( + binary, + expr=LoadAttr(expr=left, name=left_name), + args=[right], + star=None, + dstar=None) + ), + Conditional( + isolate_test=1, + test=CheckType( + expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] + ), + body=[ + ReleaseTemp(), + StoreTemp( + expr=InvokeFunction( + binary, + expr=LoadAttr(expr=right, name=right_name), + args=[left], + star=None, + dstar=None) + ), + Conditional( + isolate_test=1, + test=CheckType( + expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] + ), + body=[ + ReturnFromBlock( + expr=LoadName(name="False") + ) + ], + else_=[ + CheckType( + inverted=1, expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] + ), + ReturnFromBlock( + expr=LoadTemp() + ) + ] + ) + ], + else_=[ + CheckType( + inverted=1, expr=LoadTemp(), choices=[LoadName(name="NotImplementedType")] + ), + ReturnFromBlock( + expr=LoadTemp() + ) + ] + ) + ] + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + result = InvokeRef( + binary, + produces_result=1, + ref=subprogram + ) + + # Make nice annotations for the viewer. + + binary._left_call = subprogram.code[0].expr + binary._right_call = subprogram.code[1].body[1].expr + + return result + + def _visitBinaryOp(self, binary, left, right, left_name, right_name): + + """ + Emulate the current mechanisms by producing nodes as follows: + + InvokeRef -> Subprogram -> Try (body) -> ReturnFromBlock (expr) -> x.__add__(y) + (else) + (handler) -> Conditional (test) -> CheckType (expr) -> LoadExc + (choices) -> LoadName TypeError + (body) -> ReturnFromBlock (expr) -> y.__radd__(x) + (else) + """ + + subprogram = Subprogram(name=None, module=self.module, internal=1, returns_value=1, params=[], star=None, dstar=None) + self.current_subprograms.append(subprogram) + + subprogram.code = [ + Try(binary, 1, + body=[ + ReturnFromBlock( + expr=InvokeFunction( + binary, + expr=LoadAttr(expr=left, name=left_name), + args=[right], + star=None, + dstar=None) + ) + ], + else_=[], + finally_=[], + handler=[ + Conditional( + test=CheckType(expr=LoadExc(), choices=[LoadName(name="TypeError")]), + body=[ + ReturnFromBlock( + expr=InvokeFunction( + binary, + expr=LoadAttr(expr=right, name=right_name), + args=[left], + star=None, + dstar=None) + ) + ], + else_=[] + ) + ] + ) + ] + + self.current_subprograms.pop() + self.subprograms.append(subprogram); self.subnames[subprogram.full_name()] = subprogram + + result = InvokeRef( + binary, + produces_result=1, + ref=subprogram + ) + + # Make nice annotations for the viewer. + + binary._left_call = subprogram.code[0].body[0].expr + binary._right_call = subprogram.code[0].handler[0].body[0].expr + + return result + + def _visitBuiltin(self, builtin, name): + result = InvokeFunction(builtin, 1, expr=LoadName(name=name), args=self.dispatches(builtin.nodes)) + return result + + def _visitUnary(self, unary, name): + result = InvokeFunction(unary, 1, + expr=LoadAttr( + expr=self.dispatch(unary.expr), + name=name + ) + ) + + # Make nice annotations for the viewer. + + unary._unary_call = result + + return result + +# Convenience functions. + +def simplify(filename, builtins=0, module_name=None): + + """ + Simplify the module stored in the file with the given 'filename'. + + If the optional 'builtins' parameter is set to a true value (the default + being a false value), then the module is considered as the builtins module. + """ + + simplifier = Simplifier(builtins) + module = compiler.parseFile(filename) + compiler.misc.set_filename(filename, module) + if builtins: + name = module_name or "__builtins__" + else: + path, ext = os.path.splitext(filename) + path, name = os.path.split(path) + name = module_name or name + simplified = simplifier.process(module, name) + return simplified + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/annotate.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/annotate.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,1789 @@ +#!/usr/bin/env python + +""" +Annotate program node structures. The code in this module operates upon nodes +which are produced when simplifying AST node trees originating from the compiler +module. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +-------- + +To use this module, the easiest approach is to use the load function: + +load(filename, builtins) + +To control module importing, an importer should be constructed and employed. +Here, the standard path for module searching is used: + +importer = Importer(sys.path) +load(filename, builtins, importer) + +Underneath the load function, the annotate function provides support for +annotating modules already processed by simplify and fixnames: + +annotate(module, builtins) + +And at the most basic level, the most intricate approach involves obtaining an +Annotator object: + +annotator = Annotator() + +Then, processing an existing module with it: + +annotator.process(module) + +If a module containing built-in classes and functions has already been +annotated, such a module should be passed in as an additional argument: + +annotator.process(module, builtins) +""" + +from simplify.simplified import * +import simplify, simplify.fixnames # for the load function +import compiler +import os + +class System: + + """ + A class maintaining the state of the annotation system. When the system + counter can no longer be incremented by any annotation operation, the + system may be considered stable and fully annotated. + """ + + def __init__(self): + self.count = 0 + + def init(self, node, attr="types"): + + "Initialise a 'node' for annotation." + + if not hasattr(node, attr): + setattr(node, attr, set()) + + def annotate(self, node, types, attr="types"): + + "Annotate the given 'node' with the given 'types'." + + self.init(node, attr) + self.combine(getattr(node, attr), types) + + def combine(self, target, types): + + """ + Combine the 'target' list with the given 'types', counting new members. + """ + + for type in types: + if type not in target: + target.add(type) + self.count += 1 + +system = System() + +# Exceptions. + +class AnnotationError(SimplifiedError): + + "An error in the annotation process." + + pass + +class AnnotationMessage(Exception): + + "A lesser annotation error." + + pass + +# Annotation. + +class Annotator(Visitor): + + """ + The type annotator which traverses the program nodes, typically depth-first, + and maintains a record of the current set of types applying to the currently + considered operation. Such types are also recorded on the nodes, and a + special "system" record is maintained to monitor the level of annotation + activity with a view to recognising when no more annotations are possible. + + Throughout the annotation activity, type information consists of lists of + Attribute objects where such objects retain information about the context of + the type (since a value in the program may be associated with an object or + class) and the actual type of the value being manipulated. Upon accessing + attribute information on namespaces, additional accessor information is also + exchanged - this provides a means of distinguishing between the different + types possible when the means of constructing the namespace may depend on + run-time behaviour. + + Covered: Assign, CheckType, Conditional, Global, Import, InvokeRef, + InvokeFunction, LoadAttr, LoadExc, LoadName, LoadRef, LoadTemp, + Module, Not, Pass, Raise, ReleaseTemp, ReturnFromBlock, + ReturnFromFunction, StoreAttr, StoreName, StoreTemp, Subprogram, + Try. + """ + + def __init__(self, importer=None): + + "Initialise the visitor with an optional 'importer'." + + Visitor.__init__(self) + self.system = system + self.importer = importer or Importer() + + # Satisfy visitor issues. + + self.visitor = self + + def process(self, module, builtins=None): + + """ + Process the given 'module', using the optional 'builtins' to access + built-in classes and functions. + """ + + self.subprograms = [] + self.current_subprograms = [] + self.current_namespaces = [] + self.rerun_subprograms = {} + self.namespace = None + self.module = module + + # Process the module, supplying builtins if possible. + + self.builtins = builtins + self.global_namespace = Namespace() + + if builtins is not None: + self.builtins_namespace = builtins.namespace + else: + self.builtins_namespace = self.global_namespace + + # NOTE: Not declaring module namespace usage, even though it is used. + + self.process_node(module, self.global_namespace, 0) + + def process_node(self, node, locals, using_module_namespace): + + """ + Process a subprogram or module 'node', indicating the initial 'locals'. + Note that this method may mutate nodes in the original program. + """ + + # Recursion test. + + if node in self.current_subprograms: + if not self.rerun_subprograms.has_key(node): + self.rerun_subprograms[node] = [] + self.rerun_subprograms[node].append(locals) + return + + # Record the current subprogram and namespace. + + self.current_subprograms.append(node) + + # Determine the namespace. + + self.current_namespaces.append(self.namespace) + self.namespace = locals + + # Add namespace details to any structure involved. + + if getattr(node, "structure", None) is not None: + node.structure.namespace = Namespace() + + # Initialise bases where appropriate. + + if hasattr(node.structure, "bases"): + base_refs = [] + for base in node.structure.bases: + self.dispatch(base) + base_refs.append(self.namespace.types) + node.structure.base_refs = base_refs + + # Dispatch to the code itself. + + node.namespace = self.namespace + self.set_module_namespace(using_module_namespace) + + self.dispatch(node) + self.extract_results(node) + + while self.rerun_subprograms.has_key(node): + all_rerun_locals = self.rerun_subprograms[node] + del self.rerun_subprograms[node] + for rerun_locals in all_rerun_locals: + #print "Re-running", node, "with", rerun_locals + + self.namespace = rerun_locals + node.namespace = rerun_locals + self.set_module_namespace(using_module_namespace) + + self.dispatch(node) + self.extract_results(node) + + # Restore the previous subprogram and namespace. + + self.namespace = self.current_namespaces.pop() + self.current_subprograms.pop() + self.reset_module_namespace(using_module_namespace) + + def set_module_namespace(self, using_module_namespace): + + """ + In order to keep global accesses working, the module namespace must be + adjusted. + """ + + if using_module_namespace: + self.module.namespace = self.namespace + + def reset_module_namespace(self, using_module_namespace): + + """ + In order to keep global accesses working, the module namespace must be + reset. + """ + + if using_module_namespace: + self.module.namespace = self.namespace + + def extract_results(self, node): + + "Extract results from the namespace." + + node.namespace = self.namespace + self.system.annotate(node, self.namespace.raises, "raises") + self.system.annotate(node, self.namespace.returns, "returns") + if hasattr(node, "return_locals"): + node.return_locals.update(self.namespace.return_locals) + + def annotate(self, node, types=None): + + """ + Annotate the given 'node' in the system, using either the optional + 'types' or the namespace's current type information. + """ + + if types is None: + self.system.annotate(node, self.namespace.types) + else: + self.system.annotate(node, types) + + def annotate_parameters(self, node, items): + + """ + Annotate the given 'node' using the given 'items' and updating the + system's annotation counter. + """ + + if not hasattr(node, "paramtypes"): + node.paramtypes = {} + + for param, types in items: + if not node.paramtypes.has_key(param): + node.paramtypes[param] = set() + self.system.combine(node.paramtypes[param], types) + + # Visitor methods. + + def default(self, node): + + """ + Process the given 'node', given that it does not have a specific + handler. + """ + + raise AnnotationMessage, "Node '%s' not supported." % node + + def dispatch(self, node, *args): + try: + Visitor.dispatch(self, node, *args) + except AnnotationError, exc: + exc.add(node) + raise + except AnnotationMessage, exc: + raise AnnotationError(exc, node) + + # Specific node methods. + + def visitAssign(self, assign): + + """ + Process the 'assign' node and its contents. + """ + + self.dispatches(assign.code) + + def visitCheckType(self, checktype): + + """ + Process the 'checktype' node, finding the possible types of the + exception, and processing each choice to build a list of checked types + for the exception. + """ + + inverted = getattr(checktype, "inverted", 0) + self.dispatch(checktype.expr) + + expr_types = self.namespace.types + choice_types = set() + choices = [] + + for choice in checktype.choices: + choices.append(self.dispatch(choice)) + choice_types.update(self.namespace.types) + + for expr_type in expr_types: + in_choices = expr_type.type.get_class() in choice_types + + # Filter out types not in the choices list unless the operation is + # inverted; in which case, filter out types in the choices list. + + if not inverted and not in_choices or inverted and in_choices: + self._prune_non_accesses(checktype.expr, expr_type) + + def visitConditional(self, conditional): + + """ + Process the 'conditional' node, processing the test, body and else + clauses and recording their processed forms. The body and else clauses + are processed within their own namespaces, and the test is also + processed in its own namespace if 'isolate_test' is set on the + 'conditional' node. + """ + + # Conditionals keep local namespace changes isolated. + # With Return nodes inside the body/else sections, the changes are + # communicated to the caller. + + is_module = self.namespace is self.module.namespace + + # Where the test is closely associated with the body, save the namespace + # before entering the test. + + if conditional.isolate_test: + saved_namespace = self.namespace + self.namespace = Namespace() + if is_module: + self.module.namespace = self.namespace + self.namespace.merge_namespace(saved_namespace) + + self.dispatch(conditional.test) + + # Where the test may affect the body and the else clause, save the + # namespace after processing the test. + + if not conditional.isolate_test: + saved_namespace = self.namespace + self.namespace = Namespace() + if is_module: + self.module.namespace = self.namespace + self.namespace.merge_namespace(saved_namespace) + + # NOTE: Exception recording. + + else: + test_raises = set() + test_raises.update(self.namespace.raises) + + # Process the body clause. + + self.dispatches(conditional.body) + body_namespace = self.namespace + + # Use the saved namespace as a template for the else clause. + + self.namespace = Namespace() + if is_module: + self.module.namespace = self.namespace + self.namespace.merge_namespace(saved_namespace) + + # Process the else clause. + + self.dispatches(conditional.else_) + else_namespace = self.namespace + + # Merge the body and else namespaces. + + self.namespace = Namespace() + if is_module: + self.module.namespace = self.namespace + self.namespace.merge_namespace(body_namespace) + self.namespace.merge_namespace(else_namespace) + + # NOTE: Test of exception type pruning based on the test/body. + # Note that the checked exceptions are tested for re-raising. + + if conditional.isolate_test: + for exc_type in test_raises: + if exc_type not in body_namespace.raises: + self.namespace.revoke_exception_type(exc_type) + + def visitGlobal(self, global_): + + """ + Leave the 'global_' node unprocessed since namespaces should have + already been altered to take global names into consideration. + """ + + pass + + def visitImport(self, import_): + + """ + Process the 'import_' node, importing the module with the stated name + and storing details on the node. + """ + + module = self.importer.load(import_.name, self.builtins, getattr(import_, "alias", None)) + if module is not None: + self.namespace.set_types(set([module])) + else: + self.namespace.set_types(set()) + self.annotate(import_) # mainly for viewing purposes + + def _visitInvoke(self, invoke, invocation_types, have_args): + + """ + Process the 'invoke' node, using the given 'invocation_types' as the + list of callables to be investigated for instantiation or for the + invocation of functions or blocks. If 'have_args' is a true value, any + invocation or instantiation will involve arguments. + """ + + # Now locate and invoke the subprogram. This can be complicated because + # the target may be a class or object, and there may be many different + # related subprograms. + + invocations = [] + + # Visit each callable in turn, finding subprograms. + + for attr in invocation_types: + + # Deal with class invocations by providing instance objects. + # Here, each class is queried for the __init__ method, which may + # exist for some combinations of classes in a hierarchy but not for + # others. + + if isinstance(attr.type, Class): + attributes = get_attributes(attr.type, "__init__") + + # Deal with object invocations by using __call__ methods. + + elif isinstance(attr.type, Instance): + attributes = get_attributes(attr.type, "__call__") + + # Normal functions or methods are more straightforward. + # Here, we model them using an attribute with no context and with + # no associated accessor. + + else: + attributes = [(attr, None)] + + # Inspect each attribute and extract the subprogram. + + for attribute, accessor in attributes: + + # If a class is involved, presume that it must create a new + # object. + + if isinstance(attr.type, Class): + + # Instantiate the class. + + instance = self.new_instance(invoke, attr.type) + + # For instantiations, switch the context. + + if attribute is not None: + attribute = Attribute(instance, attribute.type) + + # Request an instance-specific initialiser. + + attribute = attr.type.get_attribute_for_instance(attribute, instance) + + # Skip cases where no callable is found. + + if attribute is not None: + + # If a subprogram is defined, invoke it. + + self.invoke_subprogram(invoke, attribute) + if attribute.type not in invocations: + invocations.append(attribute.type) + + elif not isinstance(attr.type, Class): + print "Invocation type is None for", accessor + + else: + + # Test to see if no arguments were supplied in cases where no + # initialiser was found. + + if have_args: + raise AnnotationMessage, "No initialiser found for '%s' with arguments." % attr.type + + # Special case: initialisation. + + if isinstance(attr.type, Class): + + # Associate the instance with the result of this invocation. + + self.namespace.set_types(set([Attribute(None, instance)])) + self.annotate(invoke) + + # Remember the invocations that were found, along with the return type + # information. + + invoke.invocations = invocations + self.namespace.set_types(getattr(invoke, "types", set())) + + def visitInvokeRef(self, invoke): + + """ + Process the 'invoke' node, first finding the callables indicated by the + reference. + """ + + # Where the invocation belongs to an instance but the invoked subprogram + # does not, request a special copy. + + instance = getattr(invoke, "instance", None) + if instance is not None and getattr(invoke.ref, "instance", None) is None: + if invoke.ref.copies.has_key(instance): + invoke.ref = invoke.ref.copies[instance] + else: + invoke.ref = invoke.ref.copy(instance) + #print "Created", invoke.ref, "for", getattr(invoke.ref, "instance", None) + invoke.ref.module.simplifier.subnames[invoke.ref.full_name()] = invoke.ref + invocation_types = [Attribute(None, invoke.ref)] + self._visitInvoke(invoke, invocation_types, have_args=0) + + def visitInvokeFunction(self, invoke): + + """ + Process the 'invoke' node, first finding the callables indicated by the + expression. + """ + + self.dispatch(invoke.expr) + invocation_types = self.namespace.types + + # Invocation processing starts with making sure that the arguments have + # been processed. + + self._visitInvoke(invoke, invocation_types, have_args=self.process_args(invoke)) + + def visitLoadAttr(self, loadattr): + + """ + Process the 'loadattr' node, processing and storing the expression, and + using the expression's types to construct records of accesses and + non-accesses using the stated attribute name. + """ + + self.dispatch(loadattr.expr) + types = set() + raises = set() + non_accesses = [] + accesses = {} + + # For each expression type... + + for attr in self.namespace.types: + + # Find types for the named attribute. + + attributes = get_attributes(attr.type, loadattr.name) + + # Where no attributes exist... + + if not attributes: + + # Register new invalid accesses and mark a possible exception. + + if not attr in non_accesses: + non_accesses.append(attr) + exc = self.get_builtin_instances(loadattr, "AttributeError") + raises.update(exc) + self.namespace.raises.update(exc) + + # Revoke this type from any name involved. + + self._prune_non_accesses(loadattr.expr, attr) + + # For each type found... + + for attribute, accessor in attributes: + + # For actual attributes, register the type and remember the + # access. + + if attribute is not None: + types.add(attribute) + if not accesses.has_key(attr.type): + accesses[attr.type] = [] + if not (attribute, accessor) in accesses[attr.type]: + accesses[attr.type].append((attribute, accessor)) + + # Otherwise, register new invalid accesses and note a possible + # exception. + + else: + if not attr in non_accesses: + non_accesses.append(attr) + exc = self.get_builtin_instances(loadattr, "AttributeError") + raises.update(exc) + self.namespace.raises.update(exc) + + # Revoke this type from any name involved. + + self._prune_non_accesses(loadattr.expr, attr) + + if not types: + print "No attribute found for", loadattr.name, "given", self.namespace.types + + # Remember the result types. + + self.namespace.set_types(types) + loadattr.non_accesses = non_accesses + loadattr.accesses = accesses + loadattr.raises = raises + self.annotate(loadattr) + + def _prune_non_accesses(self, expr, attr): + + """ + Prune type information from 'expr' where the given 'attr' has been + shown to be a non-access. + """ + + if isinstance(expr, LoadName): + self.namespace.revoke(expr.name, attr) + elif isinstance(expr, LoadExc): + self.namespace.revoke_exception_type(attr) + elif isinstance(expr, LoadTemp): + self.namespace.revoke_temp_type(getattr(expr, "index", None), attr) + + # LoadAttr cannot be pruned since this might unintentionally prune + # legitimate types from other applications of the referenced type, it + # almost certainly doesn't take "concurrent" mutation into + # consideration (where in a running program, the pruned type is actually + # reintroduced, making the pruning invalid), and there is no easy way of + # preserving the meaning of a namespace without either creating lots of + # specialised instances, and even then... + + #elif isinstance(expr, LoadAttr): + # for expr_attr in expr.expr.types: + # if hasattr(expr_attr.type, "namespace"): + # expr_attr.type.namespace.revoke(expr.name, attr) + + def visitLoadExc(self, loadexc): + + """ + Process the 'loadexc' node, discovering the possible exception types + raised. + """ + + self.namespace.set_types(self.namespace.raises) + self.annotate(loadexc) + + def visitLoadName(self, loadname): + + """ + Process the 'loadname' node, processing the name information on the node + to determine which types are involved with the name. + """ + + self.namespace.set_types(self.namespace.load(loadname.name)) + self.annotate(loadname) + + def visitLoadRef(self, loadref): + + """ + Process the 'loadref' node, obtaining type information about the + reference stated on the node. + """ + + self.namespace.set_types(set([Attribute(None, loadref.ref)])) + self.annotate(loadref) + + def visitLoadTemp(self, loadtemp): + + """ + Process the 'loadtemp' node, obtaining type information about the + temporary variable accessed, and removing variable information where the + 'release' attribute has been set on the node. + """ + + index = getattr(loadtemp, "index", None) + try: + if getattr(loadtemp, "release", 0): + self.namespace.set_types(self.namespace.temp[index].pop()) + else: + self.namespace.set_types(self.namespace.temp[index][-1]) + except KeyError: + raise AnnotationMessage, "Temporary store index '%s' not defined." % index + self.annotate(loadtemp) + + def visitMakeTuple(self, maketuple): + + """ + Process the 'maketuple' node and its contents. + """ + + # Get a tuple and populate it with type information for the contents. + + tuples = self.get_builtin_instances(maketuple, "tuple") + + # NOTE: This is dependent on the tuple definition in the builtins. + + for node in maketuple.nodes: + self.dispatch(node) + for t in tuples: + t.type.namespace.add("value", self.namespace.types) + + self.namespace.set_types(tuples) + self.annotate(maketuple) + + def visitModule(self, module): + + """ + Process the 'module' and its contents. + """ + + self.dispatches(module.code) + + def visitNot(self, not_): + + "Process the 'not_' node and its expression." + + self.dispatch(not_.expr) + + def visitPass(self, pass_): + + "Leave the 'pass_' node unprocessed." + + pass + + def visitRaise(self, raise_): + + """ + Process the 'raise_' node, processing any traceback information along + with the raised exception expression, converting the node into a kind of + invocation where the expression is found not to be an invocation itself. + This node affects the namespace, adding exception types to the list of + those raised in the namespace. + """ + + if getattr(raise_, "traceback", None) is not None: + self.dispatch(raise_.traceback) + self.dispatch(raise_.expr) + + # Handle bare name exceptions by converting any classes to instances. + + if not isinstance(raise_.expr, InvokeFunction): + raise_.pos_args = [] + raise_.kw_args = {} + raise_.star = None + raise_.dstar = None + types = set() + for attr in self.namespace.types: + if isinstance(attr.type, Class): + self._visitInvoke(raise_, [attr], have_args=0) + types.update(self.namespace.types) + else: + types = self.namespace.types + + self.namespace.raises.update(types) + + def visitReleaseTemp(self, releasetemp): + + """ + Process the 'releasetemp' node, removing temporary variable information + from the current namespace. + """ + + index = getattr(releasetemp, "index", None) + try: + self.namespace.temp[index].pop() + except KeyError: + raise AnnotationMessage, "Temporary store index '%s' not defined." % index + except IndexError: + pass #raise AnnotationMessage, "Temporary store index '%s' is empty." % index + + def visitResetExc(self, resetexc): + self.namespace.raises = set() + + def visitReturn(self, return_): + + """ + Process the 'return_' node, processing any expression and obtaining type + information to be accumulated in the current namespace's list of return + types. A snapshot of the namespace is taken for the purposes of + reconciling or merging namespaces where subprograms actually share + locals with their callers. + """ + + if hasattr(return_, "expr"): + self.dispatch(return_.expr) + self.namespace.returns.update(self.namespace.types) + self.annotate(return_) + self.namespace.snapshot() + + visitReturnFromBlock = visitReturn + visitReturnFromFunction = visitReturn + + def visitStoreAttr(self, storeattr): + + """ + Process the 'storeattr' node, processing the expression and target, and + using the type information obtained to build records of legitimate + writes to the stated attribute, along with "impossible" non-writes to + the attribute. + """ + + self.dispatch(storeattr.expr) + expr = self.namespace.types + self.dispatch(storeattr.lvalue) + writes = {} + non_writes = [] + for attr in self.namespace.types: + # NOTE: Impose "atomic" constraints on certain types. + if attr is None: + if not attr in non_writes: + non_writes.append(attr) + continue + attr.type.namespace.add(storeattr.name, expr) + writes[attr.type] = attr.type.namespace.load(storeattr.name) + if not writes: + print "Unable to store attribute", storeattr.name, "given", self.namespace.types + storeattr.writes = writes + storeattr.non_writes = non_writes + + def visitStoreName(self, storename): + + """ + Process the 'storename' node, processing the expression on the node and + associating the type information obtained with the stated name in the + current namespace. + """ + + self.dispatch(storename.expr) + self.namespace.store(storename.name, self.namespace.types) + self.annotate(storename) + + def visitStoreTemp(self, storetemp): + + """ + Process the 'storetemp' node, processing the expression on the node and + associating the type information obtained with a temporary variable in + the current namespace. + """ + + self.dispatch(storetemp.expr) + index = getattr(storetemp, "index", None) + if not self.namespace.temp.has_key(index): + self.namespace.temp[index] = [] + self.namespace.temp[index].append(self.namespace.types) + + def visitSubprogram(self, subprogram): + + """ + Process the 'subprogram' node, processing its contents (a group of nodes + comprising the subprogram). + """ + + self.dispatches(subprogram.code) + + def visitTry(self, try_): + + """ + Process the 'try_' node, processing the body clause in its own namespace + derived from the current namespace, processing any handler clause using + the namespace information accumulated in the body, and processing any + else and finally clauses, attempting to supply each with appropriate + namespace information. + """ + + is_module = self.namespace is self.module.namespace + + self.dispatches(try_.body) + + # Save the namespace from the body. + + body_namespace = Namespace() + body_namespace.merge_namespace(self.namespace) + + # Process the handler. + + if hasattr(try_, "handler"): + self.dispatches(try_.handler) + + # Save the namespace from the handler. + + handler_namespace = Namespace() + handler_namespace.merge_namespace(self.namespace) + + # Remember the raised exceptions encountered so far. + + raises = self.namespace.raises + + # Process the else clause. + + if hasattr(try_, "else_"): + + # Restore the body namespace for the else clause. + + self.namespace = body_namespace + if is_module: + self.module.namespace = self.namespace + + # Empty the raised exceptions for the else clause. + + self.namespace.raises = set() + self.dispatches(try_.else_) + self.namespace.raises = raises + + # Merge the namespaces. + + self.namespace = Namespace() + if is_module: + self.module.namespace = self.namespace + self.namespace.merge_namespace(body_namespace) + self.namespace.merge_namespace(handler_namespace) + + # Process the finally clause, if any. + + self.dispatches(try_.finally_) + + def visitYield(self, yield_): + raise NotImplementedError, "The yield statement is not currently supported." + + # Utility methods. + + def get_builtin_instances(self, node, name): + return set([Attribute(None, self.new_instance(node, attr.type)) for attr in self.builtins.namespace[name]]) + + def new_instance(self, node, type): + + "For the given 'node', obtain an instance from the given 'type'." + + if not type.has_instance(node): + instance = Instance() + instance.namespace = Namespace() + instance.namespace.store("__class__", set([Attribute(None, type)])) + type.add_instance(node, instance) + else: + instance = type.get_instance(node) + + return instance + + def invoke_subprogram(self, invoke, attribute): + + """ + Invoke using the given 'invoke' node the subprogram represented by the + given 'attribute'. + """ + + # Test for context information, making it into a real attribute. + + if attribute.context is not None: + context = Attribute(None, attribute.context) + target = attribute.type + else: + context = None + target = attribute.type + + # Test to see if anything has changed. + + if hasattr(invoke, "syscount") and invoke.syscount.has_key(target) and invoke.syscount[target] == self.system.count: + return + + # Remember the state of the system. + + else: + if not hasattr(invoke, "syscount"): + invoke.syscount = {} + invoke.syscount[target] = self.system.count + + # Provide the correct namespace for the invocation. + # This may be a "shared" namespace... + + if getattr(invoke, "share_locals", 0): + namespace = Namespace() + namespace.merge_namespace(self.namespace, everything=0) + using_module_namespace = self.namespace is self.module.namespace + + # Or it may be a structure... + + elif getattr(target, "structure", None): + namespace = Namespace() + using_module_namespace = 0 + + # Or it may be a new namespace populated with the supplied parameters. + + else: + items = self.make_items(invoke, target, context) + namespace = Namespace() + namespace.merge_items(items) + using_module_namespace = 0 + + # NOTE: Avoid PEP 227 (nested scopes) whilst permitting references to a + # NOTE: subprogram within itself. Do not define the name of the function + # NOTE: within a method definition. + + if getattr(target, "name", None) is not None and not getattr(target, "is_method", 0): + namespace.store(target.name, set([Attribute(None, target)])) + + # Process the subprogram. + + self.process_node(target, namespace, using_module_namespace) + + # NOTE: Improve and verify this. + # If the invocation returns a value, acquire the return types. + + if getattr(target, "returns_value", 0): + self.namespace.set_types(target.returns) + self.annotate(invoke) + + # If it is a normal block, merge the locals. + # This can happen in addition to the above because for things like + # logical expressions, the namespace can be modified whilst values are + # returned as results. + + if getattr(invoke, "share_locals", 0): + self.namespace.reset() + + # Merge the locals snapshots. + + for locals in target.return_locals: + + # For blocks returning values (such as operations), do not merge + # snapshots or results. + + if getattr(target, "returns_value", 0): + self.namespace.merge_namespace(locals, everything=0) + + # For blocks not returning values (such as loops), merge + # snapshots and results since they contain details of genuine + # returns. + + else: + self.namespace.merge_namespace(locals) + + # Incorporate any raised exceptions. + + if not hasattr(invoke, "raises"): + invoke.raises = set() + invoke.raises.update(target.raises) + self.namespace.raises.update(target.raises) + + def process_args(self, invocation): + + """ + Process the arguments associated with an 'invocation'. Return whether + any arguments were processed. + """ + + self.dispatches(invocation.pos_args) + self.dispatch_dict(invocation.kw_args) + + # Get type information for star and dstar arguments. + + if invocation.star is not None: + param, default = invocation.star + self.dispatch(default) + invocation.star = param, default + + if invocation.dstar is not None: + param, default = invocation.dstar + self.dispatch(default) + invocation.dstar = param, default + + if invocation.pos_args or invocation.kw_args or invocation.star or invocation.dstar: + return 1 + else: + return 0 + + def make_items(self, invocation, subprogram, context): + + """ + Make an items mapping for the 'invocation' of the 'subprogram' using the + given 'context' (which may be None). + """ + + # NOTE: Support class methods! + + if context is not None and isinstance(context.type, Instance): + pos_args = [Self(context)] + invocation.pos_args + else: + pos_args = invocation.pos_args + + # Duplicate the keyword arguments - we remove them in processing below. + + kw_args = {} + kw_args.update(invocation.kw_args) + + # Sort the arguments into positional and keyword arguments. + + params = subprogram.params + items = [] + star_args = [] + + # Match each positional argument, taking excess arguments as star args. + + for arg in pos_args: + if params: + param, default = params[0] + if arg is None: + arg = default + if hasattr(arg, "types"): + items.append((param, arg.types)) + else: + items.append((param, set())) # Annotation has not succeeded. + params = params[1:] + else: + star_args.append(arg) + + # Collect the remaining defaults. + + while params: + param, default = params[0] + if kw_args.has_key(param): + arg = kw_args[param] + del kw_args[param] + elif default is not None: + self.dispatch(default) + arg = default + else: + raise AnnotationMessage, "No argument supplied in '%s' for parameter '%s'." % (subprogram, param) + if hasattr(arg, "types"): + items.append((param, arg.types)) + else: + items.append((param, set())) # Annotation has not succeeded. + params = params[1:] + + dstar_args = kw_args.items() + + # Construct temporary objects. + + if star_args: + star_invocation = self.make_star_args(invocation, subprogram, star_args) + self.dispatch(star_invocation) + star_types = star_invocation.types + else: + star_types = None + + if dstar_args: + dstar_invocation = self.make_dstar_args(invocation, subprogram, dstar_args) + self.dispatch(dstar_invocation) + dstar_types = dstar_invocation.types + else: + dstar_types = None + + # NOTE: Merge the objects properly. + + star_types = star_types or invocation.star and invocation.star.types + dstar_types = dstar_types or invocation.dstar and invocation.dstar.types + + # Add star and dstar. + + if star_types is not None: + if subprogram.star is not None: + param, default = subprogram.star + items.append((param, star_types)) + else: + raise AnnotationMessage, "Invocation provides unwanted *args." + elif subprogram.star is not None: + param, default = subprogram.star + if not hasattr(default, "types"): + subprogram.star = param, self.dispatch(default) # NOTE: Review reprocessing. + items.append((param, default.types)) + + if dstar_types is not None: + if subprogram.dstar is not None: + param, default = subprogram.dstar + items.append((param, dstar_types)) + else: + raise AnnotationMessage, "Invocation provides unwanted **args." + elif subprogram.dstar is not None: + param, default = subprogram.dstar + if not hasattr(default, "types"): + subprogram.dstar = param, self.dispatch(default) # NOTE: Review reprocessing. + items.append((param, default.types)) + + # Record the parameter types. + + self.annotate_parameters(subprogram, items) + return subprogram.paramtypes.items() + + def make_star_args(self, invocation, subprogram, star_args): + + "Make a subprogram which initialises a list containing 'star_args'." + + if not hasattr(invocation, "stars"): + invocation.stars = {} + + if not invocation.stars.has_key(subprogram.full_name()): + instance = getattr(invocation, "instance", None) + + code = [ + Return( + instance=instance, + expr=MakeTuple( + instance=instance, + nodes=star_args + ) + ) + ] + + new_subprogram = Subprogram( + instance=instance, + name=None, + returns_value=1, + params=[], + star=None, + dstar=None, + code=code + ) + + subprogram.module.simplifier.subnames[new_subprogram.full_name()] = new_subprogram + + invocation.stars[subprogram.full_name()] = InvokeRef( + invocation.original, + instance=instance, + produces_result=1, + ref=new_subprogram + ) + + return invocation.stars[subprogram.full_name()] + + def make_dstar_args(self, invocation, subprogram, dstar_args): + + """ + Make a subprogram which initialises a dictionary built from the given + 'dstar_args'. + """ + + if not hasattr(invocation, "dstars"): + invocation.dstars = {} + + if not invocation.dstars.has_key(subprogram.full_name()): + instance = getattr(invocation, "instance", None) + + code=[ + StoreTemp( + instance=instance, + expr=InvokeFunction( + invocation.original, + instance=instance, + expr=LoadAttr( + instance=instance, + expr=LoadRef( + instance=instance, + ref=self.builtins + ), + name="dict", + nstype="module", + ) + ) + ) + ] + + for arg, value in dstar_args: + + # NOTE: Constant not added to table. + + constant = Constant(name=repr(arg), value=arg) + code += [ + StoreTemp( + instance=instance, + expr=InvokeFunction( + instance=instance, + expr=LoadName( + instance=instance, + name=constant.typename + ) + ), + index="const" + ), + InvokeFunction( + invocation.original, + instance=instance, + expr=LoadAttr( + instance=instance, + expr=LoadTemp( + instance=instance + ), + name="__setitem__" + ), + args=[ + LoadTemp( + instance=instance, + index="const", + release=1 + ), + value + ] + ) + ] + + code += [ + Return( + instance=instance, + expr=LoadTemp( + instance=instance, + release=1 + ) + ) + ] + + new_subprogram = Subprogram( + instance=instance, + name=None, + returns_value=1, + params=[], + star=None, + dstar=None, + code=code + ) + subprogram.module.simplifier.subnames[new_subprogram.full_name()] = new_subprogram + + invocation.dstars[subprogram.full_name()] = InvokeRef( + invocation.original, + instance=instance, + produces_result=1, + ref=new_subprogram + ) + + return invocation.dstars[subprogram.full_name()] + +# Namespace-related abstractions. + +class Namespace: + + """ + A local namespace which may either relate to a genuine set of function + locals or the initialisation of a structure or module. + """ + + def __init__(self): + + """ + Initialise the namespace with a mapping of local names to possible + types, a list of return values and of possible returned local + namespaces. The namespace also tracks the "current" types and a mapping + of temporary value names to types. + """ + + self.names = {} + self.returns = set() + self.return_locals = set() + self.raises = set() + self.temp = {} + self.types = set() + + def set_types(self, types): + + "Set the current collection of 'types'." + + self.types = types.copy() + + def add(self, name, types): + + "Add to the entry with the given 'name' the specified 'types'." + + if self.names.has_key(name): + self.names[name].update(types) + else: + self.store(name, types) + + def store(self, name, types): + + "Store in (or associate with) the given 'name' the specified 'types'." + + self.names[name] = types.copy() + + __setitem__ = store + + def load(self, name): + + "Load the types associated with the given 'name'." + + return self.names[name] + + __getitem__ = load + + def has_key(self, name): + return self.names.has_key(name) + + def revoke(self, name, type): + + "Revoke from the entry for the given 'name' the specified 'type'." + + new_types = self.names[name].copy() + new_types.remove(type) + self.names[name] = new_types + + def revoke_exception_type(self, type): + + "Revoke the given 'type' from the collection of exception types." + + self.raises.remove(type) + + def revoke_temp_type(self, index, type): + + "Revoke from the temporary variable 'index' the given 'type'." + + new_types = self.temp[index][-1].copy() + new_types.remove(type) + self.temp[index][-1] = new_types + + def merge_namespace(self, namespace, everything=1): + + """ + Merge items from the given 'namespace' with this namespace. When the + optional 'everything' parameter is set to a false value (unlike the + default), return values and locals snapshots will not be copied to this + namespace. + """ + + self.merge_items(namespace.names.items()) + self.raises.update(namespace.raises) + if everything: + self.returns.update(namespace.returns) + self.return_locals.update(namespace.return_locals) + for name, values in namespace.temp.items(): + if values: + if not self.temp.has_key(name) or not self.temp[name]: + self.temp[name] = [set()] + self.temp[name][-1].update(values[-1]) + + def merge_items(self, items): + + "Merge the given 'items' with this namespace." + + for name, types in items: + self.merge(name, types) + + def merge(self, name, types): + + "Merge the entry for the given 'name' and 'types' with this namespace." + + if not self.names.has_key(name): + self.names[name] = types.copy() + else: + existing = self.names[name] + existing.update(types) + + def snapshot(self): + + "Make a snapshot of the locals and remember them." + + namespace = Namespace() + namespace.merge_namespace(self) + self.return_locals.add(namespace) + + def reset(self): + + "Reset a namespace in preparation for merging with returned locals." + + self.names = {} + + def __repr__(self): + return repr(self.names) + " (temp) " + repr(self.temp) + +class Importer: + + "An import machine, searching for and loading modules." + + def __init__(self, path=None): + + """ + Initialise the importer with the given search 'path' - a list of + directories to search for Python modules. + """ + + self.path = path or [os.getcwd()] + self.path.append(libdir) + self.modules = {} + + def find_in_path(self, name): + + """ + Find the given module 'name' in the search path, returning None where no + such module could be found, or a 2-tuple from the 'find' method + otherwise. + """ + + for d in self.path: + m = self.find(d, name) + if m: return m + return None + + def find(self, d, name): + + """ + In the directory 'd', find the given module 'name', where 'name' can + either refer to a single file module or to a package. Return None if the + 'name' cannot be associated with either a file or a package directory, + or a 2-tuple from '_find_package' or '_find_module' otherwise. + """ + + m = self._find_package(d, name) + if m: return m + m = self._find_module(d, name) + if m: return m + return None + + def _find_module(self, d, name): + + """ + In the directory 'd', find the given module 'name', returning None where + no suitable file exists in the directory, or a 2-tuple consisting of + None (indicating that no package directory is involved) and a filename + indicating the location of the module. + """ + + name_py = name + os.extsep + "py" + filename = self._find_file(d, name_py) + if filename: + return None, filename + return None + + def _find_package(self, d, name): + + """ + In the directory 'd', find the given package 'name', returning None + where no suitable package directory exists, or a 2-tuple consisting of + a directory (indicating the location of the package directory itself) + and a filename indicating the location of the __init__.py module which + declares the package's top-level contents. + """ + + filename = self._find_file(d, name) + if filename: + init_py = "__init__" + os.path.extsep + "py" + init_py_filename = self._find_file(filename, init_py) + if init_py_filename: + return filename, init_py_filename + return None + + def _find_file(self, d, filename): + + """ + Return the filename obtained when searching the directory 'd' for the + given 'filename', or None if no actual file exists for the filename. + """ + + filename = os.path.join(d, filename) + if os.path.exists(filename): + return filename + else: + return None + + def load(self, name, builtins, alias=None): + + """ + Load the module or package with the given 'name' and using the specified + 'builtins'. Return an Attribute object referencing the loaded module or + package, or None if no such module or package exists. + """ + + if self.modules.has_key(name): + return Attribute(None, self.modules[name]) + + path = name.split(".") + m = self.find_in_path(path[0]) + if not m: + return None # NOTE: Import error. + d, filename = m + + if self.modules.has_key(path[0]): + top = module = self.modules[path[0]] + else: + top = module = self.modules[path[0]] = load(filename, builtins, path[0], self, no_annotate=1) + annotate(module, builtins, self) + + if len(path) > 1: + path_so_far = path[:1] + for p in path[1:]: + path_so_far.append(p) + m = self.find(d, p) + if not m: + return None # NOTE: Import error. + d, filename = m + module_name = ".".join(path_so_far) + + if self.modules.has_key(module_name): + submodule = self.modules[module_name] + else: + submodule = self.modules[module_name] = load(filename, builtins, module_name, self, no_annotate=1) + annotate(submodule, builtins, self) + + # Store the submodule within its parent module. + + module.namespace[p] = [Attribute(None, submodule)] + module = submodule + + if alias: + return Attribute(None, module) + else: + return Attribute(None, top) + +def combine(target, additions): + + """ + Merge into the 'target' sequence the given 'additions', preventing duplicate + items. + """ + + for addition in additions: + if addition not in target: + target.append(addition) + +def find_attributes(structure, name): + + """ + Find for the given 'structure' all attributes for the given 'name', visiting + base classes where appropriate and returning the attributes in order of + descending precedence for all possible base classes. + + The elements in the result list are 2-tuples which contain the attribute and + the structure involved in accessing the attribute. + """ + + # First attempt to search the instance/class namespace. + + try: + l = structure.namespace.load(name) + attributes = [] + for attribute in l: + attributes.append((attribute, structure)) + + # If that does not work, attempt to investigate any class or base classes. + + except KeyError: + attributes = [] + + # Investigate any instance's implementing class. + + if isinstance(structure, Instance): + for attr in structure.namespace.load("__class__"): + cls = attr.type + l = get_attributes(cls, name) + combine(attributes, l) + + # Investigate any class's base classes. + + elif isinstance(structure, Class): + + # If no base classes exist, return an indicator that no attribute + # exists. + + if not structure.base_refs: + return [(None, structure)] + + # Otherwise, find all possible base classes. + + for base_refs in structure.base_refs: + base_attributes = [] + + # For each base class, find attributes either in the base + # class or its own base classes. + + for base_ref in base_refs: + l = get_attributes(base_ref, name) + combine(base_attributes, l) + + combine(attributes, base_attributes) + + return attributes + +def get_attributes(structure, name): + + """ + Return all possible attributes for the given 'structure' having the given + 'name', wrapping each attribute in an Attribute object which includes + context information for the attribute access. + + The elements in the result list are 2-tuples which contain the attribute and + the structure involved in accessing the attribute. + """ + + if isinstance(structure, Attribute): + structure = structure.type + results = [] + for attribute, accessor in find_attributes(structure, name): + + # Detect class attribute access via instances. + + if attribute is not None and isinstance(structure, Instance) and isinstance(accessor, Class): + attribute = accessor.get_attribute_for_instance(attribute, structure) + + # Produce an attribute with the appropriate context. + + if attribute is not None and isinstance(structure, Structure): + results.append((Attribute(structure, attribute.type), accessor)) + else: + results.append((attribute, accessor)) + + return results + +def prompt(vars): + try: + while 1: + s = raw_input("> ") + print eval(s, vars) + except EOFError: + pass + +# Convenience functions. + +def load(name, builtins=None, module_name=None, importer=None, no_annotate=0): + + """ + Load the module with the given 'name' (which may be a full module path), + using the optional 'builtins' to resolve built-in names, and using the + optional 'importer' to provide a means of finding and loading modules. + """ + + module = simplify.simplify(name, builtins is None, module_name) + simplify.fixnames.fix(module, builtins) + if not no_annotate: + annotate(module, builtins, importer) + return module + +def annotate(module, builtins=None, importer=None): + + """ + Annotate the given 'module', also employing the optional 'builtins' module, + if specified. If the optional 'importer' is given, use that to find and load + modules. + """ + + annotator = Annotator(importer) + if builtins is not None: + annotator.process(module, builtins) + else: + annotator.process(module) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/fixnames.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/fixnames.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,489 @@ +#!/usr/bin/env python + +""" +Fix name-related operations. The code in this module operates upon simplified +program node trees. + +Copyright (C) 2006 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +-------- + +To use this module, the easiest approach is to use the fix function: + +fix(module) + +The more complicated approach involves instantiating a Fixer object: + +fixer = Fixer() + +Then, applying the fixer to an existing module: + +fixer.process(module) + +If a module containing built-in classes and functions exists, apply the fixer as +follows: + +fixer.process(module, builtins) +""" + +from simplify.simplified import * + +# Fixing of name-related operations. + +class Fixer(Visitor): + + """ + The name fixer which traverses the program nodes in a module, typically + depth-first, and maintains a record of name usage in the different + namespaces. As a consequence of various observations, some parts of the + program node tree are modified with different operations employed to those + originally defined. + + There are two kinds of subprograms in modules: functions/methods and + internal subprograms which support things like loops. The latter kind of + subprogram may acquire the locals from their callers and must therefore be + traversed with information from such callers. Thus, we choose the top-level + code and all functions/methods as roots for processing, following + invocations of internal subprograms in order to reach all subprograms that + are defined in each module. + + top-level + ... + invoke function + ... + invoke loop -> subprogram (internal) + ... + + subprogram (function) + ... + invoke loop -> subprogram (internal) + ... + + ... + + The above approach should guarantee that all subprograms are traversed and + that all name lookups are correctly categorised. + """ + + def __init__(self): + + "Initialise the name fixer." + + Visitor.__init__(self) + + # Satisfy visitor issues. + + self.visitor = self + + def process(self, module, builtins=None): + + """ + Process the given 'module' optionally using some 'builtins' to reference + built-in objects. + """ + + # The fixer maintains a list of transformed subprograms (added for each + # of the processing "roots" and also for each invoked internal + # subprogram), along with a list of current subprograms (used to avoid + # recursion issues) and a list of current namespaces (used to recall + # namespaces upon invoking internal subprograms). + + self.subprograms = [] + self.current_subprograms = [] + self.current_namespaces = [] + + # First, process the top-level code, finding out which names are + # defined at that level. + + self.global_namespace = None + self.module = module + self.builtins = builtins or module + + self.process_node(self.module) + + # Then, process all functions and methods, providing a global namespace. + # By setting a global namespace, we influence the resolution of names: + # those which are global to the top-level module (processed above) are + # considered as built-in names, whereas those which are global to a + # function or method are searched for in the global namespace. + + self.global_namespace = self.namespace + + for subprogram in self.module.simplifier.subprograms: + + # Internal subprograms are skipped here and processed specially via + # Invoke nodes. + + if not getattr(subprogram, "internal", 0): + self.subprograms.append(self.process_node(subprogram)) + + # Ultimately, we redefine the list of subprograms on the visitor. + + self.module.simplifier.subprograms = self.subprograms + return self.module + + def process_node(self, node, namespace=None): + + """ + Process a subprogram or module 'node', discovering from attributes on + 'node' any initial locals. Return a modified subprogram or module. + """ + + # Do not process subprograms already being processed. + + if node in self.current_subprograms: + return None + + # Obtain a namespace either based on locals or on a structure. + + structure = structure=getattr(node, "structure", None) + + # If passed some namespace, use that as the current namespace. + + if namespace is not None: + self.namespace.merge_namespace(namespace) + else: + self.namespace = NameOrganiser(structure) + + # Record the current subprogram and namespace. + + self.current_subprograms.append(node) + self.current_namespaces.append(self.namespace) + + # NOTE: Avoid PEP 227 (nested scopes) whilst permitting references to a + # NOTE: subprogram within itself. Do not define the name of the function + # NOTE: within a method definition. + + if isinstance(node, Subprogram) and getattr(node, "name", None) is not None and not getattr(node, "is_method", 0): + self.namespace.store(node.name) + + # Register the names of parameters in the namespace. + + if hasattr(node, "params"): + new_params = [] + for param, default in node.params: + new_params.append((param, self.dispatch(default))) + self.namespace.store(param) + node.params = new_params + if getattr(node, "star", None): + param, default = node.star + self.namespace.store(param) + node.star = param, self.dispatch(default) + if getattr(node, "dstar", None): + param, default = node.dstar + self.namespace.store(param) + node.dstar = param, self.dispatch(default) + + # Add namespace details to any structure involved. + + if hasattr(node, "structure") and node.structure is not None: + + # Initialise bases where appropriate. + + if hasattr(node.structure, "bases"): + bases = [] + for base in node.structure.bases: + bases.append(self.dispatch(base)) + node.structure.bases = bases + + # Dispatch to the code itself. + + result = self.dispatch(node) + result.organiser = self.namespace + + # Restore the previous subprogram and namespace. + + self.current_namespaces.pop() + if self.current_namespaces: + self.namespace = self.current_namespaces[-1] + self.current_subprograms.pop() + + return result + + # Visitor methods. + + def default(self, node): + + """ + Process the given 'node', given that it does not have a specific + handler. + """ + + for attr in ("pos_args",): + value = getattr(node, attr, None) + if value is not None: + setattr(node, attr, self.dispatches(value)) + for attr in ("kw_args",): + value = getattr(node, attr, None) + if value is not None: + setattr(node, attr, self.dispatch_dict(value)) + for attr in ("expr", "lvalue", "test", "star", "dstar"): + value = getattr(node, attr, None) + if value is not None: + setattr(node, attr, self.dispatch(value)) + for attr in ("body", "else_", "handler", "finally_", "code", "choices", "nodes"): + value = getattr(node, attr, None) + if value is not None: + setattr(node, attr, self.dispatches(value)) + return node + + def dispatch(self, node, *args): + return Visitor.dispatch(self, node, *args) + + def visitGlobal(self, global_): + for name in global_.names: + self.namespace.make_global(name) + return global_ + + def visitLoadName(self, loadname): + + "Transform the 'loadname' node to a specific, scope-sensitive node." + + scope = self.namespace.find_for_load(loadname.name) + + # For structure namespaces, load an attribute. + + if scope == "structure": + result = self.dispatch( + LoadAttr(loadname.original, loadname.defining, + expr=LoadRef(loadname.original, + ref=self.namespace.structure), + name=loadname.name, + nstype="structure") + ) + + # For global accesses (ie. those outside the local namespace)... + + elif scope == "global": + + # Where a distinct global namespace exists, examine it. + + if self.global_namespace is not None: + scope = self.global_namespace.find_for_load(loadname.name) + + # Where the name is outside the global namespace, it must be a + # built-in. + + if scope == "global": + result = self.dispatch( + LoadAttr(loadname.original, loadname.defining, + expr=LoadRef(loadname.original, + ref=self.builtins), + name=loadname.name, + nstype="module") + ) + + # Otherwise, it is within the global namespace and must be a + # global. + + else: + result = self.dispatch( + LoadAttr(loadname.original, loadname.defining, + expr=LoadRef(loadname.original, + ref=self.module), + name=loadname.name, + nstype="module") + ) + + # Where no global namespace exists, we are at the module level and + # must be accessing a built-in. + + else: + result = self.dispatch( + LoadAttr(loadname.original, loadname.defining, + expr=LoadRef(loadname.original, + ref=self.builtins), + name=loadname.name, + nstype="module") + ) + + # For local accesses... + + else: + + # Where a distinct global namespace exists, it must be a local. + + if self.global_namespace is not None: + result = loadname + + # Otherwise, we must be accessing a global (which is local at the + # module level). + + else: + result = self.dispatch( + LoadAttr(loadname.original, loadname.defining, + expr=LoadRef(loadname.original, + ref=self.module), + name=loadname.name, + nstype="module") + ) + + return result + + def visitStoreName(self, storename): + + "Transform the 'storename' node to a specific, scope-sensitive node." + + scope = self.namespace.find_for_store(storename.name) + + # For structure namespaces, store an attribute. + + if scope == "structure": + self.namespace.store(storename.name) + + return self.dispatch( + StoreAttr(storename.original, storename.defining, + lvalue=LoadRef(storename.original, + ref=self.namespace.structure), + name=storename.name, + expr=storename.expr, + nstype="structure") + ) + + # Where the name is outside the local namespace, disallow any built-in + # assignment and store the name globally. + + elif scope == "global": + return self.dispatch( + StoreAttr(storename.original, storename.defining, + lvalue=LoadRef(storename.original, + ref=self.module), + name=storename.name, + expr=storename.expr, + nstype="module") + ) + + # For local namespace accesses... + + else: + self.namespace.store(storename.name) + + # If a distinct global namespace exists, it must be a local access. + + if self.global_namespace is not None: + return storename + + # Otherwise, the name is being set at the module level and is + # considered global. + + else: + return self.dispatch( + StoreAttr(storename.original, storename.defining, + lvalue=LoadRef(storename.original, + ref=self.module), + name=storename.name, + expr=storename.expr, + nstype="module") + ) + + def visitInvokeFunction(self, invoke): + + "Transform the 'invoke' node, performing processing on subprograms." + + return self.default(invoke) + + def visitInvokeRef(self, invoke): + + "Transform the 'invoke' node, performing processing on subprograms." + + # The special case of internal subprogram invocation is addressed by + # propagating namespace information to the subprogram and processing it. + + if invoke.share_locals: + subprogram = self.process_node(invoke.ref, self.namespace) + else: + subprogram = self.process_node(invoke.ref) + + if subprogram is not None: + self.subprograms.append(subprogram) + return invoke + +class ScopeMismatch(Exception): + pass + +class NameOrganiser: + + """ + A local namespace which may either relate to a genuine set of function + locals or the initialisation of a structure. + """ + + def __init__(self, structure=None): + + "Initialise the namespace with an optional 'structure'." + + self.structure = structure + if structure is not None: + self.local = "structure" + else: + self.local = "local" + + # Names may be self.local or "global". + + self.names = {} + + def make_global(self, name): + if not self.names.has_key(name): + self.names[name] = "global" + elif self.names[name] == self.local: + raise ScopeMismatch, "Name '%s' already considered as %s." % (name, self.local) + + def find_for_load(self, name): + return self.names.get(name, "global") + + def find_for_store(self, name): + return self.names.get(name, self.local) + + def store(self, name): + if self.names.get(name) != "global": + self.names[name] = self.local + else: + raise ScopeMismatch, "Name '%s' already considered as global." % name + + def merge(self, name, scope): + if self.names.get(name) in (None, scope): + self.names[name] = scope + else: + raise ScopeMismatch, "Name '%s' already considered as %s." % (name, self.names[name]) + + def merge_namespace(self, namespace): + self.merge_items(namespace.names.items()) + + def merge_items(self, items): + for name, scope in items: + self.merge(name, scope) + + def __repr__(self): + return repr(self.names) + +# Convenience functions. + +def fix(module, builtins=None): + + """ + Fix the names in the given 'module', also employing the optional 'builtins' + module, if specified. + """ + + fixer = Fixer() + if builtins is not None: + fixer.process(module, builtins) + else: + fixer.process(module) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/simplified/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/simplified/__init__.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +""" +Simplified program representation. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +""" + +__version__ = "0.1" + +from simplified.ast import * +from simplified.data import * +from simplified.program import * +from simplified.utils import * +import os + +# Location of the built-in libraries. +# NOTE: Change this if the package structure changes. + +libdir = os.path.join(os.path.split(os.path.split(__file__)[0])[0], "lib") + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/simplified/ast.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/simplified/ast.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +""" +Additional program AST nodes. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +""" + +class Self: + + """ + A program node encapsulating object/context information in an argument list. + This is not particularly like Attribute, Class, Instance or other such + things, since it actually appears in the program representation. + """ + + def __init__(self, attribute): + self.types = set() + self.types.add(attribute) + +class Op: + + "A replacement AST node representing an operation in a Compare construct." + + def __init__(self, name, expr): + self.name = name + self.expr = expr + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/simplified/data.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/simplified/data.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,237 @@ +#!/usr/bin/env python + +""" +Simplified program data. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +""" + +from simplified.utils import Structure, WithName, name + +# Special non-program nodes. + +class _Class(Structure, WithName): + + "A Python class." + + def __init__(self, *args, **kw): + Structure.__init__(self, *args, **kw) + WithName.__init__(self) + + def full_name(self): + return "class %s" % self._full_name + +class SingleInstanceClass(_Class): + + "A Python class." + + def __init__(self, *args, **kw): + _Class.__init__(self, *args, **kw) + self.instance = None + + def has_instance(self, node): + return self.instance is not None + + def add_instance(self, node, instance): + self.instance = instance + + def get_instance(self, node): + return self.instance + + def get_instance_name(self, instance): + return self._full_name + + # Attribute propagation. + + def get_attribute_for_instance(self, attribute, instance): + return attribute + +class MultipleInstanceClass(_Class): + + "A Python class." + + def __init__(self, *args, **kw): + _Class.__init__(self, *args, **kw) + self.instances = {} + self.attributes_for_instances = {} + + def _get_key(self, node): + return id(getattr(node, "original", None)) # self.module.original + + def has_instance(self, node): + return self.instances.has_key(self._get_key(node)) + + def add_instance(self, node, instance): + self.instances[self._get_key(node)] = instance + + def get_instance(self, node): + return self.instances[self._get_key(node)] + + def get_instance_name(self, instance): + return name(instance, self._full_name) + + # Attribute propagation. + + def get_attribute_for_instance(self, attribute, instance): + + # Create specialised methods. + + if isinstance(attribute.type, Subprogram): + subprogram = attribute.type + + # Each instance may have its own version of the subprogram. + + key = (subprogram, instance) + if not self.attributes_for_instances.has_key(key): + new_subprogram = subprogram.copy(instance, subprogram.full_name()) + subprogram.module.simplifier.subnames[new_subprogram.full_name()] = new_subprogram + self.attributes_for_instances[key] = Attribute(attribute.context, new_subprogram) + print "New subprogram", new_subprogram, "for", key + + return self.attributes_for_instances[key] + + # The original nodes are returned for other attributes. + + else: + return attribute + +class SelectiveMultipleInstanceClass(MultipleInstanceClass): + + "A Python class which provides multiple instances depending on the class." + + def _get_key(self, node): + if self.namespace.has_key("__atomic__"): + return id(self) + else: + return MultipleInstanceClass._get_key(self, node) + +class ProlificMultipleInstanceClass(MultipleInstanceClass): + + """ + A Python class which provides multiple instances for different versions of + methods. In order to avoid unbounded instance production (since new + instances cause new copies of methods which in turn would cause new + instances), a relations dictionary is maintained which attempts to map + "requesting instances" to existing instances, suggesting such instances in + preference to new ones. + """ + + def __init__(self, *args, **kw): + MultipleInstanceClass.__init__(self, *args, **kw) + self.instance_relations = {} + + def _get_key(self, node): + if self.namespace.has_key("__atomic__"): + return id(self) + else: + return id(node) + + def has_instance(self, node): + requesting_instance = getattr(node, "instance", None) + #return requesting_instance is not None and requesting_instance.get_class() is self or \ + return self.instance_relations.has_key(requesting_instance) or self.instances.has_key(self._get_key(node)) + + def add_instance(self, node, instance): + requesting_instance = getattr(node, "instance", None) + print "New instance", instance, "for", id(node), requesting_instance + self.instances[self._get_key(node)] = instance + if requesting_instance is not None: + self.instance_relations[requesting_instance] = instance + requesting_instance.get_class().instance_relations[instance] = requesting_instance + + def get_instance(self, node): + requesting_instance = getattr(node, "instance", None) + #if requesting_instance is not None and requesting_instance.get_class() is self: + # return requesting_instance + return self.instance_relations.get(requesting_instance) or self.instances[self._get_key(node)] + +class Instance(Structure): + + "An instance." + + def full_name(self): + return self.get_class().get_instance_name(self) + + def get_class(self): + for n in self.namespace.load("__class__"): + return n.type + else: + raise ValueError, "__class__" + + def __repr__(self): + return "Instance of type '%s'" % self.full_name() + + def __eq__(self, other): + # NOTE: Single instance: all instances are the same + # NOTE: Multiple instances: all instances are different + return self.full_name() == other.full_name() + + def __hash__(self): + return id(self) + +class Constant: + + "A constant initialised with a type name for future processing." + + def __init__(self, name, value): + self.name = name + self.value = value + self.typename = self.value.__class__.__name__ + +class Attribute: + + """ + An attribute abstraction, indicating the type of the attribute along with + its context or origin. + """ + + def __init__(self, context, type): + self.context = context + self.type = type + + def __eq__(self, other): + return hasattr(other, "type") and other.type == self.type or other == self.type + + def __repr__(self): + return "Attribute(%s, %s)" % (repr(self.context), repr(self.type)) + + def __hash__(self): + return id(self.type) + +# Configuration setting. + +Class = SingleInstanceClass +#Class = MultipleInstanceClass + +def set_single_instance_mode(): + global Class + Class = SingleInstanceClass + +def set_multiple_instance_mode(): + global Class + Class = MultipleInstanceClass + +def set_selective_multiple_instance_mode(): + global Class + Class = SelectiveMultipleInstanceClass + +def set_prolific_multiple_instance_mode(): + global Class + Class = ProlificMultipleInstanceClass + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/simplified/program.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/simplified/program.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,412 @@ +#!/usr/bin/env python + +""" +Simplified program nodes for easier type propagation and analysis. This module +contains nodes representing program instructions or operations, program +structure or organisation, and abstract program data. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +""" + +from simplified.utils import Structure, WithName, name +import sys + +# Simplified program nodes. + +class Node: + + """ + A result node with common attributes: + + original The original node from which this node was created. + defining Whether the node defines something in the original program. + name Any name involved (variable or attribute). + index Any index involved (temporary variable name). + value Any constant value. + ref Any reference to (for example) subprograms. + nstype Any indication of the namespace type involved in a name access. + + Expression-related attributes: + + expr Any contributing expression. + lvalue Any target expression. + test Any test expression in a conditional instruction. + + Invocation and subprogram attributes: + + args Any collection of argument nodes. + params Any collection of parameter nodes and defaults. + + Tuple construction attributes: + + nodes Any expressions used to initialise a tuple + + Statement-grouping attributes: + + body Any conditional code depending on the success of a test. + else_ Any conditional code depending on the failure of a test. + handler Any exception handler code. + finally_ Any code which will be executed regardless. + code Any unconditional code. + choices Any choices which may be included in the final program. + """ + + common_attributes = "name", "index", "value", "nstype", "internal", "returns_value", "is_method", "ref", "module", "structures", "original" + expression_attributes = "expr", "lvalue", "test" + argument_attributes = "star", "dstar" + invocation_attributes = "params", # not "args" - see "pos_args", "kw_args" + grouping_attributes = "code", "body", "else_", "handler", "finally_", "choices" + + def __init__(self, original=None, defining=0, **kw): + + """ + Initialise a program node with a link to an optional 'original' AST + node. An optional 'defining' parameter (if set to a true value), sets + this node as the defining node in the original. + """ + + self.original = original + self.defining = defining + self.copies = {} + + if self.original is not None and defining: + self.original._node = self + for name, value in kw.items(): + setattr(self, name, value) + + # Annotations. + + self.types = set() + + def __repr__(self): + + "Return a readable representation." + + if hasattr(self, "full_name"): + s = "%s '%s'" % (self.__class__.__name__, self.full_name()) + elif hasattr(self, "name"): + s = "%s '%s'" % (self.__class__.__name__, self.name) + elif hasattr(self, "index"): + s = "%s (%s)" % (self.__class__.__name__, self.index) + elif hasattr(self, "value"): + s = "%s %s" % (self.__class__.__name__, repr(self.value)) + elif hasattr(self, "ref"): + s = "%s '%s'" % (self.__class__.__name__, name(self.ref, self.ref.name)) + else: + s = "%s" % (self.__class__.__name__,) + + # Annotations. + + if self.types: + return "%s -> %s" % (s, self.types) + else: + return s + + def _pprint(self, indent, continuation, s, stream=None): + + """ + Print, at the given 'indent' level, with the given 'continuation' text, + the string 's', either to the given, optional 'stream' or to standard + output, this node's "pretty" representation. + """ + + stream = stream or sys.stdout + if continuation: + print >>stream, (" " * max(0, indent - len(continuation))) + continuation + s + else: + print >>stream, (" " * indent) + s + + def pprint(self, indent=0, continuation=None, stream=None): + + """ + Print, at the given, optional 'indent', with the given optional + 'continuation' text, either to the given, optional 'stream' or to + standard output, this node's "pretty" representation along with its + children and their "pretty" representation (and so on). + """ + + stream = stream or sys.stdout + self._pprint(indent, continuation, repr(self), stream) + + # Subprogram-related details. + + if hasattr(self, "params"): + for name, default in self.params: + self._pprint(indent + 2, "( ", "%s default %s" % (name, default), stream=stream) + if hasattr(self, "star") and self.star: + name, default = self.star + self._pprint(indent + 2, "( ", "%s default %s" % (name, default), stream=stream) + if hasattr(self, "dstar") and self.dstar: + name, default = self.dstar + self._pprint(indent + 2, "( ", "%s default %s" % (name, default), stream=stream) + if getattr(self, "internal", 0): + self._pprint(indent + 2, "( ", "internal", stream=stream) + if getattr(self, "structure", 0): + self._pprint(indent + 2, "( ", "structure '%s'" % self.structure.name, stream=stream) + + # Expression-related details. + + if hasattr(self, "expr"): + self.expr.pprint(indent + 2, "- ", stream=stream) + if hasattr(self, "nodes"): + for node in self.nodes: + node.pprint(indent + 2, "- ", stream=stream) + if hasattr(self, "lvalue"): + self.lvalue.pprint(indent + 2, "->", stream=stream) + if hasattr(self, "nstype"): + self._pprint(indent + 2, "", self.nstype, stream=stream) + if hasattr(self, "args"): + for arg in self.pos_args: + arg.pprint(indent + 2, "( ", stream=stream) + for name, arg in self.kw_args.items(): + arg.pprint(indent + 2, "( ", stream=stream) + if hasattr(self, "star") and self.star: + self.star.pprint(indent + 2, "( ", stream=stream) + if hasattr(self, "dstar") and self.dstar: + self.dstar.pprint(indent + 2, "( ", stream=stream) + + # Statement-related details. + + if hasattr(self, "test"): + self.test.pprint(indent + 2, "? ", stream=stream) + for attr in self.grouping_attributes: + if hasattr(self, attr) and getattr(self, attr): + self._pprint(indent, "", "%s {" % attr, stream=stream) + for node in getattr(self, attr): + node.pprint(indent + 2, stream=stream) + self._pprint(indent, "", "}", stream=stream) + + # Annotations. + + if hasattr(self, "accesses"): + self._pprint(indent, "", "--------", stream=stream) + for ref, attributes in self.accesses.items(): + self._pprint(indent + 2, "| ", "when %s: %s" % (ref, ", ".join([("%s via %s" % attr_acc) for attr_acc in attributes])), stream=stream) + self._pprint(indent, "", "--------", stream=stream) + if hasattr(self, "writes"): + self._pprint(indent, "", "--------", stream=stream) + for ref, attribute in self.writes.items(): + self._pprint(indent + 2, "| ", "when %s: %s" % (ref, attribute), stream=stream) + self._pprint(indent, "", "--------", stream=stream) + + # Node discovery functions. + + def active(self): + + "Return the active copies of this node or a list containing this node." + + return self.copies.values() or [self] + + # Node manipulation functions. + + def copy(self, instance=None, new_name=None): + + """ + Perform a deep copy of the node, optionally specifying the 'instance' + for whom the copy has been requested and a 'new_name' for the copied + node. Return new unannotated copies of the node and its descendants. + """ + + # Copy the common attributes of this node. + + common = {} + for attr in self.common_attributes: + if hasattr(self, attr): + common[attr] = getattr(self, attr) + + # Add new attributes specially for copies. + + common["instance"] = instance + + if new_name is not None: + common["copy_of"] = self + common["name"] = new_name + + # Instantiate the copy, avoiding side-effects with original and defining. + + node = self.__class__(**common) + node.defining = self.defining + + # Add links to copies from originals. + + self.copies[instance] = node + + # Copy attributes of different types. + + for attr in self.expression_attributes: + if hasattr(self, attr): + n = getattr(self, attr) + if n is None: + n2 = n + else: + n2 = n.copy(instance) + setattr(node, attr, n2) + + for attr in self.argument_attributes: + if hasattr(self, attr): + t = getattr(self, attr) + if t is None: + t2 = t + else: + name, n = t + n2 = n.copy(instance) + t2 = name, n2 + setattr(node, attr, t2) + + for attr in self.invocation_attributes: + if hasattr(self, attr): + l = getattr(self, attr) + l2 = [] + for name, n in l: + if n is None: + l2.append((name, n)) + else: + l2.append((name, n.copy(instance))) + setattr(node, attr, l2) + + for attr in self.grouping_attributes: + if hasattr(self, attr): + l = getattr(self, attr) + setattr(node, attr, [n.copy(instance) for n in l]) + + # Arguments are usually processed further - "args" is useless. + + if hasattr(self, "pos_args"): + node.pos_args = [n.copy(instance) for n in self.pos_args] + + if hasattr(self, "kw_args"): + node.kw_args = {} + for name, n in self.kw_args.items(): + node.kw_args[name] = n.copy(instance) + + return node + +# These are the supported "operations" described by simplified program nodes. + +class Pass(Node): "A placeholder node corresponding to pass." +class Assign(Node): "A grouping node for assignment-related operations." +class Keyword(Node): "A grouping node for keyword arguments." +class Global(Node): "A global name designator." +class Import(Node): "A module import operation." +class LoadTemp(Node): "Load a previously-stored temporary value." +class LoadName(Node): "Load a named object." +class LoadAttr(Node): "Load an object attribute." +class LoadRef(Node): "Load a reference, typically a subprogram or a constant." +class LoadExc(Node): "Load a handled exception." +class ResetExc(Node): "Reset the exception state." +class StoreTemp(Node): "Store a temporary value." +class StoreName(Node): "Associate a name with an object." +class StoreAttr(Node): "Associate an object's attribute with a value." +class ReleaseTemp(Node): "Release a temporary value." +class Try(Node): "A try...except...else...finally grouping node." +class Raise(Node): "An exception raising node." +class Not(Node): "A negation of an expression." +class CheckType(Node): "Check a value's type from a list of choices." +class Return(Node): "Return an evaluated expression." +class Invoke(Node): "An invocation." +class MakeTuple(Node): "Make a tuple object." + +# There are two types of return node: return from function and return from +# block. + +class ReturnFromFunction(Return): + pass + +class ReturnFromBlock(Return): + pass + +# NOTE: Not actually supported. +# Additionally, yield statements act like return statements for the purposes +# of this system. + +class Yield(ReturnFromFunction): + pass + +# Some behaviour is set as the default in conditional nodes but may be +# overridden. + +class Conditional(Node): + + "A conditional node consisting of a test and outcomes." + + def __init__(self, *args, **kw): + self.isolate_test = 0 + Node.__init__(self, *args, **kw) + +# Invocations involve some more work to process calculated attributes. + +class InvokeFunction(Invoke): + + "A function or method invocation." + + def __init__(self, *args, **kw): + self.args = [] + self.star = None + self.dstar = None + Invoke.__init__(self, *args, **kw) + self.set_args(self.args) + self.share_locals = 0 + + def set_args(self, args): + + "Sort the 'args' into positional and keyword arguments." + + self.pos_args = [] + self.kw_args = {} + add_kw = 0 + for arg in args: + if not add_kw: + if not isinstance(arg, Keyword): + self.pos_args.append(arg) + else: + add_kw = 1 + if add_kw: + if isinstance(arg, Keyword): + self.kw_args[arg.name] = arg.expr + else: + raise TypeError, "Positional argument appears after keyword arguments in '%s'." % self + +class InvokeRef(Invoke): + + "A block or loop invocation." + + def __init__(self, *args, **kw): + self.share_locals = 1 + Invoke.__init__(self, *args, **kw) + +# Program structure nodes. + +class Module(Node, Structure): + + "A Python module." + + def full_name(self): + return "module %s" % self.name + +class Subprogram(Node, WithName): + + "A subprogram: functions, methods and loops." + + def __init__(self, *args, **kw): + Node.__init__(self, *args, **kw) + WithName.__init__(self) + self.raises = set() + self.returns = set() + self.return_locals = set() + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/simplified/utils.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/simplified/utils.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,167 @@ +#!/usr/bin/env python + +""" +Simplified program utilities. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +""" + +from compiler.visitor import ASTVisitor + +# Exceptions. + +class SimplifiedError(Exception): + + "An error in the annotation process." + + def __init__(self, exc, node, *args): + + """ + Initialise the error with an existing exception 'exc', the 'node' at + which this error occurs, along with additional optional arguments. + """ + + Exception.__init__(self, *args) + self.nodes = [node] + self.exc = exc + + def add(self, node): + + "Add the given 'node' to the path of nodes leading from the exception." + + self.nodes.append(node) + + def __str__(self): + + "Return a string showing the principal exception details." + + return "%s, %s" % (self.exc, self.nodes) + +# Elementary visitor support. + +class Visitor(ASTVisitor): + + "A visitor base class." + + def __init__(self): + ASTVisitor.__init__(self) + + def default(self, node, *args): + raise SimplifiedError, (None, node) + + def dispatch(self, node, *args): + return ASTVisitor.dispatch(self, node, *args) + + def dispatches(self, nodes, *args): + results = [] + for node in nodes: + results.append(self.dispatch(node, *args)) + return results + + def dispatch_dict(self, d, *args): + results = {} + for name, node in d.items(): + results[name] = self.dispatch(node, *args) + return results + +# Unique name registration. + +class Naming: + + "Maintain records of unique names for each simple name." + + index_separator = "-" + + def __init__(self): + self.names = {} + + def get(self, obj): + return obj._unique_name + + def set(self, obj, name): + if hasattr(obj, "_unique_name"): + return + if not self.names.has_key(name): + self.names[name] = 0 + n = self.names[name] + 1 + self.names[name] = n + obj._unique_name = "%s%s%d" % (name, self.index_separator, n) + +def name(obj, name): + + "Return a unique name for the given 'obj', indicating the base 'name'." + + naming.set(obj, name) + return naming.get(obj) + +# Naming singleton. + +naming = Naming() + +# Named nodes are those which can be referenced in some way. + +class WithName: + + "Node naming." + + def __init__(self): + + "Initialise the object's full name." + + self._full_name = name(self, self.name or "$untitled") + + def full_name(self): + + "Return the object's full name." + + return self._full_name + +# Comparable nodes based on naming. + +class Comparable: + + "Comparable nodes implementing the 'full_name' method." + + def __eq__(self, other): + + "This object is equal to 'other' if the full names are the same." + + # NOTE: Single instance: all instances are the same + # NOTE: Multiple instances: all instances are different + if hasattr(other, "full_name"): + return self.full_name() == other.full_name() + else: + return NotImplemented + + def __hash__(self): + + "The hash of this object is based on its full name." + + return hash(self.full_name()) + +# Structure nodes indicating namespace-bearing objects. + +class Structure(Comparable): + + "A non-program node containing some kind of namespace." + + def __init__(self, **kw): + for name, value in kw.items(): + setattr(self, name, value) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 simplify/viewer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/simplify/viewer.py Sun May 27 18:25:25 2007 +0200 @@ -0,0 +1,1171 @@ +#!/usr/bin/env python + +""" +View annotated sources. + +Copyright (C) 2006, 2007 Paul Boddie + +This software is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of +the License, or (at your option) any later version. + +This software is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public +License along with this library; see the file LICENCE.txt +If not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +""" + +from compiler.visitor import ASTVisitor +from simplify.simplified import * +import sys +import os +import textwrap + +# Classes. + +# HTML-related output production. + +html_header = """ + + + + Module + + + +""" + +html_footer = """ + +""" + +# Browser classes. + +class Browser(ASTVisitor): + + """ + A browsing visitor for AST nodes. + + Covered: Add, And, Assert, AssAttr, AssList, AssName, AssTuple, Assign, + AugAssign, Bitand, Break, CallFunc, Class, Compare, Const, + Continue, Dict, Discard, Div, FloorDiv, For, From, Function, + Getattr, Global, If, Import, Keyword, Lambda, List, ListComp, + ListCompFor, ListCompIf, Mod, Module, Mul, Name, Not, Or, Pass, + Power, Print, Printnl, Raise, Return, Slice, Sliceobj, Stmt, Sub, + Subscript, TryExcept, TryFinally, Tuple, UnaryAdd, UnarySub, While. + + Missing: Backquote, Bitor, Bitxor, Decorators, Ellipsis, + Exec, Invert, LeftShift, RightShift, Yield. + """ + + def __init__(self, stream): + ASTVisitor.__init__(self) + self.visitor = self + self.stream = stream + + def process(self, module): + self.stream.write(html_header) + self.dispatch(module) + self.stream.write(html_footer) + + def visitModule(self, node): + self.default(node) + + # Statements. + + def visitAssert(self, node): + self.stream.write("
\n") + self.stream.write("\n") + self._keyword("assert") + self._popup( + self._types(node._raises.active()) + ) + self.stream.write("\n") + self.dispatch(node.test) + if node.fail: + self.stream.write(", ") + self.dispatch(node.fail) + self.stream.write("
\n") + + def visitAssign(self, node): + self.stream.write("
\n") + for lvalue in node.nodes: + self.dispatch(lvalue) + self.stream.write("=\n") + self.dispatch(node.expr) + self.stream.write("
\n") + + def visitAugAssign(self, node): + self.stream.write("
\n") + self.dispatch(node.node) + self.stream.write("\n") + self.stream.write("%s\n" % node.op) + self._popup( + self._invocations(node._op_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.expr) + self.stream.write("
\n") + + def visitBreak(self, node): + self.stream.write("
\n") + self._keyword("break") + self.stream.write("
\n") + + def visitClass(self, node): + definition = node._node + definitions = definition.active() + structure = definition.expr.ref + self.stream.write("
\n" % structure.full_name()) + self.stream.write("
\n") + self._keyword("class") + self._name_start(structure.name, "class-name") + self._popup( + self._scopes(definitions) + ) + self._name_end() + bases = structure.bases + + # Suppress the "object" class appearing alone. + + if bases and not (len(bases) == 1 and bases[0].name == "object"): + self.stream.write("(") + first = 1 + for base in bases: + if not first: + self.stream.write(",\n") + self._name_start(base.name) + self._popup( + self._scopes([base]) + + self._types([base]) + ) + self._name_end() + first = 0 + self.stream.write(")") + + self.stream.write(":\n") + self._comment(self._text(structure.full_name())) + self.stream.write("
\n") + + self.stream.write("
\n") + self._doc(node) + self.dispatch(node.code) + self.stream.write("
\n") + self.stream.write("
\n") + + def visitContinue(self, node): + self.stream.write("
\n") + self._keyword("continue") + self.stream.write("
\n") + + def visitDiscard(self, node): + self.stream.write("
\n") + self.default(node) + self.stream.write("
\n") + + def visitFor(self, node): + self.stream.write("
\n") + self.stream.write("
\n") + self.stream.write("\n") + self._keyword("for") + self._popup( + self._invocations(node._next_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.assign) + self.stream.write("\n") + self._keyword("in") + self._popup( + self._invocations(node._iter_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.list) + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.body) + self.stream.write("
\n") + if node.else_ is not None: + self.stream.write("
\n") + self._keyword("else") + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.else_) + self.stream.write("
\n") + self.stream.write("
\n") + + def visitFrom(self, node): + self.stream.write("
\n") + self._keyword("from") + self.stream.write("\n") + self.stream.write(node.modname) + self._popup( + self._types(node._modname.active()) + ) + self.stream.write("\n") + self._keyword("import") + first = 1 + for (name, alias), _name in map(None, node.names, node._names): + if not first: + self.stream.write(",\n") + if alias: + self.stream.write(name + " ") + self._keyword("as") + self.stream.write("\n") + self.stream.write(alias or name) + self._popup( + self._types([_name]) + ) + self.stream.write("\n") + first = 0 + self.stream.write("
\n") + + def visitFunction(self, node): + definition = node._node + definitions = [n for n in definition.active() if not isinstance(n, Subprogram)] + subprogram = node._subprogram + subprograms = subprogram.active() + self.stream.write("
\n" % subprogram.full_name()) + self.stream.write("
\n") + self._keyword("def") + self._name_start(subprogram.name, "function-name") + self._popup( + self._scopes([definition]) + # not dependent on subprograms + self._raises(subprograms) + ) + self._name_end() + self.stream.write("(") + self._parameters(subprogram, subprograms) + self.stream.write(")") + self.stream.write(":\n") + self._comment(self._text(subprogram.full_name())) + self.stream.write("
\n") + + self.stream.write("
\n") + self._doc(node) + self.dispatch(node.code) + self.stream.write("
\n") + self.stream.write("
\n") + + def visitGlobal(self, node): + self.stream.write("
\n") + self._keyword("global") + first = 1 + for name in node.names: + if not first: + self.stream.write(",\n") + self.stream.write(name) + first = 0 + self.stream.write("
\n") + + def visitIf(self, node): + self.stream.write("
\n") + first = 1 + conditional = node._node + conditionals = conditional.active() + for compare, stmt in node.tests: + self.stream.write("
\n") + self.stream.write("\n") + if first: + self._keyword("if") + else: + self._keyword("elif") + self._popup( + self._invocations([c.test for c in conditionals]) + ) + self.stream.write("\n") + self.dispatch(compare) + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(stmt) + self.stream.write("
\n") + if conditional.else_: + conditional = conditional.else_[0] + conditionals = conditional.active() + else: + conditional = None + conditionals = [] + first = 0 + if node.else_ is not None: + self.stream.write("
\n") + self._keyword("else") + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.else_) + self.stream.write("
\n") + self.stream.write("
\n") + + def visitImport(self, node): + self.stream.write("
\n") + self._keyword("import") + first = 1 + for (name, alias), _name in map(None, node.names, node._names): + if not first: + self.stream.write(",\n") + if alias: + self.stream.write(name + " ") + self._keyword("as") + self.stream.write("\n") + self.stream.write(alias or name) + self._popup( + self._types([_name]) + ) + self.stream.write("\n") + first = 0 + self.stream.write("
\n") + + def visitPass(self, node): + self.stream.write("
\n") + self._keyword("pass") + self.stream.write("
\n") + + def visitPrint(self, node): + self.stream.write("
\n") + self._keyword("print") + if node.dest is not None: + self.stream.write(">>\n") + self.dispatch(node.dest) + for n in node.nodes: + self.dispatch(n) + self.stream.write(",\n") + self.stream.write("
\n") + + def visitPrintnl(self, node): + self.stream.write("
\n") + self._keyword("print") + if node.dest is not None: + self.stream.write(">>\n") + self.dispatch(node.dest) + first = 1 + for n in node.nodes: + if not first: + self.stream.write(",\n") + self.dispatch(n) + first = 0 + self.stream.write("
\n") + + def visitRaise(self, node): + target = node._node.expr + targets = target.active() + self.stream.write("
\n") + self.stream.write("\n") + self._keyword("raise") + self._popup( + self._invocations(targets) + ) + self.stream.write("\n") + self.dispatch(node.expr1) + if node.expr2 is not None: + self.stream.write(",\n") + self.dispatch(node.expr2) + if node.expr3 is not None: + self.stream.write(",\n") + self.dispatch(node.expr3) + self.stream.write("
\n") + + def visitReturn(self, node): + value = node._node + values = value.active() + self.stream.write("
\n") + self.stream.write("\n") + self._keyword("return") + self._popup( + self._types(values) + ) + self.stream.write("\n") + self.dispatch(node.value) + self.stream.write("
\n") + + def visitStmt(self, node): + self.stream.write("
\n") + self.default(node) + self.stream.write("
\n") + + def visitTryExcept(self, node): + self.stream.write("
\n") + self.stream.write("
\n") + self._keyword("try") + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.body) + self.stream.write("
\n") + for spec, assign, statement in node.handlers: + self.stream.write("
\n") + self._keyword("except") + if spec is not None: + self.dispatch(spec) + if assign is not None: + self.stream.write(",\n") + self.dispatch(assign) + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(statement) + self.stream.write("
\n") + if node.else_ is not None: + self.stream.write("
\n") + self._keyword("else") + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.else_) + self.stream.write("
\n") + self.stream.write("
\n") + + def visitTryFinally(self, node): + self.stream.write("
\n") + self.stream.write("
\n") + self._keyword("try") + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.body) + self.stream.write("
\n") + self.stream.write("
\n") + self._keyword("finally") + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.final) + self.stream.write("
\n") + self.stream.write("
\n") + + def visitWhile(self, node): + self.stream.write("
\n") + self.stream.write("
\n") + self.stream.write("\n") + self._keyword("while") + self._popup( + self._invocations(node._test_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.test) + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.body) + self.stream.write("
\n") + if node.else_ is not None: + self.stream.write("
\n") + self._keyword("else") + self.stream.write(":\n") + self.stream.write("
\n") + self.stream.write("
\n") + self.dispatch(node.else_) + self.stream.write("
\n") + self.stream.write("
\n") + + # Expression-related helper methods. + + def _visitBinary(self, node, name, symbol): + self.stream.write("\n" % name) + self.dispatch(node.left) + self.stream.write("\n") + self.stream.write(self._text(symbol)) + self._popup( + self._invocations(node._left_call.active() + node._right_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.right) + self.stream.write("") + + def _visitUnary(self, node, name, symbol): + self.stream.write("\n" % name) + self.stream.write("\n") + self.stream.write(symbol) + self._popup( + self._invocations(node._unary_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.expr) + self.stream.write("") + + # Expressions. + + def visitAdd(self, node): + self._visitBinary(node, "add", "+") + + def visitAnd(self, node): + self.stream.write("\n") + first = 1 + for n in node.nodes: + if not first: + self._keyword("and") + self.dispatch(n) + first = 0 + self.stream.write("") + + def visitAssAttr(self, node): + target = node._node + targets = target.active() + self.stream.write("\n") + self.dispatch(node.expr) + self.stream.write("\n") + self.stream.write(".%s\n" % self._text(node.attrname)) + self._popup( + self._scopes(targets) + + self._types(targets) + ) + self.stream.write("\n") + self.stream.write("\n") + + def visitAssList(self, node): + self.stream.write("\n") + self.stream.write("[") + self._sequence(node) + self.stream.write("]\n") + self.stream.write("\n") + + def visitAssName(self, node): + target = node._node + targets = target.active() + self._name_start(target.name) + self._popup( + self._scopes(targets) + + self._types(targets) + ) + self._name_end() + + def visitAssTuple(self, node): + self.stream.write("\n") + self.stream.write("(") + self._sequence(node) + self.stream.write(")\n") + self.stream.write("\n") + + def visitBitand(self, node): + self.stream.write("\n") + self.dispatch(node.nodes[0]) + for op in node._ops: + self.stream.write("\n") + self.stream.write(self._text(op.name)) + self._popup( + self._op(op) + ) + self.stream.write("\n") + self.dispatch(op.expr) + self.stream.write("") + + def visitCallFunc(self, node): + target = node._node + targets = target.active() + self.stream.write("\n") + self.dispatch(node.node) + self.stream.write("\n") + self.stream.write("(") + self._popup( + self._invocations(targets) + ) + self.stream.write("\n") + first = 1 + for arg in node.args: + if not first: + self.stream.write(",\n") + self.dispatch(arg) + first = 0 + if node.star_args is not None: + if not first: + self.stream.write(", *\n") + self.dispatch(node.star_args) + first = 0 + if node.dstar_args is not None: + if not first: + self.stream.write(", **\n") + self.dispatch(node.dstar_args) + first = 0 + self.stream.write(")\n") + self.stream.write("\n") + + def visitCompare(self, node): + self.stream.write("\n") + self.dispatch(node.expr) + for op in node._ops: + self.stream.write("\n") + self.stream.write(self._text(op.name)) + self._popup( + self._op(op) + ) + self.stream.write("\n") + self.dispatch(op.expr) + self.stream.write("\n") + + def visitConst(self, node): + if isinstance(node.value, (str, unicode)): + self.stream.write("\n") + self.stream.write(repr(node.value)) + if isinstance(node.value, (str, unicode)): + self.stream.write("\n") + + def visitDict(self, node): + self.stream.write("\n") + self.stream.write("{") + self._mapping(node) + self.stream.write("}\n") + self.stream.write("\n") + + def visitDiv(self, node): + self._visitBinary(node, "div", "/") + + def visitFloorDiv(self, node): + self._visitBinary(node, "floordiv", "//") + + def visitGetattr(self, node): + target = node._node + targets = target.active() + self.stream.write("\n") + self.dispatch(node.expr) + self.stream.write("\n") + self.stream.write(".%s\n" % self._text(node.attrname)) + self._popup( + self._scopes(targets) + + self._types(targets) + ) + self.stream.write("\n") + self.stream.write("\n") + + def visitKeyword(self, node): + self.stream.write("\n") + self.stream.write(node.name) + self.stream.write("=") + self.dispatch(node.expr) + self.stream.write("\n") + + def visitLambda(self, node): + definition = node._node + definitions = [n for n in definition.active() if not isinstance(n, Subprogram)] + subprogram = node._subprogram + subprograms = subprogram.active() + self.stream.write("\n") + self._keyword("lambda") + self._parameters(subprogram, subprograms) + self.dispatch(node.code) + self.stream.write("\n") + + visitList = visitAssList + + def visitListComp(self, node): + self.stream.write("\n") + self.stream.write("[") + self.dispatch(node.expr) + for qual in node.quals: + self.dispatch(qual) + self.stream.write("]\n") + self.stream.write("\n") + + def visitListCompFor(self, node): + self.stream.write("\n") + self.stream.write("\n") + self._keyword("for") + self._popup( + self._invocations(node._next_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.assign) + self.stream.write("\n") + self._keyword("in") + self._popup( + self._invocations(node._iter_call.active()) + ) + self.stream.write("\n") + self.dispatch(node.list) + for if_ in node.ifs: + self.dispatch(if_) + self.stream.write("\n") + + def visitListCompIf(self, node): + conditional = node._node + conditionals = conditional.active() + self.stream.write("\n") + self.stream.write("\n") + self._keyword("if") + self._popup( + self._invocations([c.test for c in conditionals]) + ) + self.stream.write("\n") + self.dispatch(node.test) + self.stream.write("\n") + + def visitMod(self, node): + self._visitBinary(node, "mod", "%") + + def visitMul(self, node): + self._visitBinary(node, "mul", "*") + + def visitName(self, node): + target = node._node + targets = target.active() + self._name_start(target.name) + self._popup( + self._scopes(targets) + + self._types(targets) + ) + self._name_end() + + def visitNot(self, node): + self.stream.write("\n") + self._keyword("not") + self.dispatch(node.expr) + self.stream.write("") + + def visitOr(self, node): + self.stream.write("\n") + first = 1 + for n in node.nodes: + if not first: + self._keyword("or") + self.dispatch(n) + first = 0 + self.stream.write("") + + def visitPower(self, node): + self._visitBinary(node, "power", "**") + + def visitSlice(self, node): + target = node._node + targets = target.active() + self.stream.write("\n") + self.dispatch(node.expr) + self.stream.write("\n") + self.stream.write("[") + self._popup( + self._invocations(targets) + ) + self.stream.write("\n") + if node.lower: + self.dispatch(node.lower) + self.stream.write(":") + if node.upper: + self.dispatch(node.upper) + # NOTE: Step? + self.stream.write("]") + self.stream.write("\n") + + def visitSliceobj(self, node): + self.stream.write("\n") + first = 1 + for n in node.nodes: + if not first: + self.stream.write(":") + self.dispatch(n) + self.stream.write("\n") + + def visitSub(self, node): + self._visitBinary(node, "sub", "-") + + def visitSubscript(self, node): + target = node._node + targets = target.active() + self.stream.write("\n") + self.dispatch(node.expr) + self.stream.write("\n") + self.stream.write("[") + self._popup( + self._invocations(targets) + ) + self.stream.write("\n") + first = 1 + for sub in node.subs: + if not first: + self.stream.write(", ") + self.dispatch(sub) + first = 0 + self.stream.write("]") + self.stream.write("\n") + + visitTuple = visitAssTuple + + def visitUnaryAdd(self, node): + self._visitUnary(node, "add", "+") + + def visitUnarySub(self, node): + self._visitUnary(node, "sub", "-") + + # Output preparation methods. + + def _text(self, text): + return text.replace("&", "&").replace("<", "<").replace(">", ">") + + def _attr(self, attr): + return self._text(attr).replace("'", "'").replace('"', """) + + def _url(self, url): + return self._attr(url).replace("#", "%23").replace("-", "%2d") + + def _comment(self, comment): + self.stream.write("# %s\n" % comment) + + def _keyword(self, kw): + self.stream.write("%s " % kw) + + def _doc(self, node): + if node.doc is not None: + self.stream.write("
\n")
+            self.stream.write('"""')
+            output = textwrap.dedent(node.doc.replace('"""', '\\"\\"\\"'))
+            self.stream.write(self._text(output))
+            self.stream.write('"""')
+            self.stream.write("
\n") + + def _sequence(self, node): + first = 1 + for n in node.nodes: + if not first: + self.stream.write(",\n") + self.dispatch(n) + first = 0 + + def _mapping(self, node): + first = 1 + for k, v in node.items: + if not first: + self.stream.write(",\n") + self.dispatch(k) + self.stream.write(":\n") + self.dispatch(v) + first = 0 + + def _parameters(self, subprogram, subprograms): + + # Get all the parameter lists. + + params = [] + nparams = 0 + for sub in subprograms: + params.append(sub.params) + nparams = max(nparams, len(sub.params)) + stars = [] + have_star = 0 + for sub in subprograms: + stars.append(sub.star) + if sub.star is not None: + have_star = 1 + dstars = [] + have_dstar = 0 + for sub in subprograms: + dstars.append(sub.dstar) + if sub.dstar is not None: + have_dstar = 1 + + # Traverse the parameter lists, choosing a "column" at a time. + + first = 1 + for n in range(0, nparams): + if not first: + self.stream.write(",\n") + main_param, main_default = subprogram.params[n] + self._name_start(main_param) + self._popup( + self._parameter(subprograms, params, n) + ) + self._name_end() + self._default(main_default) + first = 0 + + if have_star: + if not first: + self.stream.write(", *\n") + main_param, main_default = subprogram.star + self._name_start(main_param) + self._popup( + self._parameter(subprograms, stars) + ) + self._name_end() + self._default(main_default) + first = 0 + + if have_dstar: + if not first: + self.stream.write(", **\n") + main_param, main_default = subprogram.dstar + self._name_start(main_param) + self._popup( + self._parameter(subprograms, dstars) + ) + self._name_end() + self._default(main_default) + first = 0 + + def _parameter(self, subprograms, params, n=None): + types = set() + for i in range(0, len(subprograms)): + subprogram = subprograms[i] + if n is not None: + param, default = params[i][n] + else: + param, default = params[i] + if hasattr(subprogram, "paramtypes"): + types.update(subprogram.paramtypes[param]) + return self._types_container(types, "types") + + def _default(self, default): + if default is not None and default.original is not None: + self.stream.write("=\n") + self.dispatch(default.original) + + def _name(self, name): + self.stream.write("%s\n" % name) + + def _name_start(self, name, classes=None): + if classes is not None: + classes = " " + classes + else: + classes = "" + self.stream.write("%s\n" % (classes, name)) + + def _name_end(self): + self.stream.write("\n") + + def _popup(self, info): + if info: + self.stream.write("\n") + for section, subsection, labels in info: + self.stream.write("
\n" % section) + for label in labels: + self.stream.write("
\n" % subsection) + self.stream.write(label) + self.stream.write("
\n") + self.stream.write("
\n") + self.stream.write("\n") + + def _op(self, node): + if hasattr(node, "_left_call") and hasattr(node, "_right_call"): + return self._invocations(node._left_call.active() + node._right_call.active()) + else: + _node = node._node + if isinstance(_node, Not): + _node = _node.expr + return self._invocations(_node.active()) + + def _invocations(self, nodes): + invocations = [] + for node in nodes: + if hasattr(node, "invocations"): + invocations += node.invocations + + # Record each link, avoiding duplicates. + + links = {} + for invocation in invocations: + fn = getattr(invocation, "copy_of", invocation).full_name() + module = invocation.module.name + name = invocation.name + structures = [x.name for x in invocation.structures] + qualified_name = ".".join([module] + structures + [name]) + + # Record the label and the link texts. + + label = self._text(qualified_name) + link = (self._url(module), self._url(fn)) + links[label] = link + + # Produce the list. + + if links: + popup_labels = [] + for label, (module_name, target_name) in links.items(): + popup_labels.append("%s" % (module_name, os.path.extsep, target_name, label)) + else: + popup_labels = [] + + if popup_labels: + return [("invocations", "invocation", popup_labels)] + else: + return [] + + def _types(self, nodes): + all_types = [(getattr(n, "types", []) or flatten(getattr(n, "writes", {}).values())) for n in nodes] + types = flatten(all_types) + return self._types_container(types, "types") + + def _types_container(self, types, style_class): + labels = {} + for type in types: + fn = type.type.full_name() + labels[self._text(fn)] = None + + if labels: + return [(style_class, 'type', labels.keys())] + else: + return [] + + def _raises(self, nodes): + + "Output the exception information for the given simplified 'nodes'." + + raises = set() + for node in nodes: + if hasattr(node, "raises") and node.raises: + raises.update(node.raises) + return self._types_container(raises, "raises") + + def _scopes(self, nodes): + + "Output the scope information for the given simplified 'nodes'." + + labels = {} + for node in nodes: + + # Straightforward name loading/storing involves the local scope. + + if isinstance(node, StoreName) or isinstance(node, LoadName): + labels["(local)"] = None + + # Other loading/storing involves attributes accessed on modules, classes + # and objects. + + else: + + # Loading... + + if hasattr(node, "accesses") and node.accesses: + for ref, accesses in node.accesses.items(): + fn = ref.full_name() + for attr, access in accesses: + access_fn = access.full_name() + label = self._text(fn) + if ref != access: + label += " (via " + self._text(access_fn) + ")" + labels[label] = None + + # Storing... + + if hasattr(node, "writes") and node.writes: + for ref in node.writes.keys(): + fn = ref.full_name() + labels[self._text(fn)] = None + + # Non-loading... + + if hasattr(node, "non_accesses") and node.non_accesses: + self._types_container(node.non_accesses, "non-accesses") + + # Non-storing... + + if hasattr(node, "non_writes") and node.non_writes: + self._types_container(node.non_writes, "non-writes") + + if labels: + return [("scopes", "scope", labels.keys())] + else: + return [] + +# Utility functions. + +def flatten(lists): + result = set() + for l in lists: + result.update(l) + return result + +# Convenience functions. + +def browse(module, stream=None): + browser = Browser(stream or sys.stdout) + browser.process(module.original) + +def makedoc(module, filename): + stream = open(filename, "wb") + try: + browser = Browser(stream) + browser.process(module.original) + finally: + stream.close() + +def makedocs(module, modules, builtins): + dirname = "%s-docs" % module.name + if not os.path.exists(dirname): + os.mkdir(dirname) + for m in [module, builtins] + modules: + makedoc(m, os.path.join(dirname, "%s%sxhtml" % (m.name, os.path.extsep))) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r dfd0634cfdb6 -r efe3cfb91966 viewer.py --- a/viewer.py Sun May 27 18:19:01 2007 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1171 +0,0 @@ -#!/usr/bin/env python - -""" -View annotated sources. - -Copyright (C) 2006, 2007 Paul Boddie - -This software is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of -the License, or (at your option) any later version. - -This software is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public -License along with this library; see the file LICENCE.txt -If not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -""" - -from compiler.visitor import ASTVisitor -from simplified import * -import sys -import os -import textwrap - -# Classes. - -# HTML-related output production. - -html_header = """ - - - - Module - - - -""" - -html_footer = """ - -""" - -# Browser classes. - -class Browser(ASTVisitor): - - """ - A browsing visitor for AST nodes. - - Covered: Add, And, Assert, AssAttr, AssList, AssName, AssTuple, Assign, - AugAssign, Bitand, Break, CallFunc, Class, Compare, Const, - Continue, Dict, Discard, Div, FloorDiv, For, From, Function, - Getattr, Global, If, Import, Keyword, Lambda, List, ListComp, - ListCompFor, ListCompIf, Mod, Module, Mul, Name, Not, Or, Pass, - Power, Print, Printnl, Raise, Return, Slice, Sliceobj, Stmt, Sub, - Subscript, TryExcept, TryFinally, Tuple, UnaryAdd, UnarySub, While. - - Missing: Backquote, Bitor, Bitxor, Decorators, Ellipsis, - Exec, Invert, LeftShift, RightShift, Yield. - """ - - def __init__(self, stream): - ASTVisitor.__init__(self) - self.visitor = self - self.stream = stream - - def process(self, module): - self.stream.write(html_header) - self.dispatch(module) - self.stream.write(html_footer) - - def visitModule(self, node): - self.default(node) - - # Statements. - - def visitAssert(self, node): - self.stream.write("
\n") - self.stream.write("\n") - self._keyword("assert") - self._popup( - self._types(node._raises.active()) - ) - self.stream.write("\n") - self.dispatch(node.test) - if node.fail: - self.stream.write(", ") - self.dispatch(node.fail) - self.stream.write("
\n") - - def visitAssign(self, node): - self.stream.write("
\n") - for lvalue in node.nodes: - self.dispatch(lvalue) - self.stream.write("=\n") - self.dispatch(node.expr) - self.stream.write("
\n") - - def visitAugAssign(self, node): - self.stream.write("
\n") - self.dispatch(node.node) - self.stream.write("\n") - self.stream.write("%s\n" % node.op) - self._popup( - self._invocations(node._op_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.expr) - self.stream.write("
\n") - - def visitBreak(self, node): - self.stream.write("
\n") - self._keyword("break") - self.stream.write("
\n") - - def visitClass(self, node): - definition = node._node - definitions = definition.active() - structure = definition.expr.ref - self.stream.write("
\n" % structure.full_name()) - self.stream.write("
\n") - self._keyword("class") - self._name_start(structure.name, "class-name") - self._popup( - self._scopes(definitions) - ) - self._name_end() - bases = structure.bases - - # Suppress the "object" class appearing alone. - - if bases and not (len(bases) == 1 and bases[0].name == "object"): - self.stream.write("(") - first = 1 - for base in bases: - if not first: - self.stream.write(",\n") - self._name_start(base.name) - self._popup( - self._scopes([base]) + - self._types([base]) - ) - self._name_end() - first = 0 - self.stream.write(")") - - self.stream.write(":\n") - self._comment(self._text(structure.full_name())) - self.stream.write("
\n") - - self.stream.write("
\n") - self._doc(node) - self.dispatch(node.code) - self.stream.write("
\n") - self.stream.write("
\n") - - def visitContinue(self, node): - self.stream.write("
\n") - self._keyword("continue") - self.stream.write("
\n") - - def visitDiscard(self, node): - self.stream.write("
\n") - self.default(node) - self.stream.write("
\n") - - def visitFor(self, node): - self.stream.write("
\n") - self.stream.write("
\n") - self.stream.write("\n") - self._keyword("for") - self._popup( - self._invocations(node._next_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.assign) - self.stream.write("\n") - self._keyword("in") - self._popup( - self._invocations(node._iter_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.list) - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.body) - self.stream.write("
\n") - if node.else_ is not None: - self.stream.write("
\n") - self._keyword("else") - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.else_) - self.stream.write("
\n") - self.stream.write("
\n") - - def visitFrom(self, node): - self.stream.write("
\n") - self._keyword("from") - self.stream.write("\n") - self.stream.write(node.modname) - self._popup( - self._types(node._modname.active()) - ) - self.stream.write("\n") - self._keyword("import") - first = 1 - for (name, alias), _name in map(None, node.names, node._names): - if not first: - self.stream.write(",\n") - if alias: - self.stream.write(name + " ") - self._keyword("as") - self.stream.write("\n") - self.stream.write(alias or name) - self._popup( - self._types([_name]) - ) - self.stream.write("\n") - first = 0 - self.stream.write("
\n") - - def visitFunction(self, node): - definition = node._node - definitions = [n for n in definition.active() if not isinstance(n, Subprogram)] - subprogram = node._subprogram - subprograms = subprogram.active() - self.stream.write("
\n" % subprogram.full_name()) - self.stream.write("
\n") - self._keyword("def") - self._name_start(subprogram.name, "function-name") - self._popup( - self._scopes([definition]) + # not dependent on subprograms - self._raises(subprograms) - ) - self._name_end() - self.stream.write("(") - self._parameters(subprogram, subprograms) - self.stream.write(")") - self.stream.write(":\n") - self._comment(self._text(subprogram.full_name())) - self.stream.write("
\n") - - self.stream.write("
\n") - self._doc(node) - self.dispatch(node.code) - self.stream.write("
\n") - self.stream.write("
\n") - - def visitGlobal(self, node): - self.stream.write("
\n") - self._keyword("global") - first = 1 - for name in node.names: - if not first: - self.stream.write(",\n") - self.stream.write(name) - first = 0 - self.stream.write("
\n") - - def visitIf(self, node): - self.stream.write("
\n") - first = 1 - conditional = node._node - conditionals = conditional.active() - for compare, stmt in node.tests: - self.stream.write("
\n") - self.stream.write("\n") - if first: - self._keyword("if") - else: - self._keyword("elif") - self._popup( - self._invocations([c.test for c in conditionals]) - ) - self.stream.write("\n") - self.dispatch(compare) - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(stmt) - self.stream.write("
\n") - if conditional.else_: - conditional = conditional.else_[0] - conditionals = conditional.active() - else: - conditional = None - conditionals = [] - first = 0 - if node.else_ is not None: - self.stream.write("
\n") - self._keyword("else") - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.else_) - self.stream.write("
\n") - self.stream.write("
\n") - - def visitImport(self, node): - self.stream.write("
\n") - self._keyword("import") - first = 1 - for (name, alias), _name in map(None, node.names, node._names): - if not first: - self.stream.write(",\n") - if alias: - self.stream.write(name + " ") - self._keyword("as") - self.stream.write("\n") - self.stream.write(alias or name) - self._popup( - self._types([_name]) - ) - self.stream.write("\n") - first = 0 - self.stream.write("
\n") - - def visitPass(self, node): - self.stream.write("
\n") - self._keyword("pass") - self.stream.write("
\n") - - def visitPrint(self, node): - self.stream.write("
\n") - self._keyword("print") - if node.dest is not None: - self.stream.write(">>\n") - self.dispatch(node.dest) - for n in node.nodes: - self.dispatch(n) - self.stream.write(",\n") - self.stream.write("
\n") - - def visitPrintnl(self, node): - self.stream.write("
\n") - self._keyword("print") - if node.dest is not None: - self.stream.write(">>\n") - self.dispatch(node.dest) - first = 1 - for n in node.nodes: - if not first: - self.stream.write(",\n") - self.dispatch(n) - first = 0 - self.stream.write("
\n") - - def visitRaise(self, node): - target = node._node.expr - targets = target.active() - self.stream.write("
\n") - self.stream.write("\n") - self._keyword("raise") - self._popup( - self._invocations(targets) - ) - self.stream.write("\n") - self.dispatch(node.expr1) - if node.expr2 is not None: - self.stream.write(",\n") - self.dispatch(node.expr2) - if node.expr3 is not None: - self.stream.write(",\n") - self.dispatch(node.expr3) - self.stream.write("
\n") - - def visitReturn(self, node): - value = node._node - values = value.active() - self.stream.write("
\n") - self.stream.write("\n") - self._keyword("return") - self._popup( - self._types(values) - ) - self.stream.write("\n") - self.dispatch(node.value) - self.stream.write("
\n") - - def visitStmt(self, node): - self.stream.write("
\n") - self.default(node) - self.stream.write("
\n") - - def visitTryExcept(self, node): - self.stream.write("
\n") - self.stream.write("
\n") - self._keyword("try") - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.body) - self.stream.write("
\n") - for spec, assign, statement in node.handlers: - self.stream.write("
\n") - self._keyword("except") - if spec is not None: - self.dispatch(spec) - if assign is not None: - self.stream.write(",\n") - self.dispatch(assign) - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(statement) - self.stream.write("
\n") - if node.else_ is not None: - self.stream.write("
\n") - self._keyword("else") - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.else_) - self.stream.write("
\n") - self.stream.write("
\n") - - def visitTryFinally(self, node): - self.stream.write("
\n") - self.stream.write("
\n") - self._keyword("try") - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.body) - self.stream.write("
\n") - self.stream.write("
\n") - self._keyword("finally") - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.final) - self.stream.write("
\n") - self.stream.write("
\n") - - def visitWhile(self, node): - self.stream.write("
\n") - self.stream.write("
\n") - self.stream.write("\n") - self._keyword("while") - self._popup( - self._invocations(node._test_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.test) - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.body) - self.stream.write("
\n") - if node.else_ is not None: - self.stream.write("
\n") - self._keyword("else") - self.stream.write(":\n") - self.stream.write("
\n") - self.stream.write("
\n") - self.dispatch(node.else_) - self.stream.write("
\n") - self.stream.write("
\n") - - # Expression-related helper methods. - - def _visitBinary(self, node, name, symbol): - self.stream.write("\n" % name) - self.dispatch(node.left) - self.stream.write("\n") - self.stream.write(self._text(symbol)) - self._popup( - self._invocations(node._left_call.active() + node._right_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.right) - self.stream.write("") - - def _visitUnary(self, node, name, symbol): - self.stream.write("\n" % name) - self.stream.write("\n") - self.stream.write(symbol) - self._popup( - self._invocations(node._unary_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.expr) - self.stream.write("") - - # Expressions. - - def visitAdd(self, node): - self._visitBinary(node, "add", "+") - - def visitAnd(self, node): - self.stream.write("\n") - first = 1 - for n in node.nodes: - if not first: - self._keyword("and") - self.dispatch(n) - first = 0 - self.stream.write("") - - def visitAssAttr(self, node): - target = node._node - targets = target.active() - self.stream.write("\n") - self.dispatch(node.expr) - self.stream.write("\n") - self.stream.write(".%s\n" % self._text(node.attrname)) - self._popup( - self._scopes(targets) + - self._types(targets) - ) - self.stream.write("\n") - self.stream.write("\n") - - def visitAssList(self, node): - self.stream.write("\n") - self.stream.write("[") - self._sequence(node) - self.stream.write("]\n") - self.stream.write("\n") - - def visitAssName(self, node): - target = node._node - targets = target.active() - self._name_start(target.name) - self._popup( - self._scopes(targets) + - self._types(targets) - ) - self._name_end() - - def visitAssTuple(self, node): - self.stream.write("\n") - self.stream.write("(") - self._sequence(node) - self.stream.write(")\n") - self.stream.write("\n") - - def visitBitand(self, node): - self.stream.write("\n") - self.dispatch(node.nodes[0]) - for op in node._ops: - self.stream.write("\n") - self.stream.write(self._text(op.name)) - self._popup( - self._op(op) - ) - self.stream.write("\n") - self.dispatch(op.expr) - self.stream.write("") - - def visitCallFunc(self, node): - target = node._node - targets = target.active() - self.stream.write("\n") - self.dispatch(node.node) - self.stream.write("\n") - self.stream.write("(") - self._popup( - self._invocations(targets) - ) - self.stream.write("\n") - first = 1 - for arg in node.args: - if not first: - self.stream.write(",\n") - self.dispatch(arg) - first = 0 - if node.star_args is not None: - if not first: - self.stream.write(", *\n") - self.dispatch(node.star_args) - first = 0 - if node.dstar_args is not None: - if not first: - self.stream.write(", **\n") - self.dispatch(node.dstar_args) - first = 0 - self.stream.write(")\n") - self.stream.write("\n") - - def visitCompare(self, node): - self.stream.write("\n") - self.dispatch(node.expr) - for op in node._ops: - self.stream.write("\n") - self.stream.write(self._text(op.name)) - self._popup( - self._op(op) - ) - self.stream.write("\n") - self.dispatch(op.expr) - self.stream.write("\n") - - def visitConst(self, node): - if isinstance(node.value, (str, unicode)): - self.stream.write("\n") - self.stream.write(repr(node.value)) - if isinstance(node.value, (str, unicode)): - self.stream.write("\n") - - def visitDict(self, node): - self.stream.write("\n") - self.stream.write("{") - self._mapping(node) - self.stream.write("}\n") - self.stream.write("\n") - - def visitDiv(self, node): - self._visitBinary(node, "div", "/") - - def visitFloorDiv(self, node): - self._visitBinary(node, "floordiv", "//") - - def visitGetattr(self, node): - target = node._node - targets = target.active() - self.stream.write("\n") - self.dispatch(node.expr) - self.stream.write("\n") - self.stream.write(".%s\n" % self._text(node.attrname)) - self._popup( - self._scopes(targets) + - self._types(targets) - ) - self.stream.write("\n") - self.stream.write("\n") - - def visitKeyword(self, node): - self.stream.write("\n") - self.stream.write(node.name) - self.stream.write("=") - self.dispatch(node.expr) - self.stream.write("\n") - - def visitLambda(self, node): - definition = node._node - definitions = [n for n in definition.active() if not isinstance(n, Subprogram)] - subprogram = node._subprogram - subprograms = subprogram.active() - self.stream.write("\n") - self._keyword("lambda") - self._parameters(subprogram, subprograms) - self.dispatch(node.code) - self.stream.write("\n") - - visitList = visitAssList - - def visitListComp(self, node): - self.stream.write("\n") - self.stream.write("[") - self.dispatch(node.expr) - for qual in node.quals: - self.dispatch(qual) - self.stream.write("]\n") - self.stream.write("\n") - - def visitListCompFor(self, node): - self.stream.write("\n") - self.stream.write("\n") - self._keyword("for") - self._popup( - self._invocations(node._next_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.assign) - self.stream.write("\n") - self._keyword("in") - self._popup( - self._invocations(node._iter_call.active()) - ) - self.stream.write("\n") - self.dispatch(node.list) - for if_ in node.ifs: - self.dispatch(if_) - self.stream.write("\n") - - def visitListCompIf(self, node): - conditional = node._node - conditionals = conditional.active() - self.stream.write("\n") - self.stream.write("\n") - self._keyword("if") - self._popup( - self._invocations([c.test for c in conditionals]) - ) - self.stream.write("\n") - self.dispatch(node.test) - self.stream.write("\n") - - def visitMod(self, node): - self._visitBinary(node, "mod", "%") - - def visitMul(self, node): - self._visitBinary(node, "mul", "*") - - def visitName(self, node): - target = node._node - targets = target.active() - self._name_start(target.name) - self._popup( - self._scopes(targets) + - self._types(targets) - ) - self._name_end() - - def visitNot(self, node): - self.stream.write("\n") - self._keyword("not") - self.dispatch(node.expr) - self.stream.write("") - - def visitOr(self, node): - self.stream.write("\n") - first = 1 - for n in node.nodes: - if not first: - self._keyword("or") - self.dispatch(n) - first = 0 - self.stream.write("") - - def visitPower(self, node): - self._visitBinary(node, "power", "**") - - def visitSlice(self, node): - target = node._node - targets = target.active() - self.stream.write("\n") - self.dispatch(node.expr) - self.stream.write("\n") - self.stream.write("[") - self._popup( - self._invocations(targets) - ) - self.stream.write("\n") - if node.lower: - self.dispatch(node.lower) - self.stream.write(":") - if node.upper: - self.dispatch(node.upper) - # NOTE: Step? - self.stream.write("]") - self.stream.write("\n") - - def visitSliceobj(self, node): - self.stream.write("\n") - first = 1 - for n in node.nodes: - if not first: - self.stream.write(":") - self.dispatch(n) - self.stream.write("\n") - - def visitSub(self, node): - self._visitBinary(node, "sub", "-") - - def visitSubscript(self, node): - target = node._node - targets = target.active() - self.stream.write("\n") - self.dispatch(node.expr) - self.stream.write("\n") - self.stream.write("[") - self._popup( - self._invocations(targets) - ) - self.stream.write("\n") - first = 1 - for sub in node.subs: - if not first: - self.stream.write(", ") - self.dispatch(sub) - first = 0 - self.stream.write("]") - self.stream.write("\n") - - visitTuple = visitAssTuple - - def visitUnaryAdd(self, node): - self._visitUnary(node, "add", "+") - - def visitUnarySub(self, node): - self._visitUnary(node, "sub", "-") - - # Output preparation methods. - - def _text(self, text): - return text.replace("&", "&").replace("<", "<").replace(">", ">") - - def _attr(self, attr): - return self._text(attr).replace("'", "'").replace('"', """) - - def _url(self, url): - return self._attr(url).replace("#", "%23").replace("-", "%2d") - - def _comment(self, comment): - self.stream.write("# %s\n" % comment) - - def _keyword(self, kw): - self.stream.write("%s " % kw) - - def _doc(self, node): - if node.doc is not None: - self.stream.write("
\n")
-            self.stream.write('"""')
-            output = textwrap.dedent(node.doc.replace('"""', '\\"\\"\\"'))
-            self.stream.write(self._text(output))
-            self.stream.write('"""')
-            self.stream.write("
\n") - - def _sequence(self, node): - first = 1 - for n in node.nodes: - if not first: - self.stream.write(",\n") - self.dispatch(n) - first = 0 - - def _mapping(self, node): - first = 1 - for k, v in node.items: - if not first: - self.stream.write(",\n") - self.dispatch(k) - self.stream.write(":\n") - self.dispatch(v) - first = 0 - - def _parameters(self, subprogram, subprograms): - - # Get all the parameter lists. - - params = [] - nparams = 0 - for sub in subprograms: - params.append(sub.params) - nparams = max(nparams, len(sub.params)) - stars = [] - have_star = 0 - for sub in subprograms: - stars.append(sub.star) - if sub.star is not None: - have_star = 1 - dstars = [] - have_dstar = 0 - for sub in subprograms: - dstars.append(sub.dstar) - if sub.dstar is not None: - have_dstar = 1 - - # Traverse the parameter lists, choosing a "column" at a time. - - first = 1 - for n in range(0, nparams): - if not first: - self.stream.write(",\n") - main_param, main_default = subprogram.params[n] - self._name_start(main_param) - self._popup( - self._parameter(subprograms, params, n) - ) - self._name_end() - self._default(main_default) - first = 0 - - if have_star: - if not first: - self.stream.write(", *\n") - main_param, main_default = subprogram.star - self._name_start(main_param) - self._popup( - self._parameter(subprograms, stars) - ) - self._name_end() - self._default(main_default) - first = 0 - - if have_dstar: - if not first: - self.stream.write(", **\n") - main_param, main_default = subprogram.dstar - self._name_start(main_param) - self._popup( - self._parameter(subprograms, dstars) - ) - self._name_end() - self._default(main_default) - first = 0 - - def _parameter(self, subprograms, params, n=None): - types = set() - for i in range(0, len(subprograms)): - subprogram = subprograms[i] - if n is not None: - param, default = params[i][n] - else: - param, default = params[i] - if hasattr(subprogram, "paramtypes"): - types.update(subprogram.paramtypes[param]) - return self._types_container(types, "types") - - def _default(self, default): - if default is not None and default.original is not None: - self.stream.write("=\n") - self.dispatch(default.original) - - def _name(self, name): - self.stream.write("%s\n" % name) - - def _name_start(self, name, classes=None): - if classes is not None: - classes = " " + classes - else: - classes = "" - self.stream.write("%s\n" % (classes, name)) - - def _name_end(self): - self.stream.write("\n") - - def _popup(self, info): - if info: - self.stream.write("\n") - for section, subsection, labels in info: - self.stream.write("
\n" % section) - for label in labels: - self.stream.write("
\n" % subsection) - self.stream.write(label) - self.stream.write("
\n") - self.stream.write("
\n") - self.stream.write("\n") - - def _op(self, node): - if hasattr(node, "_left_call") and hasattr(node, "_right_call"): - return self._invocations(node._left_call.active() + node._right_call.active()) - else: - _node = node._node - if isinstance(_node, Not): - _node = _node.expr - return self._invocations(_node.active()) - - def _invocations(self, nodes): - invocations = [] - for node in nodes: - if hasattr(node, "invocations"): - invocations += node.invocations - - # Record each link, avoiding duplicates. - - links = {} - for invocation in invocations: - fn = getattr(invocation, "copy_of", invocation).full_name() - module = invocation.module.name - name = invocation.name - structures = [x.name for x in invocation.structures] - qualified_name = ".".join([module] + structures + [name]) - - # Record the label and the link texts. - - label = self._text(qualified_name) - link = (self._url(module), self._url(fn)) - links[label] = link - - # Produce the list. - - if links: - popup_labels = [] - for label, (module_name, target_name) in links.items(): - popup_labels.append("%s" % (module_name, os.path.extsep, target_name, label)) - else: - popup_labels = [] - - if popup_labels: - return [("invocations", "invocation", popup_labels)] - else: - return [] - - def _types(self, nodes): - all_types = [(getattr(n, "types", []) or flatten(getattr(n, "writes", {}).values())) for n in nodes] - types = flatten(all_types) - return self._types_container(types, "types") - - def _types_container(self, types, style_class): - labels = {} - for type in types: - fn = type.type.full_name() - labels[self._text(fn)] = None - - if labels: - return [(style_class, 'type', labels.keys())] - else: - return [] - - def _raises(self, nodes): - - "Output the exception information for the given simplified 'nodes'." - - raises = set() - for node in nodes: - if hasattr(node, "raises") and node.raises: - raises.update(node.raises) - return self._types_container(raises, "raises") - - def _scopes(self, nodes): - - "Output the scope information for the given simplified 'nodes'." - - labels = {} - for node in nodes: - - # Straightforward name loading/storing involves the local scope. - - if isinstance(node, StoreName) or isinstance(node, LoadName): - labels["(local)"] = None - - # Other loading/storing involves attributes accessed on modules, classes - # and objects. - - else: - - # Loading... - - if hasattr(node, "accesses") and node.accesses: - for ref, accesses in node.accesses.items(): - fn = ref.full_name() - for attr, access in accesses: - access_fn = access.full_name() - label = self._text(fn) - if ref != access: - label += " (via " + self._text(access_fn) + ")" - labels[label] = None - - # Storing... - - if hasattr(node, "writes") and node.writes: - for ref in node.writes.keys(): - fn = ref.full_name() - labels[self._text(fn)] = None - - # Non-loading... - - if hasattr(node, "non_accesses") and node.non_accesses: - self._types_container(node.non_accesses, "non-accesses") - - # Non-storing... - - if hasattr(node, "non_writes") and node.non_writes: - self._types_container(node.non_writes, "non-writes") - - if labels: - return [("scopes", "scope", labels.keys())] - else: - return [] - -# Utility functions. - -def flatten(lists): - result = set() - for l in lists: - result.update(l) - return result - -# Convenience functions. - -def browse(module, stream=None): - browser = Browser(stream or sys.stdout) - browser.process(module.original) - -def makedoc(module, filename): - stream = open(filename, "wb") - try: - browser = Browser(stream) - browser.process(module.original) - finally: - stream.close() - -def makedocs(module, modules, builtins): - dirname = "%s-docs" % module.name - if not os.path.exists(dirname): - os.mkdir(dirname) - for m in [module, builtins] + modules: - makedoc(m, os.path.join(dirname, "%s%sxhtml" % (m.name, os.path.extsep))) - -# vim: tabstop=4 expandtab shiftwidth=4