# HG changeset patch # User Paul Boddie # Date 1472568670 -7200 # Node ID 5366b587e3facfb210dff22f937b8612e066c356 Import of previous PythonLight code. diff -r 000000000000 -r 5366b587e3fa branching.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/branching.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,818 @@ +#!/usr/bin/env python + +""" +Track attribute usage for names. + +Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, + 2014, 2015, 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +from common import dict_for_keys, init_item + +class Branch: + + """ + A control-flow branch capturing local attribute usage for names. + Branches typically begin with assignments or function parameters and are + connected to others introduced by conditional and loop nodes. + + Branches hosting accesses, and thus providing usage information, are + contributors to preceding branches. + + Branches also provide a route from accesses back to assignments which are + the ultimate suppliers of the names involved. + """ + + def __init__(self, names, assigning=False, values=None): + + """ + Capture attribute usage for the given 'names', with the accompanying + 'values' indicating assigned values for each name, if indicated. + """ + + self.contributors = set() + self.suppliers = {} + self.assignments = set(assigning and names or []) + self.usage = {} + self.values = {} + + # Initialise usage for each name. + + for name in names: + self.usage[name] = set() + + # Initialise assigned values if any were provided. + + if values: + for name, value in zip(names, values): + if value: + self.values[name] = value + + # Computed results. + + self.combined_usage = None + + def get_assignment_sources(self, name): + + """ + Return the sources of 'name' from this branch's assignment information, + returning a list containing only this branch itself if it is the source. + """ + + if name in self.assignments: + return [self] + else: + return [b for b in self.get_all_suppliers(name) if name in b.assignments] + + def set_usage(self, name, attrname): + + """ + Record usage on the given 'name' of the attribute 'attrname'. + """ + + if self.usage.has_key(name): + self.usage[name].add(attrname) + + def get_usage(self): + + """ + Obtain usage from this node, combined with usage observed by its + contributors. Unlike the local usage which involves only a single set of + attribute names for a given variable name, the returned usage is a set + of attribute name combinations for a given variable name. For example: + + {'a': set([('p', 'q', 'r'), ('p', 'r')])} + """ + + if self.combined_usage is None: + + # Accumulate usage observations from contributors. + + all_usage = [] + + for contributor in self.contributors: + + # Record any usage that can be returned. + + all_usage.append(contributor.get_usage()) + + # Merge usage from the contributors. + + merged_usage = merge_dicts(all_usage) + + # Make the local usage compatible with the combined usage. + + usage = deepen_dict(self.usage) + + self.combined_usage = combine_dicts(usage, merged_usage, combine_sets) + + return self.combined_usage + + def get_all_suppliers(self, name, all_suppliers=None): + + "Return all branches supplying this branch with definitions of 'name'." + + all_suppliers = all_suppliers or set() + all_suppliers.add(self) + + if self.suppliers.has_key(name): + for supplier in self.suppliers[name]: + if supplier not in all_suppliers: + supplier.get_all_suppliers(name, all_suppliers) + + return all_suppliers + + def __repr__(self): + return "Branch(%r, %r)" % (self.usage.keys(), + self.assignments and True or False) + +class BranchTracker: + + """ + A tracker of attribute usage for names in a namespace. This tracker directs + usage observations to branches which are the ultimate repositories of + attribute usage information. + + As a program unit is inspected, the branches associated with names may + change. Assignments reset the branches; control-flow operations cause + branches to be accumulated from different code paths. + """ + + def __init__(self): + + # Track assignments. + + self.assignments = {} + + # Details of attributes at each active branch level. + + self.attribute_branches = [{}] # stack of branches for names + self.attribute_branch_shelves = [] # stack of shelved branches + + # Suspended branch details plus loop details. + + self.suspended_broken_branches = [] # stack of lists of dicts + self.suspended_continuing_branches = [] # stack of lists of dicts + + # Abandoned usage, useful for reviving usage for exception handlers. + + self.abandoned_branches = [[]] # stack of lists of branches + + # Returning branches are like abandoned branches but are only revived in + # finally clauses. + + self.returning_branches = [[]] + + # Branches active when starting loops. + + self.loop_branches = [] + + # Inherited usage. + + self.inherited = None + + # Structure assembly methods. + + def new_branchpoint(self, loop_node=False): + + """ + Indicate that branches diverge, initialising resources dependent on + any given 'loop_node'. + """ + + self.attribute_branch_shelves.append([]) + + if loop_node: + self.suspended_broken_branches.append([]) + self.suspended_continuing_branches.append([]) + + # Retain a record of abandoned branches. + + self.abandoned_branches.append([]) + self.returning_branches.append([]) + + def new_branch(self, loop_node=False): + + "Create a new branch." + + attribute_branches = self.attribute_branches[-1] + + branch, new_branches = self._new_branch(attribute_branches) + + if branch and loop_node: + self.loop_branches.append(branch) + + # Start using the branch for known names. + + self.attribute_branches.append(new_branches) + + def _new_branch(self, attribute_branches): + + """ + Define a new branch that will record attribute usage on known names from + 'attribute_branches'. + """ + + # Detect abandoned branches. + + if isinstance(attribute_branches, AbandonedDict): + return None, AbandonedDict() + + # Otherwise, define a new branch. + + names = attribute_branches.keys() + + new_branches = {} + branch = Branch(names) + + for name in names: + new_branches[name] = [branch] + + # Add this new branch as a contributor to the previously active + # branches. + + self._connect_branches(attribute_branches, branch) + + return branch, new_branches + + def shelve_branch(self, loop_node=False): + + "Retain the current branch for later merging." + + branches = self.attribute_branches.pop() + self.attribute_branch_shelves[-1].append(branches) + + # Connect any loop branch to the active branches as contributors. + + if loop_node: + branch = self.loop_branches.pop() + self._connect_branches(branches, branch, loop_node) + + def abandon_branch(self): + + "Abandon the current branch, retaining it for later." + + attribute_branches = self.attribute_branches[-1] + self._abandon_branch() + self.abandoned_branches[-1].append(attribute_branches) + + def abandon_returning_branch(self): + + "Abandon the current branch, retaining it for later." + + attribute_branches = self.attribute_branches[-1] + self._abandon_branch() + self.returning_branches[-1].append(attribute_branches) + + def suspend_broken_branch(self): + + "Suspend a branch for breaking out of a loop." + + attribute_branches = self.attribute_branches[-1] + + branches = self.suspended_broken_branches[-1] + branches.append(attribute_branches) + self._abandon_branch() + + def suspend_continuing_branch(self): + + "Suspend a branch for loop continuation." + + attribute_branches = self.attribute_branches[-1] + + branches = self.suspended_continuing_branches[-1] + branches.append(attribute_branches) + self._abandon_branch() + + def _abandon_branch(self): + + "Abandon the current branch." + + self.attribute_branches[-1] = AbandonedDict() + + def resume_abandoned_branches(self): + + """ + Resume branches previously abandoned. + + Abandoned branches are not reset because they may not be handled by + exception handlers after all. + """ + + current_branches = self.attribute_branches[-1] + abandoned_branches = self.abandoned_branches[-1] + merged_branches = merge_dicts(abandoned_branches + [current_branches]) + + # Replace the combined branches with a new branch applying to all active + # names, connected to the supplying branches. + + branch, new_branches = self._new_branch(merged_branches) + self.attribute_branches.append(new_branches) + + # Although returning branches should not be considered as accumulating + # usage, they do provide sources of assignments. + + if branch: + for returning_branches in self.returning_branches[-1]: + self._connect_suppliers(returning_branches, branch) + + def resume_all_abandoned_branches(self): + + """ + Resume branches previously abandoned including returning branches. + + Abandoned branches are not reset because they may not be handled by + exception handlers after all. + """ + + current_branches = self.attribute_branches[-1] + abandoned_branches = self.abandoned_branches[-1] + returning_branches = self.returning_branches[-1] + merged_branches = merge_dicts(abandoned_branches + returning_branches + [current_branches]) + self.replace_branches(merged_branches) + + # Return the previously-active branches for later restoration. + + return current_branches + + def resume_broken_branches(self): + + "Resume branches previously suspended for breaking out of a loop." + + suspended_branches = self.suspended_broken_branches.pop() + current_branches = self.attribute_branches[-1] + + # Merge suspended branches with the current branch. + + merged_branches = merge_dicts(suspended_branches + [current_branches]) + self.replace_branches(merged_branches) + + def resume_continuing_branches(self): + + "Resume branches previously suspended for loop continuation." + + suspended_branches = self.suspended_continuing_branches.pop() + current_branches = self.attribute_branches[-1] + + # Merge suspended branches with the current branch. + + merged_branches = merge_dicts(suspended_branches + [current_branches]) + self.replace_branches(merged_branches) + + def replace_branches(self, merged_branches): + + """ + Replace the 'merged_branches' with a new branch applying to all active + names, connected to the supplying branches. + """ + + branch, new_branches = self._new_branch(merged_branches) + self.attribute_branches[-1] = new_branches + + def restore_active_branches(self, branches): + + "Restore the active 'branches'." + + self.attribute_branches[-1] = branches + + def merge_branches(self): + + "Merge branches." + + # Combine the attribute branches. This ensures that a list of branches + # affected by attribute usage is maintained for the current branch. + + all_shelved_branches = self.attribute_branch_shelves.pop() + merged_branches = merge_dicts(all_shelved_branches, missing=make_missing) + self.replace_branches(merged_branches) + + # Abandoned branches are retained for exception handling purposes. + + all_abandoned_branches = self.abandoned_branches.pop() + new_abandoned_branches = merge_dicts(all_abandoned_branches) + self.abandoned_branches[-1].append(new_abandoned_branches) + + # Returning branches are retained for finally clauses. + + all_returning_branches = self.returning_branches.pop() + new_returning_branches = merge_dicts(all_returning_branches) + self.returning_branches[-1].append(new_returning_branches) + + # Internal structure assembly methods. + + def _connect_branches(self, attribute_branches, contributor, loop_node=False): + + """ + Given the 'attribute_branches' mapping, connect the branches referenced + in the mapping to the given 'contributor' branch. If 'loop_node' is + set to a true value, connect only the branches so that the 'contributor' + references the nodes supplying it with name information. + """ + + all_branches = self._connect_suppliers(attribute_branches, contributor) + if not loop_node: + self._connect_contributor(contributor, all_branches) + + def _connect_suppliers(self, attribute_branches, contributor): + + "Connect the 'attribute_branches' to the given 'contributor'." + + # Gather branches involved with all known names into a single set. + + all_branches = set() + + for name, branches in attribute_branches.items(): + all_branches.update(branches) + + # Also note receiving branches on the contributor. + + for branch in branches: + init_item(contributor.suppliers, name, set) + contributor.suppliers[name].add(branch) + + return all_branches + + def _connect_contributor(self, contributor, branches): + + "Connect the given 'contributor' branch to the given 'branches'." + + for branch in branches: + branch.contributors.add(contributor) + + # Namespace methods. + + def inherit_branches(self, tracker, names): + + """ + Propagate branches from the given 'tracker' excluding those associated + with 'names'. + """ + + # For each inherited name, create a branch connected to the inherited + # branches. + + self.inherited = {} + + for name, branches in tracker.attribute_branches[-1].items(): + + # Do not inherit any listed names (typically parameters) or any + # special names. + + if name in names or name.startswith("$"): + continue + + # Make a tentative assignment for the name. + + contributor = Branch([name], True) + init_item(self.assignments, name, list) + self.assignments[name].append(contributor) + + # Connect the inherited branch to the new one. + + for branch in branches: + init_item(contributor.suppliers, name, set) + contributor.suppliers[name].add(branch) + branch.contributors.add(contributor) + + # Record the inherited branch. + + self.inherited[name] = [contributor] + + self.attribute_branches[-1].update(self.inherited) + + def disconnect_name(self, name): + + "Disconnect inherited branches for 'name'." + + if not self.inherited or not self.inherited.has_key(name): + return + + # Remove the new branch from the inherited branches for the name. + + for contributor in self.inherited[name]: + for supplier in contributor.suppliers[name]: + supplier.contributors.remove(contributor) + del contributor.suppliers[name] + + del self.inherited[name] + + # Attribute usage methods. + + def tracking_name(self, name): + + """ + Return whether 'name' is being tracked, returning all branches doing so + if it is. + """ + + return self.assignments.has_key(name) and \ + (not self.inherited or not self.inherited.has_key(name)) and \ + self.have_name(name) + + def have_name(self, name): + + "Return whether 'name' is known, perhaps having been inherited." + + return self.attribute_branches[-1].get(name) + + def assign_names(self, names, values=None): + + """ + Define the start of usage tracking for the given 'names', each being + assigned with the corresponding 'values' if indicated. + """ + + branches = self.attribute_branches[-1] + branch = Branch(names, True, values) + + for name in names: + self.disconnect_name(name) + + branches[name] = [branch] + init_item(self.assignments, name, list) + self.assignments[name].append(branch) + + return branch + + def use_attribute(self, name, attrname): + + """ + Indicate the use on the given 'name' of an attribute with the given + 'attrname'. + + Return all branches that support 'name'. + """ + + branches = self.attribute_branches[-1] + + # Add the usage to all current branches. + + if branches.has_key(name): + for branch in branches[name]: + branch.set_usage(name, attrname) + return branches[name] + else: + return None + + # Query methods. + + def get_assignment_positions_for_branches(self, name, branches, missing=True): + + """ + Return the positions of assignments involving the given 'name' affected + by the given 'branches'. If 'missing' is set to a false value, branches + with missing name details will be excluded instead of contributing the + value None to the list of positions. + """ + + if not branches: + return [None] + + positions = set() + assignments = self.assignments[name] + + for assignment in self.get_assignments_for_branches(name, branches): + + # Use None to indicate a branch without assignment information. + + if missing and isinstance(assignment, MissingBranch): + positions.add(None) + else: + pos = assignments.index(assignment) + positions.add(pos) + + positions = list(positions) + positions.sort() + return positions + + def get_assignments_for_branches(self, name, branches, missing=True): + + """ + Return the origins of assignments involving the given 'name' affected + by the given 'branches'. The origins are a list of branches where names + are defined using assignments. If 'missing' is set to a false value, + branches with missing name details are excluded. + """ + + all_branches = [] + assignments = self.assignments[name] + + # Obtain the assignments recorded for each branch. + + for branch in branches: + + # Find the branch representing the definition of some names in the + # scope's assignments, making sure that the given name is involved. + + for assignment in branch.get_assignment_sources(name): + + # Capture branches without assignment information as well as + # genuine assignment branches. + + if assignment in assignments or missing and isinstance(assignment, MissingBranch): + all_branches.append(assignment) + + return all_branches + + def get_all_usage(self): + + """ + Convert usage observations from the tracker to a simple mapping of + names to sets of attribute names. + """ + + d = {} + for name, branches in self.assignments.items(): + d[name] = self.get_usage_from_branches_for_name(branches, name) + return d + + def get_usage_from_branches_for_name(self, branches, name): + + """ + Convert usage observations from the 'branches' to a simple list of + usage sets for the given 'name'. + """ + + l = [] + for branch in branches: + l.append(branch.get_usage()[name]) + return l + + def get_all_values(self): + + "Return a mapping from names to lists of assigned values." + + d = {} + for name, branches in self.assignments.items(): + d[name] = [branch.values.get(name) for branch in branches] + return d + +# Special objects. + +class AbandonedDict(dict): + + "A dictionary representing mappings in an abandoned branch." + + def __repr__(self): + return "AbandonedDict()" + +class MissingBranch(Branch): + + "A branch introduced during dictionary merging." + + def __repr__(self): + return "MissingBranch(%r, %r)" % (self.usage.keys(), + self.assignments and True or False) + +def make_missing(name): + + "Make a special branch indicating missing name information." + + return set([MissingBranch([name], True)]) + +# Dictionary utilities. + +def merge_dicts(dicts, ignored=AbandonedDict, missing=None): + + """ + Merge the given 'dicts' mapping keys to sets of values. + + Where 'ignored' is specified, any dictionary of the given type is ignored. + Where all dictionaries to be merged are of the given type, an instance of + the type is returned as the merged dictionary. + + Where 'missing' is specified, it provides a callable that produces a set of + suitable values for a given name. + """ + + new_dict = {} + all_names = set() + + # Determine all known names. + + for old_dict in dicts: + all_names.update(old_dict.keys()) + + # Merge the dictionaries, looking for all known names in each one. + + have_dicts = False + + for old_dict in dicts: + + # Abandoned dictionaries should not contribute information. + + if isinstance(old_dict, ignored): + continue + else: + have_dicts = True + + for name in all_names: + + # Find branches providing each name. + + if old_dict.has_key(name): + values = old_dict[name] + + # Branches not providing names may indicate usage before assignment. + + elif missing: + values = missing(name) + else: + continue + + # Initialise mappings in the resulting dictionary. + + if not new_dict.has_key(name): + new_dict[name] = set(values) + else: + new_dict[name].update(values) + + # Where no dictionaries contributed, all branches were abandoned. + + if have_dicts: + return new_dict + else: + return ignored() + +def deepen_dict(d): + + """ + Return a version of dictionary 'd' with its values converted to sets + containing each original value as a single element in each new value. + Original values are assumed to be sequences. Thus... + + {"self" : ("x", "y")} + + ...would become... + + {"self" : set([("x", "y")])} + + ...allowing other such values to be added to the set alongside the original + value. + """ + + l = [] + + for key, value in d.items(): + + # Sort the attribute name details for stable comparisons. + + value = list(value) + value.sort() + l.append((key, set([tuple(value)]))) + + return dict(l) + +def combine_sets(s1, s2): + + "Combine elements from sets 's1' and 's2'." + + if not s1: + return s2 + elif not s2: + return s1 + + s = set() + + for i1 in s1: + for i2 in s2: + + # Sort the attribute name details for stable comparisons. + + l = list(set(i1 + i2)) + l.sort() + s.add(tuple(l)) + + return s + +def combine_dicts(d1, d2, combine=combine_sets): + + """ + Combine dictionaries 'd1' and 'd2' such that the values for common keys + are themselves combined in the result. + """ + + d = {} + + for key in d1.keys(): + if d2.has_key(key): + d[key] = combine(d1[key], d2[key]) + else: + d[key] = d1[key] + + return d + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa common.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,1067 @@ +#!/usr/bin/env python + +""" +Common functions. + +Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, + 2014, 2015, 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +from errors import * +from os import listdir, makedirs, remove +from os.path import exists, isdir, join, split +import compiler + +class CommonOutput: + + "Common output functionality." + + def check_output(self): + + "Check the existing output and remove it if irrelevant." + + if not exists(self.output): + makedirs(self.output) + + details = self.importer.get_cache_details() + recorded_details = self.get_output_details() + + if recorded_details != details: + self.remove_output() + + writefile(self.get_output_details_filename(), details) + + def get_output_details_filename(self): + + "Return the output details filename." + + return join(self.output, "$details") + + def get_output_details(self): + + "Return details of the existing output." + + details_filename = self.get_output_details_filename() + + if not exists(details_filename): + return None + else: + return readfile(details_filename) + + def remove_output(self, dirname=None): + + "Remove the output." + + dirname = dirname or self.output + + for filename in listdir(dirname): + path = join(dirname, filename) + if isdir(path): + self.remove_output(path) + else: + remove(path) + +class CommonModule: + + "A common module representation." + + def __init__(self, name, importer): + + """ + Initialise this module with the given 'name' and an 'importer' which is + used to provide access to other modules when required. + """ + + self.name = name + self.importer = importer + self.filename = None + + # Inspection-related attributes. + + self.astnode = None + self.iterators = {} + self.temp = {} + self.lambdas = {} + + # Constants, literals and values. + + self.constants = {} + self.constant_values = {} + self.literals = {} + self.literal_types = {} + + # Nested namespaces. + + self.namespace_path = [] + self.in_function = False + + # Attribute chains. + + self.attrs = [] + + def __repr__(self): + return "CommonModule(%r, %r)" % (self.name, self.importer) + + def parse_file(self, filename): + + "Parse the file with the given 'filename', initialising attributes." + + self.filename = filename + self.astnode = compiler.parseFile(filename) + + # Module-relative naming. + + def get_global_path(self, name): + return "%s.%s" % (self.name, name) + + def get_namespace_path(self): + return ".".join([self.name] + self.namespace_path) + + def get_object_path(self, name): + return ".".join([self.name] + self.namespace_path + [name]) + + def get_parent_path(self): + return ".".join([self.name] + self.namespace_path[:-1]) + + # Namespace management. + + def enter_namespace(self, name): + + "Enter the namespace having the given 'name'." + + self.namespace_path.append(name) + + def exit_namespace(self): + + "Exit the current namespace." + + self.namespace_path.pop() + + # Constant reference naming. + + def get_constant_name(self, value): + + "Add a new constant to the current namespace for 'value'." + + path = self.get_namespace_path() + init_item(self.constants, path, dict) + return "$c%d" % add_counter_item(self.constants[path], value) + + # Literal reference naming. + + def get_literal_name(self): + + "Add a new literal to the current namespace." + + path = self.get_namespace_path() + init_item(self.literals, path, lambda: 0) + return "$C%d" % self.literals[path] + + def next_literal(self): + self.literals[self.get_namespace_path()] += 1 + + # Temporary iterator naming. + + def get_iterator_path(self): + return self.in_function and self.get_namespace_path() or self.name + + def get_iterator_name(self): + path = self.get_iterator_path() + init_item(self.iterators, path, lambda: 0) + return "$i%d" % self.iterators[path] + + def next_iterator(self): + self.iterators[self.get_iterator_path()] += 1 + + # Temporary variable naming. + + def get_temporary_name(self): + path = self.get_namespace_path() + init_item(self.temp, path, lambda: 0) + return "$t%d" % self.temp[path] + + def next_temporary(self): + self.temp[self.get_namespace_path()] += 1 + + # Arbitrary function naming. + + def get_lambda_name(self): + path = self.get_namespace_path() + init_item(self.lambdas, path, lambda: 0) + name = "$l%d" % self.lambdas[path] + self.lambdas[path] += 1 + return name + + def reset_lambdas(self): + self.lambdas = {} + + # Constant and literal recording. + + def get_constant_reference(self, ref, value): + + "Return a constant reference for the given 'ref' type and 'value'." + + constant_name = self.get_constant_name(value) + + # Return a reference for the constant. + + objpath = self.get_object_path(constant_name) + name_ref = ConstantValueRef(constant_name, ref.instance_of(), value) + + # Record the value and type for the constant. + + self.constant_values[objpath] = name_ref.value, name_ref.get_origin() + return name_ref + + def get_literal_reference(self, name, ref, items, cls): + + # Construct an invocation using the items as arguments. + + typename = "$L%s" % name + + invocation = compiler.ast.CallFunc( + compiler.ast.Name(typename), + items + ) + + # Get a name for the actual literal. + + instname = self.get_literal_name() + self.next_literal() + + # Record the type for the literal. + + objpath = self.get_object_path(instname) + self.literal_types[objpath] = ref.get_origin() + + # Return a wrapper for the invocation exposing the items. + + return cls( + instname, + ref.instance_of(), + self.process_structure_node(invocation), + invocation.args + ) + + # Node handling. + + def process_structure(self, node): + + """ + Within the given 'node', process the program structure. + + During inspection, this will process global declarations, adjusting the + module namespace, and import statements, building a module dependency + hierarchy. + + During translation, this will consult deduced program information and + output translated code. + """ + + l = [] + for n in node.getChildNodes(): + l.append(self.process_structure_node(n)) + return l + + def process_augassign_node(self, n): + + "Process the given augmented assignment node 'n'." + + op = operator_functions[n.op] + + if isinstance(n.node, compiler.ast.Getattr): + target = compiler.ast.AssAttr(n.node.expr, n.node.attrname, "OP_ASSIGN") + elif isinstance(n.node, compiler.ast.Name): + target = compiler.ast.AssName(n.node.name, "OP_ASSIGN") + else: + target = n.node + + assignment = compiler.ast.Assign( + [target], + compiler.ast.CallFunc( + compiler.ast.Name("$op%s" % op), + [n.node, n.expr])) + + return self.process_structure_node(assignment) + + def process_assignment_for_function(self, original_name, name): + + """ + Return an assignment operation making 'original_name' refer to the given + 'name'. + """ + + assignment = compiler.ast.Assign( + [compiler.ast.AssName(original_name, "OP_ASSIGN")], + compiler.ast.Name(name) + ) + + return self.process_structure_node(assignment) + + def process_assignment_node_items(self, n, expr): + + """ + Process the given assignment node 'n' whose children are to be assigned + items of 'expr'. + """ + + name_ref = self.process_structure_node(expr) + + # Either unpack the items and present them directly to each assignment + # node. + + if isinstance(name_ref, LiteralSequenceRef): + self.process_literal_sequence_items(n, name_ref) + + # Or have the assignment nodes access each item via the sequence API. + + else: + self.process_assignment_node_items_by_position(n, expr, name_ref) + + def process_assignment_node_items_by_position(self, n, expr, name_ref): + + """ + Process the given sequence assignment node 'n', converting the node to + the separate assignment of each target using positional access on a + temporary variable representing the sequence. Use 'expr' as the assigned + value and 'name_ref' as the reference providing any existing temporary + variable. + """ + + assignments = [] + + if isinstance(name_ref, NameRef): + temp = name_ref.name + else: + temp = self.get_temporary_name() + self.next_temporary() + + assignments.append( + compiler.ast.Assign([compiler.ast.AssName(temp, "OP_ASSIGN")], expr) + ) + + for i, node in enumerate(n.nodes): + assignments.append( + compiler.ast.Assign([node], compiler.ast.Subscript( + compiler.ast.Name(temp), "OP_APPLY", [compiler.ast.Const(i)])) + ) + + return self.process_structure_node(compiler.ast.Stmt(assignments)) + + def process_literal_sequence_items(self, n, name_ref): + + """ + Process the given assignment node 'n', obtaining from the given + 'name_ref' the items to be assigned to the assignment targets. + """ + + if len(n.nodes) == len(name_ref.items): + for node, item in zip(n.nodes, name_ref.items): + self.process_assignment_node(node, item) + else: + raise InspectError("In %s, item assignment needing %d items is given %d items." % ( + self.get_namespace_path(), len(n.nodes), len(name_ref.items))) + + def process_compare_node(self, n): + + """ + Process the given comparison node 'n', converting an operator sequence + from... + + + + ...to... + + (, ) and (, ) + """ + + invocations = [] + last = n.expr + + for op, op_node in n.ops: + op = operator_functions.get(op) + + invocations.append(compiler.ast.CallFunc( + compiler.ast.Name("$op%s" % op), + [last, op_node])) + + last = op_node + + if len(invocations) > 1: + result = compiler.ast.And(invocations) + else: + result = invocations[0] + + return self.process_structure_node(result) + + def process_dict_node(self, node): + + """ + Process the given dictionary 'node', returning a list of (key, value) + tuples. + """ + + l = [] + for key, value in node.items: + l.append(( + self.process_structure_node(key), + self.process_structure_node(value))) + return l + + def process_for_node(self, n): + + """ + Generate attribute accesses for {n.list}.__iter__ and the next method on + the iterator, producing a replacement node for the original. + """ + + node = compiler.ast.Stmt([ + + # = {n.list}.__iter__ + + compiler.ast.Assign( + [compiler.ast.AssName(self.get_iterator_name(), "OP_ASSIGN")], + compiler.ast.CallFunc( + compiler.ast.Getattr(n.list, "__iter__"), + [] + )), + + # try: + # while True: + # ... = .next() + # ... + # except StopIteration: + # pass + + compiler.ast.TryExcept( + compiler.ast.While( + compiler.ast.Name("True"), + compiler.ast.Stmt([ + compiler.ast.Assign( + [n.assign], + compiler.ast.CallFunc( + compiler.ast.Getattr(compiler.ast.Name(self.get_iterator_name()), "next"), + [] + )), + n.body]), + None), + [(compiler.ast.Name("StopIteration"), None, compiler.ast.Stmt([compiler.ast.Pass()]))], + None) + ]) + + self.next_iterator() + self.process_structure_node(node) + + def convert_ifexp_node(self, n): + + """ + Convert the given if expression node 'n'. An if expression is considered + as mapping to a function body containing an if statement as follows: + + if else + + lambda : + if : + return + else: + return + + The are populated with defaults after the node has been + processed. + """ + + node = compiler.ast.Lambda( + [], [], 0, + compiler.ast.If([ + (n.test, compiler.ast.Return(n.then)) + ], + compiler.ast.Return(n.else_) + )) + + return node + + def convert_listcomp_node(self, n): + + """ + Convert the given list comprehension node 'n'. A list comprehension is + considered as mapping to a function body containing a for loop as + follows: + + [ for in if for in if if ] + + lambda : + = [] + for in : + if : + for in : + if : + if : + .append() + return + + The are populated with defaults after the node has been + processed. + """ + + temp = "$tr" + + node = compiler.ast.Lambda( + [], [], 0, + compiler.ast.Stmt([ + + # = [] + + compiler.ast.Assign([compiler.ast.AssName(temp, "OP_ASSIGN")], + compiler.ast.List([]) + ), + + # for ... + + self.convert_listcomp_for_node(n.quals[0], n.quals[1:], n.expr, temp), + + # return + + compiler.ast.Return(compiler.ast.Name(temp)) + ])) + + return node + + def convert_listcomp_for_node(self, loop, following_loops, expr, temp): + + """ + Return a node representing 'loop', encapsulating 'following_loops' and + employing 'expr' in the innermost loop body appending to 'temp'. + """ + + if loop.ifs: + body = self.convert_listcomp_if_node(loop.ifs[0], loop.ifs[1:], following_loops, expr, temp) + elif following_loops: + body = self.convert_listcomp_for_node(following_loops[0], following_loops[1:], expr, temp) + else: + body = self.convert_listcomp_body_node(expr, temp) + + return compiler.ast.For(loop.assign, loop.list, compiler.ast.Stmt([body]), None) + + def convert_listcomp_if_node(self, if_, following_ifs, following_loops, expr, temp): + + """ + Return a node representing 'if_', encapsulating the 'following_ifs' and + 'following_loops' and employing 'expr' in the innermost loop body + appending to 'temp'. + """ + + if following_ifs: + body = self.convert_listcomp_if_node(following_ifs[0], following_ifs[1:], following_loops, expr, temp) + elif following_loops: + body = self.convert_listcomp_for_node(following_loops[0], following_loops[1:], expr, temp) + else: + body = self.convert_listcomp_body_node(expr, temp) + + return compiler.ast.If([(if_.test, compiler.ast.Stmt([body]))], None) + + def convert_listcomp_body_node(self, expr, temp): + + "Return a node appending 'expr' to 'temp'." + + return compiler.ast.Discard( + compiler.ast.CallFunc( + compiler.ast.Getattr(compiler.ast.Name(temp), "append"), [expr])) + + def process_literal_sequence_node(self, n, name, ref, cls): + + """ + Process the given literal sequence node 'n' as a function invocation, + with 'name' indicating the type of the sequence, and 'ref' being a + reference to the type. The 'cls' is used to instantiate a suitable name + reference. + """ + + if name == "dict": + items = [] + for key, value in n.items: + items.append(compiler.ast.Tuple([key, value])) + else: # name in ("list", "tuple"): + items = n.nodes + + return self.get_literal_reference(name, ref, items, cls) + + def process_operator_node(self, n): + + """ + Process the given operator node 'n' as an operator function invocation. + """ + + op = operator_functions[n.__class__.__name__] + invocation = compiler.ast.CallFunc( + compiler.ast.Name("$op%s" % op), + list(n.getChildNodes()) + ) + return self.process_structure_node(invocation) + + def process_slice_node(self, n, expr=None): + + """ + Process the given slice node 'n' as an operator function invocation. + """ + + op = n.flags == "OP_ASSIGN" and "setslice" or "getslice" + invocation = compiler.ast.CallFunc( + compiler.ast.Name("$op%s" % op), + [n.expr, n.lower or compiler.ast.Name("None"), n.upper or compiler.ast.Name("None")] + + (expr and [expr] or []) + ) + return self.process_structure_node(invocation) + + def process_sliceobj_node(self, n): + + """ + Process the given slice object node 'n' as a slice constructor. + """ + + op = "slice" + invocation = compiler.ast.CallFunc( + compiler.ast.Name("$op%s" % op), + n.nodes + ) + return self.process_structure_node(invocation) + + def process_subscript_node(self, n, expr=None): + + """ + Process the given subscript node 'n' as an operator function invocation. + """ + + op = n.flags == "OP_ASSIGN" and "setitem" or "getitem" + invocation = compiler.ast.CallFunc( + compiler.ast.Name("$op%s" % op), + [n.expr] + list(n.subs) + (expr and [expr] or []) + ) + return self.process_structure_node(invocation) + + def process_attribute_chain(self, n): + + """ + Process the given attribute access node 'n'. Return a reference + describing the expression. + """ + + # AssAttr/Getattr are nested with the outermost access being the last + # access in any chain. + + self.attrs.insert(0, n.attrname) + attrs = self.attrs + + # Break attribute chains where non-access nodes are found. + + if not self.have_access_expression(n): + self.attrs = [] + + # Descend into the expression, extending backwards any existing chain, + # or building another for the expression. + + name_ref = self.process_structure_node(n.expr) + + # Restore chain information applying to this node. + + self.attrs = attrs + + # Return immediately if the expression was another access and thus a + # continuation backwards along the chain. The above processing will + # have followed the chain all the way to its conclusion. + + if self.have_access_expression(n): + del self.attrs[0] + + return name_ref + + def have_access_expression(self, node): + + "Return whether the expression associated with 'node' is Getattr." + + return isinstance(node.expr, compiler.ast.Getattr) + + def get_name_for_tracking(self, name, path=None): + + """ + Return the name to be used for attribute usage observations involving + the given 'name' in the current namespace. If 'path' is indicated and + the name is being used outside a function, return the path value; + otherwise, return a path computed using the current namespace and the + given name. + + The intention of this method is to provide a suitably-qualified name + that can be tracked across namespaces. Where globals are being + referenced in class namespaces, they should be referenced using their + path within the module, not using a path within each class. + + It may not be possible to identify a global within a function at the + time of inspection (since a global may appear later in a file). + Consequently, globals are identified by their local name rather than + their module-qualified path. + """ + + # For functions, use the appropriate local names. + + if self.in_function: + return name + + # For static namespaces, use the given qualified name. + + elif path: + return path + + # Otherwise, establish a name in the current (module) namespace. + + else: + return self.get_object_path(name) + + def get_path_for_access(self): + + "Outside functions, register accesses at the module level." + + if not self.in_function: + return self.name + else: + return self.get_namespace_path() + + def get_module_name(self, node): + + """ + Using the given From 'node' in this module, calculate any relative import + information, returning a tuple containing a module to import along with any + names to import based on the node's name information. + + Where the returned module is given as None, whole module imports should + be performed for the returned modules using the returned names. + """ + + # Absolute import. + + if node.level == 0: + return node.modname, node.names + + # Relative to an ancestor of this module. + + else: + path = self.name.split(".") + level = node.level + + # Relative imports treat package roots as submodules. + + if split(self.filename)[-1] == "__init__.py": + level -= 1 + + if level > len(path): + raise InspectError("Relative import %r involves too many levels up from module %r" % ( + ("%s%s" % ("." * node.level, node.modname or "")), self.name)) + + basename = ".".join(path[:len(path)-level]) + + # Name imports from a module. + + if node.modname: + return "%s.%s" % (basename, node.modname), node.names + + # Relative whole module imports. + + else: + return basename, node.names + +def get_argnames(args): + + """ + Return a list of all names provided by 'args'. Since tuples may be + employed, the arguments are traversed depth-first. + """ + + l = [] + for arg in args: + if isinstance(arg, tuple): + l += get_argnames(arg) + else: + l.append(arg) + return l + +# Classes representing inspection and translation observations. + +class Result: + + "An abstract expression result." + + def is_name(self): + return False + def get_origin(self): + return None + +class NameRef(Result): + + "A reference to a name." + + def __init__(self, name, expr=None): + self.name = name + self.expr = expr + + def is_name(self): + return True + + def reference(self): + return None + + def final(self): + return None + + def __repr__(self): + return "NameRef(%r, %r)" % (self.name, self.expr) + +class LocalNameRef(NameRef): + + "A reference to a local name." + + def __init__(self, name, number): + NameRef.__init__(self, name) + self.number = number + + def __repr__(self): + return "LocalNameRef(%r, %r)" % (self.name, self.number) + +class ResolvedNameRef(NameRef): + + "A resolved name-based reference." + + def __init__(self, name, ref, expr=None): + NameRef.__init__(self, name, expr) + self.ref = ref + + def reference(self): + return self.ref + + def get_name(self): + return self.ref and self.ref.get_name() or None + + def get_origin(self): + return self.ref and self.ref.get_origin() or None + + def static(self): + return self.ref and self.ref.static() or None + + def final(self): + return self.ref and self.ref.final() or None + + def has_kind(self, kinds): + return self.ref and self.ref.has_kind(kinds) + + def __repr__(self): + return "ResolvedNameRef(%r, %r, %r)" % (self.name, self.ref, self.expr) + +class ConstantValueRef(ResolvedNameRef): + + "A constant reference representing a single literal value." + + def __init__(self, name, ref, value, number=None): + ResolvedNameRef.__init__(self, name, ref) + self.value = value + self.number = number + + def __repr__(self): + return "ConstantValueRef(%r, %r, %r, %r)" % (self.name, self.ref, self.value, self.number) + +class InstanceRef(Result): + + "An instance reference." + + def __init__(self, ref): + self.ref = ref + + def reference(self): + return self.ref + + def __repr__(self): + return "InstanceRef(%r)" % self.ref + +class LiteralSequenceRef(ResolvedNameRef): + + "A reference representing a sequence of values." + + def __init__(self, name, ref, node, items=None): + ResolvedNameRef.__init__(self, name, ref) + self.node = node + self.items = items + + def __repr__(self): + return "LiteralSequenceRef(%r, %r, %r, %r)" % (self.name, self.ref, self.node, self.items) + +# Dictionary utilities. + +def init_item(d, key, fn): + + """ + Add to 'd' an entry for 'key' using the callable 'fn' to make an initial + value where no entry already exists. + """ + + if not d.has_key(key): + d[key] = fn() + return d[key] + +def dict_for_keys(d, keys): + + "Return a new dictionary containing entries from 'd' for the given 'keys'." + + nd = {} + for key in keys: + if d.has_key(key): + nd[key] = d[key] + return nd + +def make_key(s): + + "Make sequence 's' into a tuple-based key, first sorting its contents." + + l = list(s) + l.sort() + return tuple(l) + +def add_counter_item(d, key): + + """ + Make a mapping in 'd' for 'key' to the number of keys added before it, thus + maintaining a mapping of keys to their order of insertion. + """ + + if not d.has_key(key): + d[key] = len(d.keys()) + return d[key] + +def remove_items(d1, d2): + + "Remove from 'd1' all items from 'd2'." + + for key in d2.keys(): + if d1.has_key(key): + del d1[key] + +# Set utilities. + +def first(s): + return list(s)[0] + +def same(s1, s2): + return set(s1) == set(s2) + +# General input/output. + +def readfile(filename): + + "Return the contents of 'filename'." + + f = open(filename) + try: + return f.read() + finally: + f.close() + +def writefile(filename, s): + + "Write to 'filename' the string 's'." + + f = open(filename, "w") + try: + f.write(s) + finally: + f.close() + +# General encoding. + +def sorted_output(x): + + "Sort sequence 'x' and return a string with commas separating the values." + + x = map(str, x) + x.sort() + return ", ".join(x) + +# Attribute chain decoding. + +def get_attrnames(attrnames): + if attrnames.startswith("#"): + return [attrnames] + else: + return attrnames.split(".") + +def get_attrname_from_location(location): + path, name, attrnames, access = location + return get_attrnames(attrnames)[0] + +# Useful data. + +predefined_constants = "Ellipsis", "False", "None", "NotImplemented", "True" + +operator_functions = { + + # Fundamental operations. + + "is" : "is_", + "is not" : "is_not", + + # Binary operations. + + "in" : "in_", + "not in" : "not_in", + "Add" : "add", + "Bitand" : "and_", + "Bitor" : "or_", + "Bitxor" : "xor", + "Div" : "div", + "FloorDiv" : "floordiv", + "LeftShift" : "lshift", + "Mod" : "mod", + "Mul" : "mul", + "Power" : "pow", + "RightShift" : "rshift", + "Sub" : "sub", + + # Unary operations. + + "Invert" : "invert", + "UnaryAdd" : "pos", + "UnarySub" : "neg", + + # Augmented assignment. + + "+=" : "iadd", + "-=" : "isub", + "*=" : "imul", + "/=" : "idiv", + "//=" : "ifloordiv", + "%=" : "imod", + "**=" : "ipow", + "<<=" : "ilshift", + ">>=" : "irshift", + "&=" : "iand", + "^=" : "ixor", + "|=" : "ior", + + # Comparisons. + + "==" : "eq", + "!=" : "ne", + "<" : "lt", + "<=" : "le", + ">=" : "ge", + ">" : "gt", + } + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa compiler/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/__init__.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,14 @@ +"""Package for parsing and compiling Python source code + +There are several functions defined at the top level that are imported +from modules contained in the package. + +parse(buf, mode="exec") -> AST + Converts a string containing Python source code to an abstract + syntax tree (AST). The AST is defined in compiler.ast. + +parseFile(path) -> AST + The same as parse(open(path)) +""" + +from compiler.transformer import parse, parseFile diff -r 000000000000 -r 5366b587e3fa compiler/ast.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/ast.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,1786 @@ +"""Python abstract syntax node definitions + +This file was originally generated by Tools/compiler/astgen.py +""" +from compiler.consts import CO_VARARGS, CO_VARKEYWORDS + +def flatten(seq): + l = [] + for elt in seq: + if isinstance(elt, (tuple, list)): + for elt2 in flatten(elt): + l.append(elt2) + else: + l.append(elt) + return l + +def flatten_nodes(seq): + return [n for n in flatten(seq) if isinstance(n, Node)] + +def flatten_statement(seq): + l = [] + for elt in seq: + if isinstance(elt, Stmt): + l += flatten_statement(elt) + else: + l.append(elt) + return l + +def flatten_assignment(node): + l = [] + if isinstance(node, (AssList, AssTuple)): + for n in node.nodes: + l += flatten_assignment(n) + else: + l.append(node) + return l + +def is_deletion(node): + return isinstance(node, (AssAttr, AssName)) and node.flags == "OP_DELETE" + +def docstring(s): + if s.find("\n") != -1: + if s.find("'''") != -1: + return '"""%s"""' % s.replace('"""', '\\"\\"\\"') + else: + return "'''%s'''" % s.replace("'''", "\\'\\'\\'") + else: + return repr(s) + +def indent(s): + return s.replace("\n", "\n\t") + +def get_defaults(node): + + "Return a list of (argument name, default) tuples for the 'node'." + + star = (node.flags & 4 != 0) and 1 or 0 + dstar = (node.flags & 8 != 0) and 1 or 0 + argnames = node.argnames[:] + + # Add stars to star and dstar parameters. + + if star: + argnames[-dstar-star] = "*%s" % argnames[-dstar-star] + if dstar: + argnames[-dstar] = "**%s" % argnames[-dstar] + + # Map defaults to parameters. + + defaults = [None] * (len(node.argnames) - star - dstar - len(node.defaults)) + list(node.defaults) + [None] * (star + dstar) + return zip(argnames, defaults) + +def decode_function(node): + return [(default and "%s=%s" % (argname, default) or argname) for (argname, default) in get_defaults(node)] + +nodes = {} + +class OperatorUser: + + "Operator-related node." + + pass + +class Operator(OperatorUser): + + "Operator node." + + pass + +class Node: + + "Abstract base class for ast nodes." + + def getChildren(self): + pass # implemented by subclasses + + def __iter__(self): + for n in self.getChildren(): + yield n + + def getChildNodes(self): + pass # implemented by subclasses + +class EmptyNode(Node): + pass + +class Expression(Node): + # Expression is an artificial node class to support "eval" + nodes["expression"] = "Expression" + def __init__(self, node): + self.node = node + + def getChildren(self): + return self.node, + + def getChildNodes(self): + return self.node, + + def __repr__(self): + return "Expression(%r)" % (self.node,) + + def __str__(self): + return str(self.node) + +class Add(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Add((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s + %s)" % (self.left, self.right) + +class And(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "And(%r)" % (self.nodes,) + + def __str__(self): + return "(%s)" % " and ".join(map(str, self.nodes)) + +class AssAttr(Node): + def __init__(self, expr, attrname, flags, lineno=None): + self.expr = expr + self.attrname = attrname + self.flags = flags + self.lineno = lineno + + def getChildren(self): + return self.expr, self.attrname, self.flags + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "AssAttr(%r, %r, %r)" % (self.expr, self.attrname, self.flags) + + def __str__(self): + return "%s%s.%s" % (self.flags == "OP_DELETE" and "del " or "", self.expr, self.attrname) + +class AssList(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "AssList(%r)" % (self.nodes,) + + def __str__(self): + nodes = flatten_assignment(self) + if nodes and is_deletion(nodes[0]): + return "; ".join(map(str, self.nodes)) + else: + return "[%s]" % ", ".join(map(str, self.nodes)) + +class AssName(Node): + def __init__(self, name, flags, lineno=None): + self.name = name + self.flags = flags + self.lineno = lineno + + def getChildren(self): + return self.name, self.flags + + def getChildNodes(self): + return () + + def __repr__(self): + return "AssName(%r, %r)" % (self.name, self.flags) + + def __str__(self): + return "%s%s" % (self.flags == "OP_DELETE" and "del " or "", self.name) + +class AssTuple(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "AssTuple(%r)" % (self.nodes,) + + def __str__(self): + nodes = flatten_assignment(self) + if nodes and is_deletion(nodes[0]): + return "; ".join(map(str, self.nodes)) + else: + return "(%s)" % ", ".join(map(str, self.nodes)) + +class Assert(Node): + def __init__(self, test, fail, lineno=None): + self.test = test + self.fail = fail + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.test) + children.append(self.fail) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.test) + if self.fail is not None: + nodelist.append(self.fail) + return tuple(nodelist) + + def __repr__(self): + return "Assert(%r, %r)" % (self.test, self.fail) + + def __str__(self): + return "assert %s%s" % (self.test, self.fail and ", %s" % self.fail or "") + +class Assign(Node): + def __init__(self, nodes, expr, lineno=None): + self.nodes = nodes + self.expr = expr + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.nodes)) + children.append(self.expr) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + nodelist.append(self.expr) + return tuple(nodelist) + + def __repr__(self): + return "Assign(%r, %r)" % (self.nodes, self.expr) + + def __str__(self): + return "%s = %s" % (", ".join(map(str, self.nodes)), self.expr) + +class AugAssign(Node, OperatorUser): + def __init__(self, node, op, expr, lineno=None): + self.node = node + self.op = op + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.node, self.op, self.expr + + def getChildNodes(self): + return self.node, self.expr + + def __repr__(self): + return "AugAssign(%r, %r, %r)" % (self.node, self.op, self.expr) + + def __str__(self): + return "%s %s %s" % (self.node, self.op, self.expr) + +class Backquote(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Backquote(%r)" % (self.expr,) + + def __str__(self): + return "`%s`" % self.expr + +class Bitand(Node, Operator): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Bitand(%r)" % (self.nodes,) + + def __str__(self): + return "(%s)" % " & ".join(map(str, self.nodes)) + +class Bitor(Node, Operator): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Bitor(%r)" % (self.nodes,) + + def __str__(self): + return "(%s)" % " | ".join(map(str, self.nodes)) + +class Bitxor(Node, Operator): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Bitxor(%r)" % (self.nodes,) + + def __str__(self): + return "(%s)" % " ^ ".join(map(str, self.nodes)) + +class Break(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Break()" + + def __str__(self): + return "break" + +class CallFunc(Node): + def __init__(self, node, args, star_args = None, dstar_args = None, lineno=None): + self.node = node + self.args = args + self.star_args = star_args + self.dstar_args = dstar_args + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.node) + children.extend(flatten(self.args)) + children.append(self.star_args) + children.append(self.dstar_args) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.node) + nodelist.extend(flatten_nodes(self.args)) + if self.star_args is not None: + nodelist.append(self.star_args) + if self.dstar_args is not None: + nodelist.append(self.dstar_args) + return tuple(nodelist) + + def __repr__(self): + args = [] + if self.dstar_args: + args.insert(0, repr(self.dstar_args)) + if args or self.star_args: + args.insert(0, repr(self.star_args)) + return "CallFunc(%r, %r%s)" % (self.node, self.args, args and (", %s" % ", ".join(args)) or "") + + def __str__(self): + star_args = self.star_args and ["*%s" % self.star_args] or [] + dstar_args = self.dstar_args and ["**%s" % self.dstar_args] or [] + return "%s(%s)" % (self.node, ", ".join(map(str, self.args + star_args + dstar_args))) + +class Class(Node): + def __init__(self, name, bases, doc, code, decorators = None, lineno=None): + self.name = name + self.bases = bases + self.doc = doc + self.code = code + self.decorators = decorators + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.name) + children.extend(flatten(self.bases)) + children.append(self.doc) + children.append(self.code) + children.append(self.decorators) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.bases)) + nodelist.append(self.code) + if self.decorators is not None: + nodelist.append(self.decorators) + return tuple(nodelist) + + def __repr__(self): + return "Class(%r, %r, %r, %r, %r)" % (self.name, self.bases, self.doc, self.code, self.decorators) + + def __str__(self): + return "%sclass %s%s:%s%s\n" % ( + self.decorators and "%s\n" % "\n".join([("@%s" % decorator) for decorator in self.decorators]) or "", + self.name, + self.bases and "(%s)" % ", ".join(map(str, self.bases)) or "", + self.doc and "\n\t" + docstring(self.doc) or "", + indent("\n%s" % self.code) + ) + +class Compare(Node, OperatorUser): + def __init__(self, expr, ops, lineno=None): + self.expr = expr + self.ops = ops + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.ops)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.ops)) + return tuple(nodelist) + + def __repr__(self): + return "Compare(%r, %r)" % (self.expr, self.ops) + + def __str__(self): + return "%s %s" % (self.expr, " ".join([("%s %s" % op) for op in self.ops])) + +class Const(Node): + def __init__(self, value, lineno=None): + self.value = value + self.lineno = lineno + + def getChildren(self): + return self.value, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Const(%r)" % (self.value,) + + def __str__(self): + return repr(self.value) + +class Continue(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Continue()" + + def __str__(self): + return "continue" + +class Decorators(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Decorators(%r)" % (self.nodes,) + + def __str__(self): + return "\n".join([("@%s" % node) for node in self.nodes]) + +class Dict(Node): + def __init__(self, items, lineno=None): + self.items = items + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.items)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.items)) + return tuple(nodelist) + + def __repr__(self): + return "Dict(%r)" % (self.items,) + + def __str__(self): + return "{%s}" % ", ".join([("%s : %s" % (key, value)) for (key, value) in self.items]) + +class Discard(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Discard(%r)" % (self.expr,) + + def __str__(self): + return str(self.expr) + +class Div(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Div((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s / %s)" % (self.left, self.right) + +class Ellipsis(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Ellipsis()" + + def __str__(self): + return "..." + +class Exec(Node): + def __init__(self, expr, locals, globals, lineno=None): + self.expr = expr + self.locals = locals + self.globals = globals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.locals) + children.append(self.globals) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + if self.locals is not None: + nodelist.append(self.locals) + if self.globals is not None: + nodelist.append(self.globals) + return tuple(nodelist) + + def __repr__(self): + return "Exec(%r, %r, %r)" % (self.expr, self.locals, self.globals) + + def __str__(self): + return "exec %s%s%s" % (self.expr, self.locals and "in %s" % self.locals or "", + self.globals and ", %s" % self.globals or "") + +class FloorDiv(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "FloorDiv((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s // %s)" % (self.left, self.right) + +class For(Node): + def __init__(self, assign, list, body, else_, lineno=None): + self.assign = assign + self.list = list + self.body = body + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.assign) + children.append(self.list) + children.append(self.body) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.assign) + nodelist.append(self.list) + nodelist.append(self.body) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "For(%r, %r, %r, %r)" % (self.assign, self.list, self.body, self.else_) + + def __str__(self): + return "for %s in %s:%s%s" % ( + self.assign, self.list, + indent("\n%s" % self.body), + self.else_ and "\nelse:%s" % indent("\n%s" % self.else_) or "" + ) + +class From(Node): + def __init__(self, modname, names, level, lineno=None): + self.modname = modname + self.names = names + self.level = level + self.lineno = lineno + + def getChildren(self): + return self.modname, self.names, self.level + + def getChildNodes(self): + return () + + def __repr__(self): + return "From(%r, %r, %r)" % (self.modname, self.names, self.level) + + def __str__(self): + return "from %s import %s" % (self.modname, + ", ".join([(alias and "%s as %s" % (name, alias) or name) for (name, alias) in self.names])) + +class Function(Node): + def __init__(self, decorators, name, argnames, defaults, flags, doc, code, lineno=None): + self.decorators = decorators + self.name = name + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.doc = doc + self.code = code + self.lineno = lineno + self.varargs = self.kwargs = None + if flags & CO_VARARGS: + self.varargs = 1 + if flags & CO_VARKEYWORDS: + self.kwargs = 1 + + def getChildren(self): + children = [] + children.append(self.decorators) + children.append(self.name) + children.append(self.argnames) + children.extend(flatten(self.defaults)) + children.append(self.flags) + children.append(self.doc) + children.append(self.code) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + if self.decorators is not None: + nodelist.append(self.decorators) + nodelist.extend(flatten_nodes(self.defaults)) + nodelist.append(self.code) + return tuple(nodelist) + + def __repr__(self): + return "Function(%r, %r, %r, %r, %r, %r, %r)" % (self.decorators, self.name, self.argnames, self.defaults, self.flags, self.doc, self.code) + + def __str__(self): + parameters = decode_function(self) + + return "%sdef %s(%s):%s%s\n" % ( + self.decorators and "%s\n" % "\n".join([("@%s" % decorator) for decorator in self.decorators]) or "", + self.name, + ", ".join(parameters), + self.doc and "\n\n\t%s\n" % docstring(self.doc) or "", + indent("\n%s" % self.code) + ) + +class GenExpr(Node): + def __init__(self, code, lineno=None): + self.code = code + self.lineno = lineno + self.argnames = ['.0'] + self.varargs = self.kwargs = None + + def getChildren(self): + return self.code, + + def getChildNodes(self): + return self.code, + + def __repr__(self): + return "GenExpr(%r)" % (self.code,) + + def __str__(self): + return str(self.code) + +class GenExprFor(Node): + def __init__(self, assign, iter, ifs, lineno=None): + self.assign = assign + self.iter = iter + self.ifs = ifs + self.lineno = lineno + self.is_outmost = False + + def getChildren(self): + children = [] + children.append(self.assign) + children.append(self.iter) + children.extend(flatten(self.ifs)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.assign) + nodelist.append(self.iter) + nodelist.extend(flatten_nodes(self.ifs)) + return tuple(nodelist) + + def __repr__(self): + return "GenExprFor(%r, %r, %r)" % (self.assign, self.iter, self.ifs) + + def __str__(self): + return "for %s in %s%s" % ( + self.assign, self.iter, + self.ifs and " ".join(map(str, self.ifs)) or "" + ) + +class GenExprIf(Node): + def __init__(self, test, lineno=None): + self.test = test + self.lineno = lineno + + def getChildren(self): + return self.test, + + def getChildNodes(self): + return self.test, + + def __repr__(self): + return "GenExprIf(%r)" % (self.test,) + + def __str__(self): + return "if %s" % self.test + +class GenExprInner(Node): + def __init__(self, expr, quals, lineno=None): + self.expr = expr + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "GenExprInner(%r, %r)" % (self.expr, self.quals) + + def __str__(self): + return "%s %s" % (self.expr, " ".join(map(str, self.quals))) + +class Getattr(Node): + def __init__(self, expr, attrname, lineno=None): + self.expr = expr + self.attrname = attrname + self.lineno = lineno + + def getChildren(self): + return self.expr, self.attrname + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Getattr(%r, %r)" % (self.expr, self.attrname) + + def __str__(self): + return "%s.%s" % (self.expr, self.attrname) + +class Global(Node): + def __init__(self, names, lineno=None): + self.names = names + self.lineno = lineno + + def getChildren(self): + return self.names, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Global(%r)" % (self.names,) + + def __str__(self): + return "global %s" % ", ".join(map(str, self.names)) + +class If(Node): + def __init__(self, tests, else_, lineno=None): + self.tests = tests + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.tests)) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.tests)) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "If(%r, %r)" % (self.tests, self.else_) + + def __str__(self): + tests = [("%sif %s:%s" % (i > 0 and "el" or "", test, indent("\n%s" % body))) for (i, (test, body)) in enumerate(self.tests)] + return "%s%s" % ( + "\n".join(tests), + self.else_ and "\nelse:%s" % indent("\n%s" % self.else_) or "" + ) + +class IfExp(Node): + def __init__(self, test, then, else_, lineno=None): + self.test = test + self.then = then + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + return self.test, self.then, self.else_ + + def getChildNodes(self): + return self.test, self.then, self.else_ + + def __repr__(self): + return "IfExp(%r, %r, %r)" % (self.test, self.then, self.else_) + + def __str__(self): + return "%s if %s else %s" % (self.then, self.test, self.else_) + +class Import(Node): + def __init__(self, names, lineno=None): + self.names = names + self.lineno = lineno + + def getChildren(self): + return self.names, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Import(%r)" % (self.names,) + + def __str__(self): + return "import %s" % ( + ", ".join([(alias and "%s as %s" % (name, alias) or name) for (name, alias) in self.names])) + +class Invert(Node, Operator): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Invert(%r)" % (self.expr,) + + def __str__(self): + return "~%s" % self.expr + +class Keyword(Node): + def __init__(self, name, expr, lineno=None): + self.name = name + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.name, self.expr + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Keyword(%r, %r)" % (self.name, self.expr) + + def __str__(self): + return "%s=%s" % (self.name, self.expr) + +class Lambda(Node): + def __init__(self, argnames, defaults, flags, code, lineno=None): + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.code = code + self.lineno = lineno + self.varargs = self.kwargs = None + if flags & CO_VARARGS: + self.varargs = 1 + if flags & CO_VARKEYWORDS: + self.kwargs = 1 + + def getChildren(self): + children = [] + children.append(self.argnames) + children.extend(flatten(self.defaults)) + children.append(self.flags) + children.append(self.code) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.defaults)) + nodelist.append(self.code) + return tuple(nodelist) + + def __repr__(self): + return "Lambda(%r, %r, %r, %r)" % (self.argnames, self.defaults, self.flags, self.code) + + def __str__(self): + parameters = decode_function(self) + return "lambda %s: %s" % (", ".join(parameters), self.code) + +class LeftShift(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "LeftShift((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s << %s)" % (self.left, self.right) + +class List(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "List(%r)" % (self.nodes,) + + def __str__(self): + return "[%s]" % ", ".join(map(str, self.nodes)) + +class ListComp(Node): + def __init__(self, expr, quals, lineno=None): + self.expr = expr + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "ListComp(%r, %r)" % (self.expr, self.quals) + + def __str__(self): + return "[%s %s]" % (self.expr, " ".join(map(str, self.quals))) + +class ListCompFor(Node): + def __init__(self, assign, list, ifs, lineno=None): + self.assign = assign + self.list = list + self.ifs = ifs + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.assign) + children.append(self.list) + children.extend(flatten(self.ifs)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.assign) + nodelist.append(self.list) + nodelist.extend(flatten_nodes(self.ifs)) + return tuple(nodelist) + + def __repr__(self): + return "ListCompFor(%r, %r, %r)" % (self.assign, self.list, self.ifs) + + def __str__(self): + return "for %s in %s%s" % ( + self.assign, self.list, + self.ifs and " ".join(map(str, self.ifs)) or "" + ) + +class ListCompIf(Node): + def __init__(self, test, lineno=None): + self.test = test + self.lineno = lineno + + def getChildren(self): + return self.test, + + def getChildNodes(self): + return self.test, + + def __repr__(self): + return "ListCompIf(%r)" % (self.test,) + + def __str__(self): + return " if %s" % self.test + +class SetComp(Node): + def __init__(self, expr, quals, lineno=None): + self.expr = expr + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "SetComp(%r, %r)" % (self.expr, self.quals) + + def __str__(self): + return "{%s %s}" % (self.expr, " ".join(map(str, self.quals))) + +class DictComp(Node): + def __init__(self, key, value, quals, lineno=None): + self.key = key + self.value = value + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.key) + children.append(self.value) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.key) + nodelist.append(self.value) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "DictComp(%r, %r, %r)" % (self.key, self.value, self.quals) + + def __str__(self): + return "{%s : %s %s}" % (self.key, self.value, " ".join(map(str, self.quals))) + +class Mod(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Mod((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s %% %s)" % (self.left, self.right) + +class Module(Node): + def __init__(self, doc, node, lineno=None): + self.doc = doc + self.node = node + self.lineno = lineno + + def getChildren(self): + return self.doc, self.node + + def getChildNodes(self): + return self.node, + + def __repr__(self): + return "Module(%r, %r)" % (self.doc, self.node) + + def __str__(self): + return "%s%s" % (self.doc and "%s\n\n" % docstring(self.doc) or "", self.node) + +class Mul(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Mul((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s * %s)" % (self.left, self.right) + +class Name(Node): + def __init__(self, name, lineno=None): + self.name = name + self.lineno = lineno + + def getChildren(self): + return self.name, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Name(%r)" % (self.name,) + + def __str__(self): + return str(self.name) + +class Not(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Not(%r)" % (self.expr,) + + def __str__(self): + return "not %s" % self.expr + +class Or(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Or(%r)" % (self.nodes,) + + def __str__(self): + return "(%s)" % " or ".join(map(str, self.nodes)) + +class Pass(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Pass()" + + def __str__(self): + return "pass" + +class Power(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Power((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s ** %s)" % (self.left, self.right) + +class Print(Node): + def __init__(self, nodes, dest, lineno=None): + self.nodes = nodes + self.dest = dest + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.nodes)) + children.append(self.dest) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + if self.dest is not None: + nodelist.append(self.dest) + return tuple(nodelist) + + def __repr__(self): + return "Print(%r, %r)" % (self.nodes, self.dest) + + def __str__(self): + dest = self.dest and [">>%s" % self.dest] or [] + return "print %s," % ", ".join(map(str, dest + self.nodes)) + +class Printnl(Node): + def __init__(self, nodes, dest, lineno=None): + self.nodes = nodes + self.dest = dest + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.nodes)) + children.append(self.dest) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + if self.dest is not None: + nodelist.append(self.dest) + return tuple(nodelist) + + def __repr__(self): + return "Printnl(%r, %r)" % (self.nodes, self.dest) + + def __str__(self): + dest = self.dest and [">>%s" % self.dest] or [] + return "print %s" % ", ".join(map(str, dest + self.nodes)) + +class Raise(Node): + def __init__(self, expr1, expr2, expr3, lineno=None): + self.expr1 = expr1 + self.expr2 = expr2 + self.expr3 = expr3 + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr1) + children.append(self.expr2) + children.append(self.expr3) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + if self.expr1 is not None: + nodelist.append(self.expr1) + if self.expr2 is not None: + nodelist.append(self.expr2) + if self.expr3 is not None: + nodelist.append(self.expr3) + return tuple(nodelist) + + def __repr__(self): + return "Raise(%r, %r, %r)" % (self.expr1, self.expr2, self.expr3) + + def __str__(self): + args = self.expr1 and [self.expr1] or [] + args += self.expr2 and [self.expr2] or [] + args += self.expr3 and [self.expr3] or [] + return "raise %s" % ", ".join(map(str, args)) + +class Return(Node): + def __init__(self, value, lineno=None): + self.value = value + self.lineno = lineno + + def getChildren(self): + return self.value, + + def getChildNodes(self): + return self.value, + + def __repr__(self): + return "Return(%r)" % (self.value,) + + def __str__(self): + return "return %s" % self.value + +class RightShift(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "RightShift((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s >> %s)" % (self.left, self.right) + +class Set(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Set(%r)" % (self.nodes,) + + def __str__(self): + return "{%s}" % ", ".join(map(str, self.nodes)) + +class Slice(Node, OperatorUser): + def __init__(self, expr, flags, lower, upper, lineno=None): + self.expr = expr + self.flags = flags + self.lower = lower + self.upper = upper + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.flags) + children.append(self.lower) + children.append(self.upper) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + if self.lower is not None: + nodelist.append(self.lower) + if self.upper is not None: + nodelist.append(self.upper) + return tuple(nodelist) + + def __repr__(self): + return "Slice(%r, %r, %r, %r)" % (self.expr, self.flags, self.lower, self.upper) + + def __str__(self): + args = [self.lower or "", self.upper or ""] + return "%s%s[%s]" % (self.flags == "OP_DELETE" and "del " or "", self.expr, ":".join(map(str, args))) + +class Sliceobj(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Sliceobj(%r)" % (self.nodes,) + + def __str__(self): + return ":".join(map(str, self.nodes)) + +class Stmt(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Stmt(%r)" % (self.nodes,) + + def __str__(self): + return "\n".join(map(str, flatten_statement(self.nodes))) + +class Sub(Node, Operator): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Sub((%r, %r))" % (self.left, self.right) + + def __str__(self): + return "(%s - %s)" % (self.left, self.right) + +class Subscript(Node, OperatorUser): + def __init__(self, expr, flags, subs, lineno=None): + self.expr = expr + self.flags = flags + self.subs = subs + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.flags) + children.extend(flatten(self.subs)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.subs)) + return tuple(nodelist) + + def __repr__(self): + return "Subscript(%r, %r, %r)" % (self.expr, self.flags, self.subs) + + def __str__(self): + return "%s%s[%s]" % (self.flags == "OP_DELETE" and "del " or "", self.expr, ",".join(map(str, self.subs))) + +class TryExcept(Node): + def __init__(self, body, handlers, else_, lineno=None): + self.body = body + self.handlers = handlers + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.body) + children.extend(flatten(self.handlers)) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.body) + nodelist.extend(flatten_nodes(self.handlers)) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "TryExcept(%r, %r, %r)" % (self.body, self.handlers, self.else_) + + def __str__(self): + handlers = [ + ("\nexcept%s%s:%s" % (spec and " %s" % spec or "", assign and ", %s" % assign or "", indent("\n%s" % statement))) + for (spec, assign, statement) in self.handlers + ] + + return "try:%s%s%s" % ( + indent("\n%s" % self.body), + "".join(handlers), + self.else_ and "\nelse:%s" % indent("\n%s" % self.else_) or "" + ) + +class TryFinally(Node): + def __init__(self, body, final, lineno=None): + self.body = body + self.final = final + self.lineno = lineno + + def getChildren(self): + return self.body, self.final + + def getChildNodes(self): + return self.body, self.final + + def __repr__(self): + return "TryFinally(%r, %r)" % (self.body, self.final) + + def __str__(self): + return "try:%s\nfinally:%s" % ( + indent("\n%s" % self.body), + indent("\n%s" % self.final) + ) + +class Tuple(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Tuple(%r)" % (self.nodes,) + + def __str__(self): + return "(%s)" % ", ".join(map(str, self.nodes)) + +class UnaryAdd(Node, Operator): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "UnaryAdd(%r)" % (self.expr,) + + def __str__(self): + return "+%s" % self.expr + +class UnarySub(Node, Operator): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "UnarySub(%r)" % (self.expr,) + + def __str__(self): + return "-%s" % self.expr + +class While(Node): + def __init__(self, test, body, else_, lineno=None): + self.test = test + self.body = body + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.test) + children.append(self.body) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.test) + nodelist.append(self.body) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "While(%r, %r, %r)" % (self.test, self.body, self.else_) + + def __str__(self): + return "while %s:%s%s" % ( + self.test, + indent("\n%s" % self.body), + self.else_ and "\nelse:%s" % indent("\n%s" % self.else_) or "" + ) + +class With(Node): + def __init__(self, expr, vars, body, lineno=None): + self.expr = expr + self.vars = vars + self.body = body + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.vars) + children.append(self.body) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + if self.vars is not None: + nodelist.append(self.vars) + nodelist.append(self.body) + return tuple(nodelist) + + def __repr__(self): + return "With(%r, %r, %r)" % (self.expr, self.vars, self.body) + + def __str__(self): + return "with %s%s:%s" % ( + self.expr, + self.vars and " as %s" % ", ".join(map(str, self.vars)), + indent("\n%s" % self.body), + ) + +class Yield(Node): + def __init__(self, value, lineno=None): + self.value = value + self.lineno = lineno + + def getChildren(self): + return self.value, + + def getChildNodes(self): + return self.value, + + def __repr__(self): + return "Yield(%r)" % (self.value,) + + def __str__(self): + return "yield %s" % self.value + +for name, obj in globals().items(): + if isinstance(obj, type) and issubclass(obj, Node): + nodes[name.lower()] = obj diff -r 000000000000 -r 5366b587e3fa compiler/consts.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/consts.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,23 @@ +# operation flags +OP_ASSIGN = 'OP_ASSIGN' +OP_DELETE = 'OP_DELETE' +OP_APPLY = 'OP_APPLY' + +SC_LOCAL = 1 +SC_GLOBAL_IMPLICIT = 2 +SC_GLOBAL_EXPLICT = 3 +SC_FREE = 4 +SC_CELL = 5 +SC_UNKNOWN = 6 + +CO_OPTIMIZED = 0x0001 +CO_NEWLOCALS = 0x0002 +CO_VARARGS = 0x0004 +CO_VARKEYWORDS = 0x0008 +CO_NESTED = 0x0010 +CO_GENERATOR = 0x0020 +CO_GENERATOR_ALLOWED = 0 +CO_FUTURE_DIVISION = 0x2000 +CO_FUTURE_ABSIMPORT = 0x4000 +CO_FUTURE_WITH_STATEMENT = 0x8000 +CO_FUTURE_PRINT_FUNCTION = 0x10000 diff -r 000000000000 -r 5366b587e3fa compiler/transformer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/transformer.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,1515 @@ +"""Parse tree transformation module. + +Transforms Python source code into an abstract syntax tree (AST) +defined in the ast module. + +The simplest ways to invoke this module are via parse and parseFile. +parse(buf) -> AST +parseFile(path) -> AST +""" + +# Original version written by Greg Stein (gstein@lyra.org) +# and Bill Tutt (rassilon@lima.mudlib.org) +# February 1997. +# +# Modifications and improvements for Python 2.0 by Jeremy Hylton and +# Mark Hammond +# +# Some fixes to try to have correct line number on almost all nodes +# (except Module, Discard and Stmt) added by Sylvain Thenault +# +# Portions of this file are: +# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. +# +# This module is provided under a BSD-ish license. See +# http://www.opensource.org/licenses/bsd-license.html +# and replace OWNER, ORGANIZATION, and YEAR as appropriate. + +from compiler.ast import * +import parser +import symbol +import token + +class WalkerError(StandardError): + pass + +from compiler.consts import CO_VARARGS, CO_VARKEYWORDS +from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY + +def parseFile(path): + f = open(path, "U") + # XXX The parser API tolerates files without a trailing newline, + # but not strings without a trailing newline. Always add an extra + # newline to the file contents, since we're going through the string + # version of the API. + src = f.read() + "\n" + f.close() + return parse(src) + +def parse(buf, mode="exec"): + if mode == "exec" or mode == "single": + return Transformer().parsesuite(buf) + elif mode == "eval": + return Transformer().parseexpr(buf) + else: + raise ValueError("compile() arg 3 must be" + " 'exec' or 'eval' or 'single'") + +def extractLineNo(ast): + if not isinstance(ast[1], tuple): + # get a terminal node + return ast[2] + for child in ast[1:]: + if isinstance(child, tuple): + lineno = extractLineNo(child) + if lineno is not None: + return lineno + +def Node(*args): + kind = args[0] + if kind in nodes: + try: + return nodes[kind](*args[1:]) + except TypeError: + print nodes[kind], len(args), args + raise + else: + raise WalkerError, "Can't find appropriate Node type: %s" % str(args) + #return apply(ast.Node, args) + +class Transformer: + """Utility object for transforming Python parse trees. + + Exposes the following methods: + tree = transform(ast_tree) + tree = parsesuite(text) + tree = parseexpr(text) + tree = parsefile(fileob | filename) + """ + + def __init__(self): + self._dispatch = {} + for value, name in symbol.sym_name.items(): + if hasattr(self, name): + self._dispatch[value] = getattr(self, name) + self._dispatch[token.NEWLINE] = self.com_NEWLINE + self._atom_dispatch = {token.LPAR: self.atom_lpar, + token.LSQB: self.atom_lsqb, + token.LBRACE: self.atom_lbrace, + token.BACKQUOTE: self.atom_backquote, + token.NUMBER: self.atom_number, + token.STRING: self.atom_string, + token.NAME: self.atom_name, + } + self.encoding = None + + def transform(self, tree): + """Transform an AST into a modified parse tree.""" + if not (isinstance(tree, tuple) or isinstance(tree, list)): + tree = parser.st2tuple(tree, line_info=1) + return self.compile_node(tree) + + def parsesuite(self, text): + """Return a modified parse tree for the given suite text.""" + return self.transform(parser.suite(text)) + + def parseexpr(self, text): + """Return a modified parse tree for the given expression text.""" + return self.transform(parser.expr(text)) + + def parsefile(self, file): + """Return a modified parse tree for the contents of the given file.""" + if type(file) == type(''): + file = open(file) + return self.parsesuite(file.read()) + + # -------------------------------------------------------------- + # + # PRIVATE METHODS + # + + def compile_node(self, node): + ### emit a line-number node? + n = node[0] + + if n == symbol.encoding_decl: + self.encoding = node[2] + node = node[1] + n = node[0] + + if n == symbol.single_input: + return self.single_input(node[1:]) + if n == symbol.file_input: + return self.file_input(node[1:]) + if n == symbol.eval_input: + return self.eval_input(node[1:]) + if n == symbol.lambdef: + return self.lambdef(node[1:]) + if n == symbol.funcdef: + return self.funcdef(node[1:]) + if n == symbol.classdef: + return self.classdef(node[1:]) + + raise WalkerError, ('unexpected node type', n) + + def single_input(self, node): + ### do we want to do anything about being "interactive" ? + + # NEWLINE | simple_stmt | compound_stmt NEWLINE + n = node[0][0] + if n != token.NEWLINE: + return self.com_stmt(node[0]) + + return Pass() + + def file_input(self, nodelist): + doc = self.get_docstring(nodelist, symbol.file_input) + if doc is not None: + i = 1 + else: + i = 0 + stmts = [] + for node in nodelist[i:]: + if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: + self.com_append_stmt(stmts, node) + return Module(doc, Stmt(stmts)) + + def eval_input(self, nodelist): + # from the built-in function input() + ### is this sufficient? + return Expression(self.com_node(nodelist[0])) + + def decorator_name(self, nodelist): + listlen = len(nodelist) + assert listlen >= 1 and listlen % 2 == 1 + + item = self.atom_name(nodelist) + i = 1 + while i < listlen: + assert nodelist[i][0] == token.DOT + assert nodelist[i + 1][0] == token.NAME + item = Getattr(item, nodelist[i + 1][1]) + i += 2 + + return item + + def decorator(self, nodelist): + # '@' dotted_name [ '(' [arglist] ')' ] + assert len(nodelist) in (3, 5, 6) + assert nodelist[0][0] == token.AT + assert nodelist[-1][0] == token.NEWLINE + + assert nodelist[1][0] == symbol.dotted_name + funcname = self.decorator_name(nodelist[1][1:]) + + if len(nodelist) > 3: + assert nodelist[2][0] == token.LPAR + expr = self.com_call_function(funcname, nodelist[3]) + else: + expr = funcname + + return expr + + def decorators(self, nodelist): + # decorators: decorator ([NEWLINE] decorator)* NEWLINE + items = [] + for dec_nodelist in nodelist: + assert dec_nodelist[0] == symbol.decorator + items.append(self.decorator(dec_nodelist[1:])) + return Decorators(items) + + def decorated(self, nodelist): + assert nodelist[0][0] == symbol.decorators + if nodelist[1][0] == symbol.funcdef: + n = [nodelist[0]] + list(nodelist[1][1:]) + return self.funcdef(n) + elif nodelist[1][0] == symbol.classdef: + decorators = self.decorators(nodelist[0][1:]) + cls = self.classdef(nodelist[1][1:]) + cls.decorators = decorators + return cls + raise WalkerError() + + def funcdef(self, nodelist): + # -6 -5 -4 -3 -2 -1 + # funcdef: [decorators] 'def' NAME parameters ':' suite + # parameters: '(' [varargslist] ')' + + if len(nodelist) == 6: + assert nodelist[0][0] == symbol.decorators + decorators = self.decorators(nodelist[0][1:]) + else: + assert len(nodelist) == 5 + decorators = None + + lineno = nodelist[-4][2] + name = nodelist[-4][1] + args = nodelist[-3][2] + + if args[0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(args[1:]) + else: + names = defaults = () + flags = 0 + doc = self.get_docstring(nodelist[-1]) + + # code for function + code = self.com_node(nodelist[-1]) + + if doc is not None: + assert isinstance(code, Stmt) + assert isinstance(code.nodes[0], Discard) + del code.nodes[0] + return Function(decorators, name, names, defaults, flags, doc, code, + lineno=lineno) + + def lambdef(self, nodelist): + # lambdef: 'lambda' [varargslist] ':' test + if nodelist[2][0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(nodelist[2][1:]) + else: + names = defaults = () + flags = 0 + + # code for lambda + code = self.com_node(nodelist[-1]) + + return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) + old_lambdef = lambdef + + def classdef(self, nodelist): + # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite + + name = nodelist[1][1] + doc = self.get_docstring(nodelist[-1]) + if nodelist[2][0] == token.COLON: + bases = [] + elif nodelist[3][0] == token.RPAR: + bases = [] + else: + bases = self.com_bases(nodelist[3]) + + # code for class + code = self.com_node(nodelist[-1]) + + if doc is not None: + assert isinstance(code, Stmt) + assert isinstance(code.nodes[0], Discard) + del code.nodes[0] + + return Class(name, bases, doc, code, lineno=nodelist[1][2]) + + def stmt(self, nodelist): + return self.com_stmt(nodelist[0]) + + small_stmt = stmt + flow_stmt = stmt + compound_stmt = stmt + + def simple_stmt(self, nodelist): + # small_stmt (';' small_stmt)* [';'] NEWLINE + stmts = [] + for i in range(0, len(nodelist), 2): + self.com_append_stmt(stmts, nodelist[i]) + return Stmt(stmts) + + def parameters(self, nodelist): + raise WalkerError + + def varargslist(self, nodelist): + raise WalkerError + + def fpdef(self, nodelist): + raise WalkerError + + def fplist(self, nodelist): + raise WalkerError + + def dotted_name(self, nodelist): + raise WalkerError + + def comp_op(self, nodelist): + raise WalkerError + + def trailer(self, nodelist): + raise WalkerError + + def sliceop(self, nodelist): + raise WalkerError + + def argument(self, nodelist): + raise WalkerError + + # -------------------------------------------------------------- + # + # STATEMENT NODES (invoked by com_node()) + # + + def expr_stmt(self, nodelist): + # augassign testlist | testlist ('=' testlist)* + en = nodelist[-1] + exprNode = self.lookup_node(en)(en[1:]) + if len(nodelist) == 1: + return Discard(exprNode, lineno=exprNode.lineno) + if nodelist[1][0] == token.EQUAL: + nodesl = [] + for i in range(0, len(nodelist) - 2, 2): + nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) + return Assign(nodesl, exprNode, lineno=nodelist[1][2]) + else: + lval = self.com_augassign(nodelist[0]) + op = self.com_augassign_op(nodelist[1]) + return AugAssign(lval, op[1], exprNode, lineno=op[2]) + raise WalkerError, "can't get here" + + def print_stmt(self, nodelist): + # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) + items = [] + if len(nodelist) == 1: + start = 1 + dest = None + elif nodelist[1][0] == token.RIGHTSHIFT: + assert len(nodelist) == 3 \ + or nodelist[3][0] == token.COMMA + dest = self.com_node(nodelist[2]) + start = 4 + else: + dest = None + start = 1 + for i in range(start, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + if nodelist[-1][0] == token.COMMA: + return Print(items, dest, lineno=nodelist[0][2]) + return Printnl(items, dest, lineno=nodelist[0][2]) + + def del_stmt(self, nodelist): + return self.com_assign(nodelist[1], OP_DELETE) + + def pass_stmt(self, nodelist): + return Pass(lineno=nodelist[0][2]) + + def break_stmt(self, nodelist): + return Break(lineno=nodelist[0][2]) + + def continue_stmt(self, nodelist): + return Continue(lineno=nodelist[0][2]) + + def return_stmt(self, nodelist): + # return: [testlist] + if len(nodelist) < 2: + return Return(Const(None), lineno=nodelist[0][2]) + return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) + + def yield_stmt(self, nodelist): + expr = self.com_node(nodelist[0]) + return Discard(expr, lineno=expr.lineno) + + def yield_expr(self, nodelist): + if len(nodelist) > 1: + value = self.com_node(nodelist[1]) + else: + value = Const(None) + return Yield(value, lineno=nodelist[0][2]) + + def raise_stmt(self, nodelist): + # raise: [test [',' test [',' test]]] + if len(nodelist) > 5: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + if len(nodelist) > 3: + expr2 = self.com_node(nodelist[3]) + else: + expr2 = None + if len(nodelist) > 1: + expr1 = self.com_node(nodelist[1]) + else: + expr1 = None + return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) + + def import_stmt(self, nodelist): + # import_stmt: import_name | import_from + assert len(nodelist) == 1 + return self.com_node(nodelist[0]) + + def import_name(self, nodelist): + # import_name: 'import' dotted_as_names + return Import(self.com_dotted_as_names(nodelist[1]), + lineno=nodelist[0][2]) + + def import_from(self, nodelist): + # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | + # '(' import_as_names ')' | import_as_names) + assert nodelist[0][1] == 'from' + idx = 1 + while nodelist[idx][1] == '.': + idx += 1 + level = idx - 1 + if nodelist[idx][0] == symbol.dotted_name: + fromname = self.com_dotted_name(nodelist[idx]) + idx += 1 + else: + fromname = "" + assert nodelist[idx][1] == 'import' + if nodelist[idx + 1][0] == token.STAR: + return From(fromname, [('*', None)], level, + lineno=nodelist[0][2]) + else: + node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)] + return From(fromname, self.com_import_as_names(node), level, + lineno=nodelist[0][2]) + + def global_stmt(self, nodelist): + # global: NAME (',' NAME)* + names = [] + for i in range(1, len(nodelist), 2): + names.append(nodelist[i][1]) + return Global(names, lineno=nodelist[0][2]) + + def exec_stmt(self, nodelist): + # exec_stmt: 'exec' expr ['in' expr [',' expr]] + expr1 = self.com_node(nodelist[1]) + if len(nodelist) >= 4: + expr2 = self.com_node(nodelist[3]) + if len(nodelist) >= 6: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + else: + expr2 = expr3 = None + + return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) + + def assert_stmt(self, nodelist): + # 'assert': test, [',' test] + expr1 = self.com_node(nodelist[1]) + if (len(nodelist) == 4): + expr2 = self.com_node(nodelist[3]) + else: + expr2 = None + return Assert(expr1, expr2, lineno=nodelist[0][2]) + + def if_stmt(self, nodelist): + # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] + tests = [] + for i in range(0, len(nodelist) - 3, 4): + testNode = self.com_node(nodelist[i + 1]) + suiteNode = self.com_node(nodelist[i + 3]) + tests.append((testNode, suiteNode)) + + if len(nodelist) % 4 == 3: + elseNode = self.com_node(nodelist[-1]) +## elseNode.lineno = nodelist[-1][1][2] + else: + elseNode = None + return If(tests, elseNode, lineno=nodelist[0][2]) + + def while_stmt(self, nodelist): + # 'while' test ':' suite ['else' ':' suite] + + testNode = self.com_node(nodelist[1]) + bodyNode = self.com_node(nodelist[3]) + + if len(nodelist) > 4: + elseNode = self.com_node(nodelist[6]) + else: + elseNode = None + + return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) + + def for_stmt(self, nodelist): + # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] + + assignNode = self.com_assign(nodelist[1], OP_ASSIGN) + listNode = self.com_node(nodelist[3]) + bodyNode = self.com_node(nodelist[5]) + + if len(nodelist) > 8: + elseNode = self.com_node(nodelist[8]) + else: + elseNode = None + + return For(assignNode, listNode, bodyNode, elseNode, + lineno=nodelist[0][2]) + + def try_stmt(self, nodelist): + return self.com_try_except_finally(nodelist) + + def with_stmt(self, nodelist): + return self.com_with(nodelist) + + def suite(self, nodelist): + # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT + if len(nodelist) == 1: + return self.com_stmt(nodelist[0]) + + stmts = [] + for node in nodelist: + if node[0] == symbol.stmt: + self.com_append_stmt(stmts, node) + return Stmt(stmts) + + # -------------------------------------------------------------- + # + # EXPRESSION NODES (invoked by com_node()) + # + + def testlist(self, nodelist): + # testlist: expr (',' expr)* [','] + # testlist_safe: test [(',' test)+ [',']] + # exprlist: expr (',' expr)* [','] + return self.com_binary(Tuple, nodelist) + + testlist_safe = testlist # XXX + testlist1 = testlist + exprlist = testlist + + def testlist_comp(self, nodelist): + # test ( comp_for | (',' test)* [','] ) + assert nodelist[0][0] == symbol.test + if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for: + test = self.com_node(nodelist[0]) + return self.com_generator_expression(test, nodelist[1]) + return self.testlist(nodelist) + + def test(self, nodelist): + # or_test ['if' or_test 'else' test] | lambdef + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: + return self.lambdef(nodelist[0]) + then = self.com_node(nodelist[0]) + if len(nodelist) > 1: + assert len(nodelist) == 5 + assert nodelist[1][1] == 'if' + assert nodelist[3][1] == 'else' + test = self.com_node(nodelist[2]) + else_ = self.com_node(nodelist[4]) + return IfExp(test, then, else_, lineno=nodelist[1][2]) + return then + + def or_test(self, nodelist): + # and_test ('or' and_test)* | lambdef + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: + return self.lambdef(nodelist[0]) + return self.com_binary(Or, nodelist) + old_test = or_test + + def and_test(self, nodelist): + # not_test ('and' not_test)* + return self.com_binary(And, nodelist) + + def not_test(self, nodelist): + # 'not' not_test | comparison + result = self.com_node(nodelist[-1]) + if len(nodelist) == 2: + return Not(result, lineno=nodelist[0][2]) + return result + + def comparison(self, nodelist): + # comparison: expr (comp_op expr)* + node = self.com_node(nodelist[0]) + if len(nodelist) == 1: + return node + + results = [] + for i in range(2, len(nodelist), 2): + nl = nodelist[i-1] + + # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' + # | 'in' | 'not' 'in' | 'is' | 'is' 'not' + n = nl[1] + if n[0] == token.NAME: + type = n[1] + if len(nl) == 3: + if type == 'not': + type = 'not in' + else: + type = 'is not' + else: + type = _cmp_types[n[0]] + + lineno = nl[1][2] + results.append((type, self.com_node(nodelist[i]))) + + # we need a special "compare" node so that we can distinguish + # 3 < x < 5 from (3 < x) < 5 + # the two have very different semantics and results (note that the + # latter form is always true) + + return Compare(node, results, lineno=lineno) + + def expr(self, nodelist): + # xor_expr ('|' xor_expr)* + return self.com_binary(Bitor, nodelist) + + def xor_expr(self, nodelist): + # xor_expr ('^' xor_expr)* + return self.com_binary(Bitxor, nodelist) + + def and_expr(self, nodelist): + # xor_expr ('&' xor_expr)* + return self.com_binary(Bitand, nodelist) + + def shift_expr(self, nodelist): + # shift_expr ('<<'|'>>' shift_expr)* + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.LEFTSHIFT: + node = LeftShift([node, right], lineno=nodelist[1][2]) + elif nodelist[i-1][0] == token.RIGHTSHIFT: + node = RightShift([node, right], lineno=nodelist[1][2]) + else: + raise ValueError, "unexpected token: %s" % nodelist[i-1][0] + return node + + def arith_expr(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.PLUS: + node = Add([node, right], lineno=nodelist[1][2]) + elif nodelist[i-1][0] == token.MINUS: + node = Sub([node, right], lineno=nodelist[1][2]) + else: + raise ValueError, "unexpected token: %s" % nodelist[i-1][0] + return node + + def term(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + t = nodelist[i-1][0] + if t == token.STAR: + node = Mul([node, right]) + elif t == token.SLASH: + node = Div([node, right]) + elif t == token.PERCENT: + node = Mod([node, right]) + elif t == token.DOUBLESLASH: + node = FloorDiv([node, right]) + else: + raise ValueError, "unexpected token: %s" % t + node.lineno = nodelist[1][2] + return node + + def factor(self, nodelist): + elt = nodelist[0] + t = elt[0] + node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) + # need to handle (unary op)constant here... + if t == token.PLUS: + return UnaryAdd(node, lineno=elt[2]) + elif t == token.MINUS: + return UnarySub(node, lineno=elt[2]) + elif t == token.TILDE: + node = Invert(node, lineno=elt[2]) + return node + + def power(self, nodelist): + # power: atom trailer* ('**' factor)* + node = self.com_node(nodelist[0]) + for i in range(1, len(nodelist)): + elt = nodelist[i] + if elt[0] == token.DOUBLESTAR: + return Power([node, self.com_node(nodelist[i+1])], + lineno=elt[2]) + + node = self.com_apply_trailer(node, elt) + + return node + + def atom(self, nodelist): + return self._atom_dispatch[nodelist[0][0]](nodelist) + + def atom_lpar(self, nodelist): + if nodelist[1][0] == token.RPAR: + return Tuple((), lineno=nodelist[0][2]) + return self.com_node(nodelist[1]) + + def atom_lsqb(self, nodelist): + if nodelist[1][0] == token.RSQB: + return List((), lineno=nodelist[0][2]) + return self.com_list_constructor(nodelist[1]) + + def atom_lbrace(self, nodelist): + if nodelist[1][0] == token.RBRACE: + return Dict((), lineno=nodelist[0][2]) + return self.com_dictorsetmaker(nodelist[1]) + + def atom_backquote(self, nodelist): + return Backquote(self.com_node(nodelist[1])) + + def atom_number(self, nodelist): + ### need to verify this matches compile.c + k = eval(nodelist[0][1]) + return Const(k, lineno=nodelist[0][2]) + + def decode_literal(self, lit): + if self.encoding: + # this is particularly fragile & a bit of a + # hack... changes in compile.c:parsestr and + # tokenizer.c must be reflected here. + if self.encoding not in ['utf-8', 'iso-8859-1']: + lit = unicode(lit, 'utf-8').encode(self.encoding) + return eval("# coding: %s\n%s" % (self.encoding, lit)) + else: + return eval(lit) + + def atom_string(self, nodelist): + k = '' + for node in nodelist: + k += self.decode_literal(node[1]) + return Const(k, lineno=nodelist[0][2]) + + def atom_name(self, nodelist): + return Name(nodelist[0][1], lineno=nodelist[0][2]) + + # -------------------------------------------------------------- + # + # INTERNAL PARSING UTILITIES + # + + # The use of com_node() introduces a lot of extra stack frames, + # enough to cause a stack overflow compiling test.test_parser with + # the standard interpreter recursionlimit. The com_node() is a + # convenience function that hides the dispatch details, but comes + # at a very high cost. It is more efficient to dispatch directly + # in the callers. In these cases, use lookup_node() and call the + # dispatched node directly. + + def lookup_node(self, node): + return self._dispatch[node[0]] + + def com_node(self, node): + # Note: compile.c has handling in com_node for del_stmt, pass_stmt, + # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, + # and compound_stmt. + # We'll just dispatch them. + return self._dispatch[node[0]](node[1:]) + + def com_NEWLINE(self, *args): + # A ';' at the end of a line can make a NEWLINE token appear + # here, Render it harmless. (genc discards ('discard', + # ('const', xxxx)) Nodes) + return Discard(Const(None)) + + def com_arglist(self, nodelist): + # varargslist: + # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) + # | fpdef ['=' test] (',' fpdef ['=' test])* [','] + # fpdef: NAME | '(' fplist ')' + # fplist: fpdef (',' fpdef)* [','] + names = [] + defaults = [] + flags = 0 + + i = 0 + while i < len(nodelist): + node = nodelist[i] + if node[0] == token.STAR or node[0] == token.DOUBLESTAR: + if node[0] == token.STAR: + node = nodelist[i+1] + if node[0] == token.NAME: + names.append(node[1]) + flags = flags | CO_VARARGS + i = i + 3 + + if i < len(nodelist): + # should be DOUBLESTAR + t = nodelist[i][0] + if t == token.DOUBLESTAR: + node = nodelist[i+1] + else: + raise ValueError, "unexpected token: %s" % t + names.append(node[1]) + flags = flags | CO_VARKEYWORDS + + break + + # fpdef: NAME | '(' fplist ')' + names.append(self.com_fpdef(node)) + + i = i + 1 + if i < len(nodelist) and nodelist[i][0] == token.EQUAL: + defaults.append(self.com_node(nodelist[i + 1])) + i = i + 2 + elif len(defaults): + # we have already seen an argument with default, but here + # came one without + raise SyntaxError, "non-default argument follows default argument" + + # skip the comma + i = i + 1 + + return names, defaults, flags + + def com_fpdef(self, node): + # fpdef: NAME | '(' fplist ')' + if node[1][0] == token.LPAR: + return self.com_fplist(node[2]) + return node[1][1] + + def com_fplist(self, node): + # fplist: fpdef (',' fpdef)* [','] + if len(node) == 2: + return self.com_fpdef(node[1]) + list = [] + for i in range(1, len(node), 2): + list.append(self.com_fpdef(node[i])) + return tuple(list) + + def com_dotted_name(self, node): + # String together the dotted names and return the string + name = "" + for n in node: + if type(n) == type(()) and n[0] == 1: + name = name + n[1] + '.' + return name[:-1] + + def com_dotted_as_name(self, node): + assert node[0] == symbol.dotted_as_name + node = node[1:] + dot = self.com_dotted_name(node[0][1:]) + if len(node) == 1: + return dot, None + assert node[1][1] == 'as' + assert node[2][0] == token.NAME + return dot, node[2][1] + + def com_dotted_as_names(self, node): + assert node[0] == symbol.dotted_as_names + node = node[1:] + names = [self.com_dotted_as_name(node[0])] + for i in range(2, len(node), 2): + names.append(self.com_dotted_as_name(node[i])) + return names + + def com_import_as_name(self, node): + assert node[0] == symbol.import_as_name + node = node[1:] + assert node[0][0] == token.NAME + if len(node) == 1: + return node[0][1], None + assert node[1][1] == 'as', node + assert node[2][0] == token.NAME + return node[0][1], node[2][1] + + def com_import_as_names(self, node): + assert node[0] == symbol.import_as_names + node = node[1:] + names = [self.com_import_as_name(node[0])] + for i in range(2, len(node), 2): + names.append(self.com_import_as_name(node[i])) + return names + + def com_bases(self, node): + bases = [] + for i in range(1, len(node), 2): + bases.append(self.com_node(node[i])) + return bases + + def com_try_except_finally(self, nodelist): + # ('try' ':' suite + # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] + # | 'finally' ':' suite)) + + if nodelist[3][0] == token.NAME: + # first clause is a finally clause: only try-finally + return TryFinally(self.com_node(nodelist[2]), + self.com_node(nodelist[5]), + lineno=nodelist[0][2]) + + #tryexcept: [TryNode, [except_clauses], elseNode)] + clauses = [] + elseNode = None + finallyNode = None + for i in range(3, len(nodelist), 3): + node = nodelist[i] + if node[0] == symbol.except_clause: + # except_clause: 'except' [expr [(',' | 'as') expr]] */ + if len(node) > 2: + expr1 = self.com_node(node[2]) + if len(node) > 4: + expr2 = self.com_assign(node[4], OP_ASSIGN) + else: + expr2 = None + else: + expr1 = expr2 = None + clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) + + if node[0] == token.NAME: + if node[1] == 'else': + elseNode = self.com_node(nodelist[i+2]) + elif node[1] == 'finally': + finallyNode = self.com_node(nodelist[i+2]) + try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, + lineno=nodelist[0][2]) + if finallyNode: + return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) + else: + return try_except + + def com_with(self, nodelist): + # with_stmt: 'with' with_item (',' with_item)* ':' suite + body = self.com_node(nodelist[-1]) + for i in range(len(nodelist) - 3, 0, -2): + ret = self.com_with_item(nodelist[i], body, nodelist[0][2]) + if i == 1: + return ret + body = ret + + def com_with_item(self, nodelist, body, lineno): + # with_item: test ['as' expr] + if len(nodelist) == 4: + var = self.com_assign(nodelist[3], OP_ASSIGN) + else: + var = None + expr = self.com_node(nodelist[1]) + return With(expr, var, body, lineno=lineno) + + def com_augassign_op(self, node): + assert node[0] == symbol.augassign + return node[1] + + def com_augassign(self, node): + """Return node suitable for lvalue of augmented assignment + + Names, slices, and attributes are the only allowable nodes. + """ + l = self.com_node(node) + if l.__class__ in (Name, Slice, Subscript, Getattr): + return l + raise SyntaxError, "can't assign to %s" % l.__class__.__name__ + + def com_assign(self, node, assigning): + # return a node suitable for use as an "lvalue" + # loop to avoid trivial recursion + while 1: + t = node[0] + if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp): + if len(node) > 2: + return self.com_assign_tuple(node, assigning) + node = node[1] + elif t in _assign_types: + if len(node) > 2: + raise SyntaxError, "can't assign to operator" + node = node[1] + elif t == symbol.power: + if node[1][0] != symbol.atom: + raise SyntaxError, "can't assign to operator" + if len(node) > 2: + primary = self.com_node(node[1]) + for i in range(2, len(node)-1): + ch = node[i] + if ch[0] == token.DOUBLESTAR: + raise SyntaxError, "can't assign to operator" + primary = self.com_apply_trailer(primary, ch) + return self.com_assign_trailer(primary, node[-1], + assigning) + node = node[1] + elif t == symbol.atom: + t = node[1][0] + if t == token.LPAR: + node = node[2] + if node[0] == token.RPAR: + raise SyntaxError, "can't assign to ()" + elif t == token.LSQB: + node = node[2] + if node[0] == token.RSQB: + raise SyntaxError, "can't assign to []" + return self.com_assign_list(node, assigning) + elif t == token.NAME: + return self.com_assign_name(node[1], assigning) + else: + raise SyntaxError, "can't assign to literal" + else: + raise SyntaxError, "bad assignment (%s)" % t + + def com_assign_tuple(self, node, assigning): + assigns = [] + for i in range(1, len(node), 2): + assigns.append(self.com_assign(node[i], assigning)) + return AssTuple(assigns, lineno=extractLineNo(node)) + + def com_assign_list(self, node, assigning): + assigns = [] + for i in range(1, len(node), 2): + if i + 1 < len(node): + if node[i + 1][0] == symbol.list_for: + raise SyntaxError, "can't assign to list comprehension" + assert node[i + 1][0] == token.COMMA, node[i + 1] + assigns.append(self.com_assign(node[i], assigning)) + return AssList(assigns, lineno=extractLineNo(node)) + + def com_assign_name(self, node, assigning): + return AssName(node[1], assigning, lineno=node[2]) + + def com_assign_trailer(self, primary, node, assigning): + t = node[1][0] + if t == token.DOT: + return self.com_assign_attr(primary, node[2], assigning) + if t == token.LSQB: + return self.com_subscriptlist(primary, node[2], assigning) + if t == token.LPAR: + raise SyntaxError, "can't assign to function call" + raise SyntaxError, "unknown trailer type: %s" % t + + def com_assign_attr(self, primary, node, assigning): + return AssAttr(primary, node[1], assigning, lineno=node[-1]) + + def com_binary(self, constructor, nodelist): + "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." + l = len(nodelist) + if l == 1: + n = nodelist[0] + return self.lookup_node(n)(n[1:]) + items = [] + for i in range(0, l, 2): + n = nodelist[i] + items.append(self.lookup_node(n)(n[1:])) + return constructor(items, lineno=extractLineNo(nodelist)) + + def com_stmt(self, node): + result = self.lookup_node(node)(node[1:]) + assert result is not None + if isinstance(result, Stmt): + return result + return Stmt([result]) + + def com_append_stmt(self, stmts, node): + result = self.lookup_node(node)(node[1:]) + assert result is not None + if isinstance(result, Stmt): + stmts.extend(result.nodes) + else: + stmts.append(result) + + def com_list_constructor(self, nodelist): + # listmaker: test ( list_for | (',' test)* [','] ) + values = [] + for i in range(1, len(nodelist)): + if nodelist[i][0] == symbol.list_for: + assert len(nodelist[i:]) == 1 + return self.com_list_comprehension(values[0], + nodelist[i]) + elif nodelist[i][0] == token.COMMA: + continue + values.append(self.com_node(nodelist[i])) + return List(values, lineno=values[0].lineno) + + def com_list_comprehension(self, expr, node): + return self.com_comprehension(expr, None, node, 'list') + + def com_comprehension(self, expr1, expr2, node, type): + # list_iter: list_for | list_if + # list_for: 'for' exprlist 'in' testlist [list_iter] + # list_if: 'if' test [list_iter] + + # XXX should raise SyntaxError for assignment + # XXX(avassalotti) Set and dict comprehensions should have generator + # semantics. In other words, they shouldn't leak + # variables outside of the comprehension's scope. + + lineno = node[1][2] + fors = [] + while node: + t = node[1][1] + if t == 'for': + assignNode = self.com_assign(node[2], OP_ASSIGN) + compNode = self.com_node(node[4]) + newfor = ListCompFor(assignNode, compNode, []) + newfor.lineno = node[1][2] + fors.append(newfor) + if len(node) == 5: + node = None + elif type == 'list': + node = self.com_list_iter(node[5]) + else: + node = self.com_comp_iter(node[5]) + elif t == 'if': + test = self.com_node(node[2]) + newif = ListCompIf(test, lineno=node[1][2]) + newfor.ifs.append(newif) + if len(node) == 3: + node = None + elif type == 'list': + node = self.com_list_iter(node[3]) + else: + node = self.com_comp_iter(node[3]) + else: + raise SyntaxError, \ + ("unexpected comprehension element: %s %d" + % (node, lineno)) + if type == 'list': + return ListComp(expr1, fors, lineno=lineno) + elif type == 'set': + return SetComp(expr1, fors, lineno=lineno) + elif type == 'dict': + return DictComp(expr1, expr2, fors, lineno=lineno) + else: + raise ValueError("unexpected comprehension type: " + repr(type)) + + def com_list_iter(self, node): + assert node[0] == symbol.list_iter + return node[1] + + def com_comp_iter(self, node): + assert node[0] == symbol.comp_iter + return node[1] + + def com_generator_expression(self, expr, node): + # comp_iter: comp_for | comp_if + # comp_for: 'for' exprlist 'in' test [comp_iter] + # comp_if: 'if' test [comp_iter] + + lineno = node[1][2] + fors = [] + while node: + t = node[1][1] + if t == 'for': + assignNode = self.com_assign(node[2], OP_ASSIGN) + genNode = self.com_node(node[4]) + newfor = GenExprFor(assignNode, genNode, [], + lineno=node[1][2]) + fors.append(newfor) + if (len(node)) == 5: + node = None + else: + node = self.com_comp_iter(node[5]) + elif t == 'if': + test = self.com_node(node[2]) + newif = GenExprIf(test, lineno=node[1][2]) + newfor.ifs.append(newif) + if len(node) == 3: + node = None + else: + node = self.com_comp_iter(node[3]) + else: + raise SyntaxError, \ + ("unexpected generator expression element: %s %d" + % (node, lineno)) + fors[0].is_outmost = True + return GenExpr(GenExprInner(expr, fors), lineno=lineno) + + def com_dictorsetmaker(self, nodelist): + # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | + # (test (comp_for | (',' test)* [','])) ) + assert nodelist[0] == symbol.dictorsetmaker + nodelist = nodelist[1:] + if len(nodelist) == 1 or nodelist[1][0] == token.COMMA: + # set literal + items = [] + for i in range(0, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + return Set(items, lineno=items[0].lineno) + elif nodelist[1][0] == symbol.comp_for: + # set comprehension + expr = self.com_node(nodelist[0]) + return self.com_comprehension(expr, None, nodelist[1], 'set') + elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for: + # dict comprehension + assert nodelist[1][0] == token.COLON + key = self.com_node(nodelist[0]) + value = self.com_node(nodelist[2]) + return self.com_comprehension(key, value, nodelist[3], 'dict') + else: + # dict literal + items = [] + for i in range(0, len(nodelist), 4): + items.append((self.com_node(nodelist[i]), + self.com_node(nodelist[i+2]))) + return Dict(items, lineno=items[0][0].lineno) + + def com_apply_trailer(self, primaryNode, nodelist): + t = nodelist[1][0] + if t == token.LPAR: + return self.com_call_function(primaryNode, nodelist[2]) + if t == token.DOT: + return self.com_select_member(primaryNode, nodelist[2]) + if t == token.LSQB: + return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) + + raise SyntaxError, 'unknown node type: %s' % t + + def com_select_member(self, primaryNode, nodelist): + if nodelist[0] != token.NAME: + raise SyntaxError, "member must be a name" + return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) + + def com_call_function(self, primaryNode, nodelist): + if nodelist[0] == token.RPAR: + return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) + args = [] + kw = 0 + star_node = dstar_node = None + len_nodelist = len(nodelist) + i = 1 + while i < len_nodelist: + node = nodelist[i] + + if node[0]==token.STAR: + if star_node is not None: + raise SyntaxError, 'already have the varargs indentifier' + star_node = self.com_node(nodelist[i+1]) + i = i + 3 + continue + elif node[0]==token.DOUBLESTAR: + if dstar_node is not None: + raise SyntaxError, 'already have the kwargs indentifier' + dstar_node = self.com_node(nodelist[i+1]) + i = i + 3 + continue + + # positional or named parameters + kw, result = self.com_argument(node, kw, star_node) + + if len_nodelist != 2 and isinstance(result, GenExpr) \ + and len(node) == 3 and node[2][0] == symbol.comp_for: + # allow f(x for x in y), but reject f(x for x in y, 1) + # should use f((x for x in y), 1) instead of f(x for x in y, 1) + raise SyntaxError, 'generator expression needs parenthesis' + + args.append(result) + i = i + 2 + + return CallFunc(primaryNode, args, star_node, dstar_node, + lineno=extractLineNo(nodelist)) + + def com_argument(self, nodelist, kw, star_node): + if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for: + test = self.com_node(nodelist[1]) + return 0, self.com_generator_expression(test, nodelist[2]) + if len(nodelist) == 2: + if kw: + raise SyntaxError, "non-keyword arg after keyword arg" + if star_node: + raise SyntaxError, "only named arguments may follow *expression" + return 0, self.com_node(nodelist[1]) + result = self.com_node(nodelist[3]) + n = nodelist[1] + while len(n) == 2 and n[0] != token.NAME: + n = n[1] + if n[0] != token.NAME: + raise SyntaxError, "keyword can't be an expression (%s)"%n[0] + node = Keyword(n[1], result, lineno=n[2]) + return 1, node + + def com_subscriptlist(self, primary, nodelist, assigning): + # slicing: simple_slicing | extended_slicing + # simple_slicing: primary "[" short_slice "]" + # extended_slicing: primary "[" slice_list "]" + # slice_list: slice_item ("," slice_item)* [","] + + # backwards compat slice for '[i:j]' + if len(nodelist) == 2: + sub = nodelist[1] + if (sub[1][0] == token.COLON or \ + (len(sub) > 2 and sub[2][0] == token.COLON)) and \ + sub[-1][0] != symbol.sliceop: + return self.com_slice(primary, sub, assigning) + + subscripts = [] + for i in range(1, len(nodelist), 2): + subscripts.append(self.com_subscript(nodelist[i])) + return Subscript(primary, assigning, subscripts, + lineno=extractLineNo(nodelist)) + + def com_subscript(self, node): + # slice_item: expression | proper_slice | ellipsis + ch = node[1] + t = ch[0] + if t == token.DOT and node[2][0] == token.DOT: + return Ellipsis() + if t == token.COLON or len(node) > 2: + return self.com_sliceobj(node) + return self.com_node(ch) + + def com_sliceobj(self, node): + # proper_slice: short_slice | long_slice + # short_slice: [lower_bound] ":" [upper_bound] + # long_slice: short_slice ":" [stride] + # lower_bound: expression + # upper_bound: expression + # stride: expression + # + # Note: a stride may be further slicing... + + items = [] + + if node[1][0] == token.COLON: + items.append(Const(None)) + i = 2 + else: + items.append(self.com_node(node[1])) + # i == 2 is a COLON + i = 3 + + if i < len(node) and node[i][0] == symbol.test: + items.append(self.com_node(node[i])) + i = i + 1 + else: + items.append(Const(None)) + + # a short_slice has been built. look for long_slice now by looking + # for strides... + for j in range(i, len(node)): + ch = node[j] + if len(ch) == 2: + items.append(Const(None)) + else: + items.append(self.com_node(ch[2])) + return Sliceobj(items, lineno=extractLineNo(node)) + + def com_slice(self, primary, node, assigning): + # short_slice: [lower_bound] ":" [upper_bound] + lower = upper = None + if len(node) == 3: + if node[1][0] == token.COLON: + upper = self.com_node(node[2]) + else: + lower = self.com_node(node[1]) + elif len(node) == 4: + lower = self.com_node(node[1]) + upper = self.com_node(node[3]) + return Slice(primary, assigning, lower, upper, + lineno=extractLineNo(node)) + + def get_docstring(self, node, n=None): + if n is None: + n = node[0] + node = node[1:] + if n == symbol.suite: + if len(node) == 1: + return self.get_docstring(node[0]) + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.file_input: + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.atom: + if node[0][0] == token.STRING: + s = '' + for t in node: + s = s + eval(t[1]) + return s + return None + if n == symbol.stmt or n == symbol.simple_stmt \ + or n == symbol.small_stmt: + return self.get_docstring(node[0]) + if n in _doc_nodes and len(node) == 1: + return self.get_docstring(node[0]) + return None + + +_doc_nodes = [ + symbol.expr_stmt, + symbol.testlist, + symbol.testlist_safe, + symbol.test, + symbol.or_test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + ] + +# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' +# | 'in' | 'not' 'in' | 'is' | 'is' 'not' +_cmp_types = { + token.LESS : '<', + token.GREATER : '>', + token.EQEQUAL : '==', + token.EQUAL : '==', + token.LESSEQUAL : '<=', + token.GREATEREQUAL : '>=', + token.NOTEQUAL : '!=', + } + +_legal_node_types = [ + symbol.funcdef, + symbol.classdef, + symbol.stmt, + symbol.small_stmt, + symbol.flow_stmt, + symbol.simple_stmt, + symbol.compound_stmt, + symbol.expr_stmt, + symbol.print_stmt, + symbol.del_stmt, + symbol.pass_stmt, + symbol.break_stmt, + symbol.continue_stmt, + symbol.return_stmt, + symbol.raise_stmt, + symbol.import_stmt, + symbol.global_stmt, + symbol.exec_stmt, + symbol.assert_stmt, + symbol.if_stmt, + symbol.while_stmt, + symbol.for_stmt, + symbol.try_stmt, + symbol.with_stmt, + symbol.suite, + symbol.testlist, + symbol.testlist_safe, + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.exprlist, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + symbol.atom, + symbol.yield_stmt, + symbol.yield_expr, + ] + +_assign_types = [ + symbol.test, + symbol.or_test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + ] + +_names = {} +for k, v in symbol.sym_name.items(): + _names[k] = v +for k, v in token.tok_name.items(): + _names[k] = v + +def debug_tree(tree): + l = [] + for elt in tree: + if isinstance(elt, int): + l.append(_names.get(elt, elt)) + elif isinstance(elt, str): + l.append(elt) + else: + l.append(debug_tree(elt)) + return l diff -r 000000000000 -r 5366b587e3fa encoders.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/encoders.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,128 @@ +#!/usr/bin/env python + +""" +Encoder functions, producing representations of program objects. + +Copyright (C) 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +# Output encoding and decoding for the summary files. + +def encode_attrnames(attrnames): + + "Encode the 'attrnames' representing usage." + + return ", ".join(attrnames) or "{}" + +def encode_constrained(constrained): + + "Encode the 'constrained' status for program summaries." + + return constrained and "constrained" or "deduced" + +def encode_usage(usage): + + "Encode attribute details from 'usage'." + + all_attrnames = [] + for t in usage: + all_attrnames.append(t) + return ", ".join(all_attrnames) or "{}" + +def encode_access_location(t): + + "Encode the access location 't'." + + path, name, attrname, version = t + return "%s %s %s:%d" % (path, name or "{}", attrname, version) + +def encode_location(t): + + "Encode the general location 't' in a concise form." + + path, name, attrname, version = t + if name is not None and version is not None: + return "%s %s:%d" % (path, name, version) + elif name is not None: + return "%s %s" % (path, name) + else: + return "%s :%s" % (path, attrname) + +def encode_modifiers(modifiers): + + "Encode assignment details from 'modifiers'." + + all_modifiers = [] + for t in modifiers: + all_modifiers.append(encode_modifier_term(t)) + return "".join(all_modifiers) + +def encode_modifier_term(t): + + "Encode modifier 't' representing assignment status." + + assignment = t + return assignment and "A" or "_" + +def decode_modifier_term(s): + + "Decode modifier term 's' representing assignment status." + + return s == "A" + +# Output program encoding. + +def encode_function_pointer(path): + + "Encode 'path' as a reference to an output program function." + + return "__fn_%s" % encode_path(path) + +def encode_instantiator_pointer(path): + + "Encode 'path' as a reference to an output program instantiator." + + return "__new_%s" % encode_path(path) + +def encode_path(path): + + "Encode 'path' as an output program object, translating special symbols." + + if path in reserved_words: + return "__%s" % path + else: + return path.replace("#", "__").replace("$", "__").replace(".", "_") + +def encode_symbol(symbol_type, path=None): + + "Encode a symbol with the given 'symbol_type' and optional 'path'." + + return "__%s%s" % (symbol_type, path and "_%s" % encode_path(path) or "") + +# Output language reserved words. + +reserved_words = [ + "break", "char", "const", "continue", + "default", "double", "else", + "float", "for", + "if", "int", "long", + "NULL", + "return", "struct", + "typedef", + "void", "while", + ] + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa errors.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/errors.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +""" +Error classes. + +Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 + 2014, 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +class ProcessingError(Exception): + + "A processing error." + + pass + +class ProgramError(ProcessingError): + + "A general program processing error." + + def __init__(self, message): + self.message = message + + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, self.message) + + def __str__(self): + return self.message + +class NodeProcessingError(ProcessingError): + + "A processing error associated with a particular program node." + + def __init__(self, message, unit_name=None, astnode=None): + self.message = message + self.unit_name = unit_name + self.astnode = astnode + + def get_lineno(self, node): + + "Search for line number information associated with 'node'." + + if node is None: + return None + + lineno = node.lineno + if lineno is not None: + return lineno + else: + for child in node.getChildNodes(): + lineno = self.get_lineno(child) + if lineno is not None: + return lineno + return None + + def __repr__(self): + return "%s(%r, %r, %r)" % (self.__class__.__name__, self.message, self.unit_name, self.astnode) + + def __str__(self): + lineno = self.get_lineno(self.astnode) + return "Error in %s%s: %s" % (self.unit_name, lineno and (" at line %s" % lineno) or "", self.message) + +class InspectError(NodeProcessingError): + + "An error during the module inspection process." + + pass + +class TranslateError(NodeProcessingError): + + "An error during the module translation process." + + pass + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa importer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/importer.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,710 @@ +#!/usr/bin/env python + +""" +Import logic. + +Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, + 2014, 2015, 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +from errors import ProgramError +from os.path import exists, extsep, getmtime, join +from os import listdir, makedirs, remove +from common import init_item, readfile, writefile +from referencing import Reference +import inspector +import sys + +class Importer: + + "An import machine, searching for and loading modules." + + def __init__(self, path, cache=None, verbose=False): + + """ + Initialise the importer with the given search 'path' - a list of + directories to search for Python modules. + + The optional 'cache' should be the name of a directory used to store + cached module information. + + The optional 'verbose' parameter causes output concerning the activities + of the object to be produced if set to a true value (not the default). + """ + + self.path = path + self.cache = cache + self.verbose = verbose + + self.modules = {} + self.modules_ordered = [] + self.loading = set() + self.hidden = {} + self.revealing = {} + self.invalidated = set() + + self.objects = {} + self.classes = {} + self.function_parameters = {} + self.function_defaults = {} + self.function_targets = {} + self.function_arguments = {} + + # Derived information. + + self.subclasses = {} + + # Attributes of different object types. + + self.all_class_attrs = {} + self.all_instance_attrs = {} + self.all_instance_attr_constants = {} + self.all_combined_attrs = {} + self.all_module_attrs = {} + self.all_shadowed_attrs = {} + + # References to external names and aliases within program units. + + self.all_name_references = {} + self.all_initialised_names = {} + self.all_aliased_names = {} + + # General attribute accesses. + + self.all_attr_accesses = {} + self.all_const_accesses = {} + self.all_attr_access_modifiers = {} + + # Constant literals and values. + + self.all_constants = {} + self.all_constant_values = {} + + self.make_cache() + + def make_cache(self): + if self.cache and not exists(self.cache): + makedirs(self.cache) + + def check_cache(self, details): + + """ + Check whether the cache applies for the given 'details', invalidating it + if it does not. + """ + + recorded_details = self.get_cache_details() + + if recorded_details != details: + self.remove_cache() + + writefile(self.get_cache_details_filename(), details) + + def get_cache_details_filename(self): + + "Return the filename for the cache details." + + return join(self.cache, "$details") + + def get_cache_details(self): + + "Return details of the cache." + + details_filename = self.get_cache_details_filename() + + if not exists(details_filename): + return None + else: + return readfile(details_filename) + + def remove_cache(self): + + "Remove the contents of the cache." + + for filename in listdir(self.cache): + remove(join(self.cache, filename)) + + def to_cache(self): + + "Write modules to the cache." + + if self.cache: + for module_name, module in self.modules.items(): + module.to_cache(join(self.cache, module_name)) + + # Object retrieval and storage. + + def get_object(self, name): + + """ + Return a reference for the given 'name' or None if no such object + exists. + """ + + return self.objects.get(name) + + def set_object(self, name, value=None): + + "Set the object with the given 'name' and the given 'value'." + + if isinstance(value, Reference): + ref = value.alias(name) + else: + ref = Reference(value, name) + + self.objects[name] = ref + + # Indirect object retrieval. + + def get_attributes(self, ref, attrname): + + """ + Return attributes provided by 'ref' for 'attrname'. Class attributes + may be provided by instances. + """ + + kind = ref.get_kind() + if kind == "": + ref = self.get_class_attribute(ref.get_origin(), attrname) + return ref and set([ref]) or set() + elif kind == "": + return self.get_combined_attributes(ref.get_origin(), attrname) + elif kind == "": + ref = self.get_module_attribute(ref.get_origin(), attrname) + return ref and set([ref]) or set() + else: + return set() + + def get_class_attribute(self, object_type, attrname): + + "Return from 'object_type' the details of class attribute 'attrname'." + + attr = self.all_class_attrs[object_type].get(attrname) + return attr and self.get_object(attr) + + def get_instance_attributes(self, object_type, attrname): + + """ + Return from 'object_type' the details of instance attribute 'attrname'. + """ + + consts = self.all_instance_attr_constants.get(object_type) + attrs = set() + for attr in self.all_instance_attrs[object_type].get(attrname, []): + attrs.add(consts and consts.get(attrname) or Reference("", attr)) + return attrs + + def get_combined_attributes(self, object_type, attrname): + + """ + Return from 'object_type' the details of class or instance attribute + 'attrname'. + """ + + ref = self.get_class_attribute(object_type, attrname) + refs = ref and set([ref]) or set() + refs.update(self.get_instance_attributes(object_type, attrname)) + return refs + + def get_module_attribute(self, object_type, attrname): + + "Return from 'object_type' the details of module attribute 'attrname'." + + if attrname in self.all_module_attrs[object_type]: + return self.get_object("%s.%s" % (object_type, attrname)) + else: + return None + + # Module management. + + def get_modules(self): + + "Return all modules known to the importer." + + return self.modules.values() + + def get_module(self, name, hidden=False): + + "Return the module with the given 'name'." + + if not self.modules.has_key(name): + return None + + # Obtain the module and attempt to reveal it. + + module = self.modules[name] + if not hidden: + self.reveal_module(module) + return module + + def reveal_module(self, module): + + "Check if 'module' is hidden and reveal it." + + if module.name in self.hidden: + del self.hidden[module.name] + + # Reveal referenced modules. + + module.reveal_referenced() + + def set_revealing(self, module, name, instigator): + + """ + Make the revealing of 'module' conditional on 'name' for the given + 'instigator' of the reveal operation. + """ + + self.revealing[module.name].add((name, instigator)) + + # Program operations. + + def initialise(self, filename, reset=False): + + """ + Initialise a program whose main module is 'filename', resetting the + cache if 'reset' is true. Return the main module. + """ + + if reset: + self.remove_cache() + self.check_cache(filename) + + # Load the program itself. + + m = self.load_from_file(filename) + + # Resolve dependencies within the program. + + for module in self.modules_ordered: + module.resolve() + + return m + + def finalise(self): + + "Finalise the inspected program." + + self.finalise_classes() + self.remove_hidden() + self.to_cache() + self.set_class_types() + self.define_instantiators() + self.collect_constants() + + def finalise_classes(self): + + "Finalise the class relationships and attributes." + + self.derive_inherited_attrs() + self.derive_subclasses() + self.derive_shadowed_attrs() + + def derive_inherited_attrs(self): + + "Derive inherited attributes for classes throughout the program." + + for name in self.classes.keys(): + self.propagate_attrs_for_class(name) + + def propagate_attrs_for_class(self, name, visited=None): + + "Propagate inherited attributes for class 'name'." + + # Visit classes only once. + + if self.all_combined_attrs.has_key(name): + return + + visited = visited or [] + + if name in visited: + raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name) + + visited.append(name) + + class_attrs = {} + instance_attrs = {} + + # Aggregate the attributes from base classes, recording the origins of + # applicable attributes. + + for base in self.classes[name][::-1]: + + # Get the identity of the class from the reference. + + base = base.get_origin() + + # Define the base class completely before continuing with this + # class. + + self.propagate_attrs_for_class(base, visited) + class_attrs.update(self.all_class_attrs[base]) + + # Instance attribute origins are combined if different. + + for key, values in self.all_instance_attrs[base].items(): + init_item(instance_attrs, key, set) + instance_attrs[key].update(values) + + # Class attributes override those defined earlier in the hierarchy. + + class_attrs.update(self.all_class_attrs.get(name, {})) + + # Instance attributes are merely added if not already defined. + + for key in self.all_instance_attrs.get(name, []): + if not instance_attrs.has_key(key): + instance_attrs[key] = set(["%s.%s" % (name, key)]) + + self.all_class_attrs[name] = class_attrs + self.all_instance_attrs[name] = instance_attrs + self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys()) + + def derive_subclasses(self): + + "Derive subclass details for classes." + + for name, bases in self.classes.items(): + for base in bases: + + # Get the identity of the class from the reference. + + base = base.get_origin() + self.subclasses[base].add(name) + + def derive_shadowed_attrs(self): + + "Derive shadowed attributes for classes." + + for name, attrs in self.all_instance_attrs.items(): + attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys()) + if attrs: + self.all_shadowed_attrs[name] = attrs + + def remove_hidden(self): + + "Remove all hidden modules." + + # First reveal any modules exposing names. + + for modname, names in self.revealing.items(): + module = self.modules[modname] + + # Obtain the imported names and determine whether they should cause + # the module to be revealed. + + for (name, instigator) in names: + if module is not instigator: + + # Only if an object is provided by the module should the + # module be revealed. References to objects in other modules + # should not in themselves expose the module in which those + # references occur. + + ref = module.get_global(name) + if ref and ref.provided_by_module(module.name): + self.reveal_module(module) + instigator.revealed.add(module) + + # Then remove all modules that are still hidden. + + for modname in self.hidden: + module = self.modules[modname] + module.unpropagate() + del self.modules[modname] + ref = self.objects.get(modname) + if ref and ref.get_kind() == "": + del self.objects[modname] + + def set_class_types(self): + + "Set the type of each class." + + ref = self.get_object("__builtins__.type") + for attrs in self.all_class_attrs.values(): + attrs["__class__"] = ref.get_origin() + + def define_instantiators(self): + + """ + Consolidate parameter and default details, incorporating initialiser + details to define instantiator signatures. + """ + + for cls, attrs in self.all_class_attrs.items(): + initialiser = attrs["__init__"] + self.function_parameters[cls] = self.function_parameters[initialiser][1:] + self.function_defaults[cls] = self.function_defaults[initialiser] + + def collect_constants(self): + + "Get constants from all active modules." + + for module in self.modules.values(): + self.all_constants.update(module.constants) + + # Import methods. + + def find_in_path(self, name): + + """ + Find the given module 'name' in the search path, returning None where no + such module could be found, or a 2-tuple from the 'find' method + otherwise. + """ + + for d in self.path: + m = self.find(d, name) + if m: return m + return None + + def find(self, d, name): + + """ + In the directory 'd', find the given module 'name', where 'name' can + either refer to a single file module or to a package. Return None if the + 'name' cannot be associated with either a file or a package directory, + or a 2-tuple from '_find_package' or '_find_module' otherwise. + """ + + m = self._find_package(d, name) + if m: return m + m = self._find_module(d, name) + if m: return m + return None + + def _find_module(self, d, name): + + """ + In the directory 'd', find the given module 'name', returning None where + no suitable file exists in the directory, or a 2-tuple consisting of + None (indicating that no package directory is involved) and a filename + indicating the location of the module. + """ + + name_py = name + extsep + "py" + filename = self._find_file(d, name_py) + if filename: + return None, filename + return None + + def _find_package(self, d, name): + + """ + In the directory 'd', find the given package 'name', returning None + where no suitable package directory exists, or a 2-tuple consisting of + a directory (indicating the location of the package directory itself) + and a filename indicating the location of the __init__.py module which + declares the package's top-level contents. + """ + + filename = self._find_file(d, name) + if filename: + init_py = "__init__" + extsep + "py" + init_py_filename = self._find_file(filename, init_py) + if init_py_filename: + return filename, init_py_filename + return None + + def _find_file(self, d, filename): + + """ + Return the filename obtained when searching the directory 'd' for the + given 'filename', or None if no actual file exists for the filename. + """ + + filename = join(d, filename) + if exists(filename): + return filename + else: + return None + + def load(self, name, return_leaf=False, hidden=False): + + """ + Load the module or package with the given 'name'. Return an object + referencing the loaded module or package, or None if no such module or + package exists. + + Where 'return_leaf' is specified, the final module in the chain is + returned. Where 'hidden' is specified, the module is marked as hidden. + """ + + if return_leaf: + name_for_return = name + else: + name_for_return = name.split(".")[0] + + # Loaded modules are returned immediately. + # Modules may be known but not yet loading (having been registered as + # submodules), loading, loaded, or completely unknown. + + module = self.get_module(name, hidden) + + if module: + return self.modules[name_for_return] + + # Otherwise, modules are loaded. + + if self.verbose: + print >>sys.stderr, "Loading", name + + # Split the name into path components, and try to find the uppermost in + # the search path. + + path = name.split(".") + path_so_far = [] + top = module = None + + for p in path: + + # Get the module's filesystem details. + + if not path_so_far: + m = self.find_in_path(p) + elif d: + m = self.find(d, p) + else: + m = None + + path_so_far.append(p) + module_name = ".".join(path_so_far) + + if not m: + if self.verbose: + print >>sys.stderr, "Not found (%s)" % name + + return None # NOTE: Import error. + + # Get the module itself. + + d, filename = m + submodule = self.load_from_file(filename, module_name, hidden) + + if module is None: + top = submodule + + module = submodule + + # Return either the deepest or the uppermost module. + + return return_leaf and module or top + + def load_from_file(self, filename, module_name=None, hidden=False): + + "Load the module from the given 'filename'." + + if module_name is None: + module_name = "__main__" + + module = self.modules.get(module_name) + + if not module: + + # Try to load from cache. + + module = self.load_from_cache(filename, module_name, hidden) + if module: + return module + + # If no cache entry exists, load from file. + + module = inspector.InspectedModule(module_name, self) + self.add_module(module_name, module) + self.update_cache_validity(module) + + # Initiate loading if not already in progress. + + if not module.loaded and module not in self.loading: + self._load(module, module_name, hidden, lambda m: m.parse, filename) + + return module + + def update_cache_validity(self, module): + + "Make 'module' valid in the cache, but invalidate accessing modules." + + self.invalidated.update(module.accessing_modules) + if module.name in self.invalidated: + self.invalidated.remove(module.name) + + def source_is_new(self, filename, module_name): + + "Return whether 'filename' is newer than the cached 'module_name'." + + if self.cache: + cache_filename = join(self.cache, module_name) + return not exists(cache_filename) or \ + getmtime(filename) > getmtime(cache_filename) or \ + module_name in self.invalidated + else: + return True + + def load_from_cache(self, filename, module_name, hidden=False): + + "Return a module residing in the cache." + + module = self.modules.get(module_name) + + if not self.source_is_new(filename, module_name): + + if not module: + module = inspector.CachedModule(module_name, self) + self.add_module(module_name, module) + + if not module.loaded and module not in self.loading: + filename = join(self.cache, module_name) + self._load(module, module_name, hidden, lambda m: m.from_cache, filename) + + return module + + def _load(self, module, module_name, hidden, fn, filename): + + """ + Load 'module' for the given 'module_name', with the module being hidden + if 'hidden' is a true value, and with 'fn' performing an invocation on + the module with the given 'filename'. + """ + + # Indicate that the module is hidden if requested. + + if hidden: + self.hidden[module_name] = module + + # Indicate that loading is in progress and load the module. + + self.loading.add(module) + if self.verbose: + print >>sys.stderr, "Loading", filename + fn(module)(filename) + if self.verbose: + print >>sys.stderr, "Loaded", filename + self.loading.remove(module) + + self.modules_ordered.append(module) + + def add_module(self, module_name, module): + + """ + Return the module with the given 'module_name', adding a new module + object if one does not already exist. + """ + + self.modules[module_name] = module + self.objects[module_name] = Reference("", module_name) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa inspector.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/inspector.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,1765 @@ +#!/usr/bin/env python + +""" +Inspect and obtain module structure. + +Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, + 2014, 2015, 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +from branching import BranchTracker +from common import * +from modules import * +from os import listdir +from os.path import extsep, split, splitext +from referencing import Reference +import compiler +import sys + +class AccessRef(Result): + + """ + A reference to an attribute access that is generally only returned from a + processed access for possible initialiser resolution for assignments. + """ + + def __init__(self, original_name, attrnames, number): + self.original_name = original_name + self.attrnames = attrnames + self.number = number + + def reference(self): + return None + + def __repr__(self): + return "AccessRef(%r, %r, %r)" % (self.original_name, self.attrnames, self.number) + +class InvocationRef(Result): + + "An invocation of a name reference." + + def __init__(self, name_ref): + self.name_ref = name_ref + + def __repr__(self): + return "InvocationRef(%r)" % self.name_ref + +class InspectedModule(BasicModule, CacheWritingModule): + + "A module inspector." + + def __init__(self, name, importer): + BasicModule.__init__(self, name, importer) + self.in_class = False + self.in_conditional = False + self.global_attr_accesses = {} + + # Nested scope handling. + + self.parent_function = None + self.propagated_names = {} + + # Usage tracking. + + self.trackers = [] + self.attr_accessor_branches = {} + + def __repr__(self): + return "InspectedModule(%r, %r)" % (self.name, self.importer) + + def parse(self, filename): + + "Parse the file having the given 'filename'." + + self.parse_file(filename) + + # Inspect the module. + + self.start_tracking_in_module() + + # Detect and record imports and globals declared in the module. + + self.assign_general_local("__name__", self.get_constant("str", self.name)) + self.assign_general_local("__file__", self.get_constant("str", filename)) + self.process_structure(self.astnode) + + # Set the class of the module after the definition has occurred. + + ref = self.get_builtin("object") + self.set_name("__class__", ref) + + # Get module-level attribute usage details. + + self.stop_tracking_in_module() + + # Check names used and resolve them. + + self.register_submodules() + self.loaded = True + + def register_submodules(self): + + "For package modules add submodule details." + + if splitext(split(self.filename)[1])[0] == "__init__": + for subname in listdir(split(self.filename)[0]): + name, ext = splitext(subname) + + # Add modules whose names are not already defined as globals. + + if ext == ("%spy" % extsep) and name != "__init__" and not self.get_global(name): + module_name = self.get_global_path(name) + top, submodule = self.get_module(module_name, True) + self.set_module(name, submodule, hidden=True) + + def check_special(self): + + "Check special names." + + for name, value in self.special.items(): + if value.has_kind(""): + self.find_imported_name(name, self.name) + self.special[name] = self.get_object(value.get_origin()) + + def check_names_used(self): + + "Check the names used by each function." + + for path in self.names_used.keys(): + self.check_names_used_for_path(path) + + def check_names_used_for_path(self, path): + + "Check the names used by the given 'path'." + + names = self.names_used.get(path) + if not names: + return + + in_function = self.function_locals.has_key(path) + + for name in names: + if name in predefined_constants or in_function and name in self.function_locals[path]: + continue + + # Resolve names that have been imported locally. + + self.find_imported_name(name, path) + + # Find local definitions. + + key = "%s.%s" % (path, name) + ref = self.get_object(key) + if ref: + self.name_references[key] = ref.final() or key + self.resolve_accesses(path, name, ref) + continue + + # Resolve names that have been imported globally. + + self.find_imported_name(name, self.name) + + # Find global or built-in definitions. + + ref = self.get_global_or_builtin(name) + objpath = ref and (ref.final() or ref.get_name()) + if objpath: + self.name_references[key] = objpath + self.resolve_accesses(path, name, ref) + continue + + print >>sys.stderr, "Name not recognised: %s in %s" % (name, path) + init_item(self.names_missing, path, set) + self.names_missing[path].add(name) + + def resolve_members(self): + + "Resolve any members referring to deferred references." + + for name, ref in self.objects.items(): + if ref.has_kind(""): + ref = self.get_object(ref.get_origin()) + ref = ref.alias(name) + self.importer.objects[name] = self.objects[name] = ref + + def resolve_accesses(self, path, name, ref): + + """ + Resolve any unresolved accesses in the function at the given 'path' + for the given 'name' corresponding to the indicated 'ref'. Note that + this mechanism cannot resolve things like inherited methods because + they are not recorded as program objects in their inherited locations. + """ + + attr_accesses = self.global_attr_accesses.get(path) + all_attrnames = attr_accesses and attr_accesses.get(name) + + if not all_attrnames: + return + + # Insist on constant accessors. + + if not ref.has_kind(["", ""]): + return + + found_attrnames = set() + + for attrnames in all_attrnames: + + # Start with the resolved name, adding attributes. + + attrs = ref.get_path() + remaining = attrnames.split(".") + last_ref = ref + + # Add each component, testing for a constant object. + + while remaining: + attrname = remaining[0] + attrs.append(attrname) + del remaining[0] + + # Find any constant object reference. + + attr_ref = self.get_object(".".join(attrs)) + + # Non-constant accessors terminate the traversal. + + if not attr_ref.has_kind(["", "", ""]): + + # Provide the furthest constant accessor unless the final + # access can be resolved. + + if remaining: + remaining.insert(0, attrs.pop()) + else: + last_ref = attr_ref + break + + # A module may expose an attribute imported from a hidden + # module. + + elif last_ref.has_kind(""): + module, leaf_module = self.get_module(last_ref.get_origin()) + self.find_imported_name(attrname, module.name, module) + + # Follow any reference to a constant object. + # Where the given name refers to an object in another location, + # switch to the other location in order to be able to test its + # attributes. + + last_ref = attr_ref + attrs = attr_ref.get_path() + + # Record a constant accessor only if an object was found + # that is different from the namespace involved. + + if last_ref: + objpath = ".".join(attrs) + if objpath != path: + + # Establish a constant access. + + init_item(self.const_accesses, path, dict) + self.const_accesses[path][(name, attrnames)] = (objpath, last_ref, ".".join(remaining)) + + if len(attrs) > 1: + found_attrnames.add(attrs[1]) + + # Remove any usage records for the name. + + if found_attrnames: + + # NOTE: Should be only one name version. + + versions = [] + for version in self.attr_usage[path][name]: + new_usage = set() + for usage in version: + if found_attrnames.intersection(usage): + new_usage.add(tuple(set(usage).difference(found_attrnames))) + else: + new_usage.add(usage) + versions.append(new_usage) + + self.attr_usage[path][name] = versions + + def resolve_initialisers(self): + + "Resolve initialiser values for names." + + # Get the initialisers in each namespace. + + for path, name_initialisers in self.name_initialisers.items(): + const_accesses = self.const_accesses.get(path) + + # Resolve values for each name in a scope. + + for name, values in name_initialisers.items(): + if path == self.name: + assigned_path = name + else: + assigned_path = "%s.%s" % (path, name) + + initialised_names = {} + aliased_names = {} + + for i, name_ref in enumerate(values): + + # Unwrap invocations. + + if isinstance(name_ref, InvocationRef): + invocation = True + name_ref = name_ref.name_ref + else: + invocation = False + + # Obtain a usable reference from names or constants. + + if isinstance(name_ref, ResolvedNameRef): + if not name_ref.reference(): + continue + ref = name_ref.reference() + + # Obtain a reference from instances. + + elif isinstance(name_ref, InstanceRef): + if not name_ref.reference(): + continue + ref = name_ref.reference() + + # Resolve accesses that employ constants. + + elif isinstance(name_ref, AccessRef): + ref = None + + if const_accesses: + resolved_access = const_accesses.get((name_ref.original_name, name_ref.attrnames)) + if resolved_access: + objpath, ref, remaining_attrnames = resolved_access + if remaining_attrnames: + ref = None + + # Accesses that do not employ constants cannot be resolved, + # but they may be resolvable later. + + if not ref: + if not invocation: + aliased_names[i] = name_ref.original_name, name_ref.attrnames, name_ref.number + continue + + # Attempt to resolve a plain name reference. + + elif isinstance(name_ref, LocalNameRef): + key = "%s.%s" % (path, name_ref.name) + origin = self.name_references.get(key) + + # Accesses that do not refer to known static objects + # cannot be resolved, but they may be resolvable later. + + if not origin: + if not invocation: + aliased_names[i] = name_ref.name, None, name_ref.number + continue + + ref = self.get_object(origin) + if not ref: + continue + + elif isinstance(name_ref, NameRef): + key = "%s.%s" % (path, name_ref.name) + origin = self.name_references.get(key) + if not origin: + continue + + ref = self.get_object(origin) + if not ref: + continue + + else: + continue + + # Convert class invocations to instances. + + if invocation: + ref = ref.has_kind("") and ref.instance_of() or None + + if ref: + initialised_names[i] = ref + + if initialised_names: + self.initialised_names[assigned_path] = initialised_names + if aliased_names: + self.aliased_names[assigned_path] = aliased_names + + def resolve_literals(self): + + "Resolve constant value types." + + # Get the constants defined in each namespace. + + for path, constants in self.constants.items(): + for constant, n in constants.items(): + objpath = "%s.$c%d" % (path, n) + _constant, value_type = self.constant_values[objpath] + self.initialised_names[objpath] = {0 : Reference("", value_type)} + + # Get the literals defined in each namespace. + + for path, literals in self.literals.items(): + for n in range(0, literals): + objpath = "%s.$C%d" % (path, n) + value_type = self.literal_types[objpath] + self.initialised_names[objpath] = {0 : Reference("", value_type)} + + def remove_redundant_accessors(self): + + "Remove now-redundant modifier and accessor information." + + for path, const_accesses in self.const_accesses.items(): + accesses = self.attr_accessors.get(path) + modifiers = self.attr_access_modifiers.get(path) + if not accesses: + continue + for access in const_accesses.keys(): + if accesses.has_key(access): + del accesses[access] + if modifiers and modifiers.has_key(access): + del modifiers[access] + + def set_invocation_usage(self): + + """ + Discard the current invocation storage figures, retaining the maximum + values. + """ + + for path, (current, maximum) in self.function_targets.items(): + self.importer.function_targets[path] = self.function_targets[path] = maximum + + for path, (current, maximum) in self.function_arguments.items(): + self.importer.function_arguments[path] = self.function_arguments[path] = maximum + + # Module structure traversal. + + def process_structure_node(self, n): + + "Process the individual node 'n'." + + # Module global detection. + + if isinstance(n, compiler.ast.Global): + self.process_global_node(n) + + # Module import declarations. + + elif isinstance(n, compiler.ast.From): + self.process_from_node(n) + + elif isinstance(n, compiler.ast.Import): + self.process_import_node(n) + + # Nodes using operator module functions. + + elif isinstance(n, compiler.ast.Operator): + return self.process_operator_node(n) + + elif isinstance(n, compiler.ast.AugAssign): + self.process_augassign_node(n) + + elif isinstance(n, compiler.ast.Compare): + return self.process_compare_node(n) + + elif isinstance(n, compiler.ast.Slice): + return self.process_slice_node(n) + + elif isinstance(n, compiler.ast.Sliceobj): + return self.process_sliceobj_node(n) + + elif isinstance(n, compiler.ast.Subscript): + return self.process_subscript_node(n) + + # Namespaces within modules. + + elif isinstance(n, compiler.ast.Class): + self.process_class_node(n) + + elif isinstance(n, compiler.ast.Function): + self.process_function_node(n, n.name) + + elif isinstance(n, compiler.ast.Lambda): + return self.process_lambda_node(n) + + # Assignments. + + elif isinstance(n, compiler.ast.Assign): + + # Handle each assignment node. + + for node in n.nodes: + self.process_assignment_node(node, n.expr) + + # Assignments within non-Assign nodes. + + elif isinstance(n, compiler.ast.AssName): + self.process_assignment_node(n, None) + + elif isinstance(n, compiler.ast.AssAttr): + self.process_attribute_access(n) + + # Accesses. + + elif isinstance(n, compiler.ast.Getattr): + return self.process_attribute_access(n) + + # Name recording for later testing. + + elif isinstance(n, compiler.ast.Name): + return self.process_name_node(n) + + # Conditional statement tracking. + + elif isinstance(n, compiler.ast.For): + self.process_for_node(n) + + elif isinstance(n, compiler.ast.While): + self.process_while_node(n) + + elif isinstance(n, compiler.ast.If): + self.process_if_node(n) + + elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): + return self.process_logical_node(n) + + # Exception control-flow tracking. + + elif isinstance(n, compiler.ast.TryExcept): + self.process_try_node(n) + + elif isinstance(n, compiler.ast.TryFinally): + self.process_try_finally_node(n) + + # Control-flow modification statements. + + elif isinstance(n, compiler.ast.Break): + self.trackers[-1].suspend_broken_branch() + + elif isinstance(n, compiler.ast.Continue): + self.trackers[-1].suspend_continuing_branch() + + elif isinstance(n, compiler.ast.Raise): + self.process_structure(n) + self.trackers[-1].abandon_branch() + + elif isinstance(n, compiler.ast.Return): + self.process_structure(n) + self.trackers[-1].abandon_returning_branch() + + # Invocations. + + elif isinstance(n, compiler.ast.CallFunc): + return self.process_invocation_node(n) + + # Constant usage. + + elif isinstance(n, compiler.ast.Const): + return self.get_literal_instance(n, n.value.__class__.__name__) + + elif isinstance(n, compiler.ast.Dict): + return self.get_literal_instance(n, "dict") + + elif isinstance(n, compiler.ast.List): + return self.get_literal_instance(n, "list") + + elif isinstance(n, compiler.ast.Tuple): + return self.get_literal_instance(n, "tuple") + + # List comprehensions and if expressions. + + elif isinstance(n, compiler.ast.ListComp): + self.process_listcomp_node(n) + + elif isinstance(n, compiler.ast.IfExp): + self.process_ifexp_node(n) + + # All other nodes are processed depth-first. + + else: + self.process_structure(n) + + # By default, no expression details are returned. + + return None + + # Specific node handling. + + def process_assignment_node(self, n, expr): + + "Process the individual node 'n' to be assigned the contents of 'expr'." + + # Names and attributes are assigned the entire expression. + + if isinstance(n, compiler.ast.AssName): + + name_ref = expr and self.process_structure_node(expr) + + # Name assignments populate either function namespaces or the + # general namespace hierarchy. + + self.assign_general_local(n.name, name_ref) + + # Record usage of the name. + + self.record_name(n.name) + + elif isinstance(n, compiler.ast.AssAttr): + if expr: self.process_structure_node(expr) + self.process_attribute_access(n) + + # Lists and tuples are matched against the expression and their + # items assigned to expression items. + + elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): + self.process_assignment_node_items(n, expr) + + # Slices and subscripts are permitted within assignment nodes. + + elif isinstance(n, compiler.ast.Slice): + self.process_slice_node(n, expr) + + elif isinstance(n, compiler.ast.Subscript): + self.process_subscript_node(n, expr) + + def process_attribute_access(self, n): + + "Process the given attribute access node 'n'." + + # Obtain any completed chain and return the reference to it. + + name_ref = self.process_attribute_chain(n) + if self.have_access_expression(n): + return name_ref + + # Where the start of the chain of attributes has been reached, determine + # the complete access. + + # Given a non-access node, this chain can be handled in its entirety, + # either being name-based and thus an access rooted on a name, or being + # based on some other node and thus an anonymous access of some kind. + + path = self.get_namespace_path() + + # Start with the the full attribute chain. + + remaining = self.attrs + attrnames = ".".join(remaining) + + # If the accessor cannot be identified, or where attributes + # remain in an attribute chain, record the anonymous accesses. + + if not isinstance(name_ref, NameRef): # includes ResolvedNameRef + + assignment = isinstance(n, compiler.ast.AssAttr) + + init_item(self.attr_accesses, path, set) + self.attr_accesses[path].add(attrnames) + + self.record_access_details(None, attrnames, assignment) + del self.attrs[0] + return + + # Name-based accesses will handle the first attribute in a + # chain. + + else: + attrname = remaining[0] + + # Attribute assignments are used to identify instance attributes. + + if isinstance(n, compiler.ast.AssAttr) and \ + self.in_class and self.in_function and n.expr.name == "self": + + self.set_instance_attr(attrname) + + # Record attribute usage using any name local to this namespace, + # if assigned in the namespace, or using an external name + # (presently just globals within classes). + + name = self.get_name_for_tracking(name_ref.name, name_ref.final()) + tracker = self.trackers[-1] + + immediate_access = len(self.attrs) == 1 + assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) + + del self.attrs[0] + + # Record global-based chains for subsequent resolution. + + is_global = self.in_function and not self.function_locals[path].has_key(name) or \ + not self.in_function + + if is_global: + self.record_global_access_details(name, attrnames) + + # Make sure the name is being tracked: global names will not + # already be initialised in a branch and must be added + # explicitly. + + if not tracker.have_name(name): + tracker.assign_names([name]) + if self.in_function: + self.scope_globals[path].add(name) + + # Record attribute usage in the tracker, and record the branch + # information for the access. + + branches = tracker.use_attribute(name, attrname) + + if not branches: + print >>sys.stderr, "In %s, name %s is accessed using %s before an assignment." % ( + path, name, attrname) + branches = tracker.use_attribute(name, attrname) + + self.record_branches_for_access(branches, name, attrnames) + access_number = self.record_access_details(name, attrnames, assignment) + return AccessRef(name, attrnames, access_number) + + def process_class_node(self, n): + + "Process the given class node 'n'." + + path = self.get_namespace_path() + + # To avoid notions of class "versions" where the same definition + # might be parameterised with different state and be referenced + # elsewhere (as base classes, for example), classes in functions or + # conditions are forbidden. + + if self.in_function or self.in_conditional: + print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( + path, n.name) + return + + # Resolve base classes. + + bases = [] + + for base in n.bases: + base_class = self.get_class(base) + + if not base_class: + print >>sys.stderr, "In %s, class %s has unidentifiable base classes." % ( + path, n.name) + return + else: + bases.append(base_class) + + # Record bases for the class and retain the class name. + + class_name = self.get_object_path(n.name) + + if not bases and class_name != "__builtins__.core.object": + ref = self.get_object("__builtins__.object") + bases.append(ref) + + self.importer.classes[class_name] = self.classes[class_name] = bases + self.importer.subclasses[class_name] = set() + self.scope_globals[class_name] = set() + + # Set the definition before entering the namespace rather than + # afterwards because methods may reference it. In normal Python, + # a class is not accessible until the definition is complete, but + # methods can generally reference it since upon being called the + # class will already exist. + + self.set_definition(n.name, "") + + in_class = self.in_class + self.in_class = class_name + self.set_instance_attr("__class__", Reference("", class_name)) + self.enter_namespace(n.name) + self.set_name("__fn__") # special instantiator attribute + self.set_name("__args__") # special instantiator attribute + self.assign_general_local("__name__", self.get_constant("str", class_name)) + self.process_structure_node(n.code) + self.exit_namespace() + self.in_class = in_class + + def process_from_node(self, n): + + "Process the given node 'n', importing from another module." + + path = self.get_namespace_path() + + modname, names = self.get_module_name(n) + + # Load the mentioned module. + + top, module = self.get_module(modname, True) + self.set_module(None, module, hidden=True) + + if not module: + print >>sys.stderr, "In %s, from statement importing from %s failed." % ( + path, modname) + + # Attempt to obtain the referenced objects. + + for name, alias in n.names: + + # NOTE: Package submodules are not implicitly imported. + + if name == "*": + if module: + + # Warn about a circular import that probably doesn't find + # the names. + + if not module.loaded: + print >>sys.stderr, "In %s, from statement performs circular import %s of %s." % ( + path, modname) + + for name, value in module.get_globals().items(): + if name != "__name__": + value = ResolvedNameRef(name, value) + self.set_general_local(name, value) + self.set_imported_name(name, modname) + break + + # Explicit names. + + ref = self.import_name_from_module(name, modname, module, alias) + value = ResolvedNameRef(alias or name, ref) + self.set_general_local(alias or name, value) + self.set_imported_name(name, modname, alias) + + def import_name_from_module(self, name, modname, module, alias=None): + + """ + Import 'name' from the module having the given 'modname', with 'module' + having been obtained for the module name, using 'alias' for the imported + name in the current namespace. + """ + + path = self.get_namespace_path() + + if module and module.get_global(name): + value = module.get_global(name) + + # Warn about an import that fails to provide a name, perhaps due + # to a circular import. + + if not value: + print >>sys.stderr, "In %s, from statement cannot import %s from %s%s." % ( + path, name, modname, not module.loaded and "(circular import)") + + return value + + # Record the name as a dependency. + + else: + return Reference("", "%s.%s" % (modname, name)) + + def process_function_node(self, n, name): + + """ + Process the given function or lambda node 'n' with the given 'name'. + """ + + is_lambda = isinstance(n, compiler.ast.Lambda) + + # Where a function is declared conditionally, use a separate name for + # the definition, and assign the definition to the stated name. + + if (self.in_conditional or self.in_function) and not is_lambda: + original_name = name + name = self.get_lambda_name() + else: + original_name = None + + # Initialise argument and local records. + + function_name = self.get_object_path(name) + + argnames = self.importer.function_parameters[function_name] = \ + self.function_parameters[function_name] = get_argnames(n.argnames) + locals = self.function_locals[function_name] = {} + + for argname in argnames: + locals[argname] = Reference("") + + globals = self.scope_globals[function_name] = set() + + # Process the defaults. + + defaults = self.importer.function_defaults[function_name] = \ + self.function_defaults[function_name] = [] + + for argname, default in compiler.ast.get_defaults(n): + if default: + + # Obtain any reference for the default. + + name_ref = self.process_structure_node(default) + defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference(""))) + + # Reset conditional tracking to focus on the function contents. + + parent_function = self.parent_function + self.parent_function = self.in_function and self.get_namespace_path() or None + + in_conditional = self.in_conditional + self.in_conditional = False + + in_function = self.in_function + self.in_function = function_name + + self.enter_namespace(name) + + # Track attribute usage within the namespace. + + path = self.get_namespace_path() + init_item(self.propagated_names, path, set) + + self.start_tracking(locals) + self.process_structure_node(n.code) + self.stop_tracking() + + # Propagate names from parent scopes. + + for local in self.propagated_names[path]: + if not local in argnames and self.trackers[-1].have_name(local): + argnames.append(local) + defaults.append((local, Reference(""))) + self.set_function_local(local) + + # Exit to the parent and note propagated names. + + self.exit_namespace() + + parent = self.get_namespace_path() + if self.propagated_names.has_key(parent): + for local in self.propagated_names[path]: + self.propagated_names[parent].add(local) + + # Update flags. + + self.in_function = in_function + self.in_conditional = in_conditional + self.parent_function = parent_function + + # Define the function using the appropriate name. + + self.set_definition(name, "") + + # Where a function is set conditionally, assign the name. + + if original_name: + self.process_assignment_for_function(original_name, name) + + def process_global_node(self, n): + + """ + Process the given "global" node 'n'. + """ + + path = self.get_namespace_path() + + if path != self.name: + self.scope_globals[path].update(n.names) + + def process_if_node(self, n): + + """ + Process the given "if" node 'n'. + """ + + tracker = self.trackers[-1] + tracker.new_branchpoint() + + for test, body in n.tests: + self.process_structure_node(test) + + tracker.new_branch() + + in_conditional = self.in_conditional + self.in_conditional = True + self.process_structure_node(body) + self.in_conditional = in_conditional + + tracker.shelve_branch() + + # Maintain a branch for the else clause. + + tracker.new_branch() + if n.else_: + self.process_structure_node(n.else_) + tracker.shelve_branch() + + tracker.merge_branches() + + def process_ifexp_node(self, n): + + "Process the given if expression node 'n'." + + name_ref = self.process_structure_node(self.convert_ifexp_node(n)) + + path = self.get_namespace_path() + self.allocate_arguments(path, self.function_defaults[name_ref.get_origin()]) + self.deallocate_arguments(path, self.function_defaults[name_ref.get_origin()]) + + return InvocationRef(name_ref) + + def process_import_node(self, n): + + "Process the given import node 'n'." + + path = self.get_namespace_path() + + # Load the mentioned module. + + for name, alias in n.names: + module, leaf_module = self.get_module(name, alias) + + if not module: + print >>sys.stderr, "In %s, import statement importing from %s failed." % ( + path, name) + if module and not module.loaded: + print >>sys.stderr, "In %s, import statement performs circular import of %s." % ( + path, name) + + self.set_module(alias or name.split(".")[0], module, leaf_module) + + def process_invocation_node(self, n): + + "Process the given invocation node 'n'." + + path = self.get_namespace_path() + + self.allocate_arguments(path, n.args) + + try: + # Process the expression, obtaining any identified reference. + + name_ref = self.process_structure_node(n.node) + + # Process the arguments. + + for arg in n.args: + self.process_structure_node(arg) + + # Detect class invocations. + + if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind(""): + return InstanceRef(name_ref.reference().instance_of()) + + elif isinstance(name_ref, NameRef): + return InvocationRef(name_ref) + + return None + + finally: + self.deallocate_arguments(path, n.args) + + def process_lambda_node(self, n): + + "Process the given lambda node 'n'." + + name = self.get_lambda_name() + self.process_function_node(n, name) + + origin = self.get_object_path(name) + return ResolvedNameRef(name, Reference("", origin)) + + def process_listcomp_node(self, n): + + "Process the given list comprehension node 'n'." + + name_ref = self.process_structure_node(self.convert_listcomp_node(n)) + + path = self.get_namespace_path() + self.allocate_arguments(path, self.function_defaults[name_ref.get_origin()]) + self.deallocate_arguments(path, self.function_defaults[name_ref.get_origin()]) + + return InvocationRef(name_ref) + + def process_logical_node(self, n): + + "Process the given operator node 'n'." + + self.process_operator_chain(n.nodes, self.process_structure_node) + + def process_name_node(self, n): + + "Process the given name node 'n'." + + path = self.get_namespace_path() + + # Special names. + + if n.name.startswith("$"): + value = self.get_special(n.name) + if value: + return value + + # Special case for operator functions introduced through code + # transformations. + + if n.name.startswith("$op"): + + # Obtain the location of the actual function defined in the operator + # package. + + op = n.name[len("$op"):] + + # Access the operator module. + + top, module = self.get_module("operator", True) + self.set_module(None, module, hidden=True) + + # Link the operation to the operator module definition in this + # module. + + self.set_imported_name(op, "operator", n.name, self.name) + + # Attempt to get a reference. + + ref = self.import_name_from_module(op, "operator", module) + ref = self.get_object("operator.%s" % op) or ref + + # Record the imported name and provide the resolved name reference. + + value = ResolvedNameRef(n.name, ref) + self.set_special(n.name, value) + return value + + # Record usage of the name. + + self.record_name(n.name) + + # Search for unknown names in non-function scopes immediately. + # External names in functions are resolved later. + + ref = self.find_name(n.name) + if ref: + return ResolvedNameRef(n.name, ref) + + # Global name. + + elif self.in_function and n.name in self.scope_globals[path]: + return NameRef(n.name) + + # Examine other names. + + else: + tracker = self.trackers[-1] + + # Check local names. + + branches = tracker.tracking_name(n.name) + + # Find names inherited from a parent scope. + + if not branches and self.parent_function: + branches = tracker.have_name(n.name) + if branches: + self.propagate_name(n.name) + + # Local or inherited name. + + if branches: + self.record_branches_for_access(branches, n.name, None) + access_number = self.record_access_details(n.name, None, False) + return LocalNameRef(n.name, access_number) + + # Possible global name. + + else: + return NameRef(n.name) + + def process_operator_chain(self, nodes, fn): + + """ + Process the given chain of 'nodes', applying 'fn' to each node or item. + Each node starts a new conditional region, effectively making a deeply- + nested collection of if-like statements. + """ + + tracker = self.trackers[-1] + + for item in nodes: + tracker.new_branchpoint() + tracker.new_branch() + fn(item) + + for item in nodes[:-1]: + tracker.shelve_branch() + tracker.new_branch() + tracker.shelve_branch() + tracker.merge_branches() + + tracker.shelve_branch() + tracker.merge_branches() + + def process_try_node(self, n): + + """ + Process the given "try...except" node 'n'. + """ + + tracker = self.trackers[-1] + tracker.new_branchpoint() + + self.process_structure_node(n.body) + + for name, var, handler in n.handlers: + if name is not None: + self.process_structure_node(name) + + # Any abandoned branches from the body can now be resumed in a new + # branch. + + tracker.resume_abandoned_branches() + + # Establish the local for the handler. + + if var is not None: + self.process_structure_node(var) + if handler is not None: + self.process_structure_node(handler) + + tracker.shelve_branch() + + # The else clause maintains the usage from the body but without the + # abandoned branches since they would never lead to the else clause + # being executed. + + if n.else_: + tracker.new_branch() + self.process_structure_node(n.else_) + tracker.shelve_branch() + + # Without an else clause, a null branch propagates the successful + # outcome. + + else: + tracker.new_branch() + tracker.shelve_branch() + + tracker.merge_branches() + + def process_try_finally_node(self, n): + + """ + Process the given "try...finally" node 'n'. + """ + + tracker = self.trackers[-1] + self.process_structure_node(n.body) + + # Any abandoned branches from the body can now be resumed. + + branches = tracker.resume_all_abandoned_branches() + self.process_structure_node(n.final) + + # At the end of the finally clause, abandoned branches are discarded. + + tracker.restore_active_branches(branches) + + def process_while_node(self, n): + + "Process the given while node 'n'." + + tracker = self.trackers[-1] + tracker.new_branchpoint(loop_node=True) + + # Evaluate any test or iterator outside the loop. + + self.process_structure_node(n.test) + + # Propagate attribute usage to branches. + + tracker.new_branch(loop_node=True) + + # Enter the loop. + + in_conditional = self.in_conditional + self.in_conditional = True + self.process_structure_node(n.body) + self.in_conditional = in_conditional + + # Continuing branches are resumed before any test. + + tracker.resume_continuing_branches() + + # Evaluate any continuation test within the body. + + self.process_structure_node(n.test) + + tracker.shelve_branch(loop_node=True) + + # Support the non-looping condition. + + tracker.new_branch() + tracker.shelve_branch() + + tracker.merge_branches() + + # Evaluate any else clause outside branches. + + if n.else_: + self.process_structure_node(n.else_) + + # Connect broken branches to the code after any loop. + + tracker.resume_broken_branches() + + # Branch tracking methods. + + def start_tracking(self, names): + + """ + Start tracking attribute usage for names in the current namespace, + immediately registering the given 'names'. + """ + + path = self.get_namespace_path() + parent = self.trackers[-1] + tracker = BranchTracker() + self.trackers.append(tracker) + + # For functions created from expressions or for functions within + # functions, propagate usage to the new namespace. + + if self.parent_function: + tracker.inherit_branches(parent, names) + + # Record the given names established as new branches. + + tracker.assign_names(names) + + def assign_name(self, name, name_ref): + + "Assign to 'name' the given 'name_ref' in the current namespace." + + name = self.get_name_for_tracking(name) + self.trackers[-1].assign_names([name], [name_ref]) + + def stop_tracking(self): + + """ + Stop tracking attribute usage, recording computed usage for the current + namespace. + """ + + path = self.get_namespace_path() + tracker = self.trackers.pop() + self.record_assignments_for_access(tracker) + + self.attr_usage[path] = tracker.get_all_usage() + self.name_initialisers[path] = tracker.get_all_values() + + def start_tracking_in_module(self): + + "Start tracking attribute usage in the module." + + tracker = BranchTracker() + self.trackers.append(tracker) + + def stop_tracking_in_module(self): + + "Stop tracking attribute usage in the module." + + tracker = self.trackers[0] + self.record_assignments_for_access(tracker) + self.attr_usage[self.name] = tracker.get_all_usage() + self.name_initialisers[self.name] = tracker.get_all_values() + + def propagate_name(self, name): + + "Propagate the given 'name' into the current namespace." + + path = self.get_namespace_path() + self.propagated_names[path].add(name) + + def record_assignments_for_access(self, tracker): + + """ + For the current path, use the given 'tracker' to record assignment + version information for attribute accesses. + """ + + path = self.get_path_for_access() + + if not self.attr_accessor_branches.has_key(path): + return + + init_item(self.attr_accessors, path, dict) + attr_accessors = self.attr_accessors[path] + + # Obtain the branches applying during each access. + + for access, all_branches in self.attr_accessor_branches[path].items(): + name, attrnames = access + init_item(attr_accessors, access, list) + + # Obtain the assignments applying to each branch. + + for branches in all_branches: + positions = tracker.get_assignment_positions_for_branches(name, branches) + + # Detect missing name information. + + if None in positions: + globals = self.global_attr_accesses.get(path) + accesses = globals and globals.get(name) + if not accesses: + print >>sys.stderr, "In %s, %s may not be defined when used." % ( + self.get_namespace_path(), name) + positions.remove(None) + + attr_accessors[access].append(positions) + + def record_branches_for_access(self, branches, name, attrnames): + + """ + Record the given 'branches' for an access involving the given 'name' and + 'attrnames'. + """ + + access = name, attrnames + path = self.get_path_for_access() + + init_item(self.attr_accessor_branches, path, dict) + attr_accessor_branches = self.attr_accessor_branches[path] + + init_item(attr_accessor_branches, access, list) + attr_accessor_branches[access].append(branches) + + def record_access_details(self, name, attrnames, assignment): + + """ + For the given 'name' and 'attrnames', record an access indicating + whether 'assignment' is occurring. + + These details correspond to accesses otherwise recorded by the attribute + accessor and attribute access dictionaries. + """ + + access = name, attrnames + path = self.get_path_for_access() + + init_item(self.attr_access_modifiers, path, dict) + init_item(self.attr_access_modifiers[path], access, list) + + access_number = len(self.attr_access_modifiers[path][access]) + self.attr_access_modifiers[path][access].append(assignment) + return access_number + + def record_global_access_details(self, name, attrnames): + + """ + Record details of a global access via the given 'name' involving the + indicated 'attrnames'. + """ + + path = self.get_namespace_path() + + init_item(self.global_attr_accesses, path, dict) + init_item(self.global_attr_accesses[path], name, set) + self.global_attr_accesses[path][name].add(attrnames) + + # Namespace modification. + + def record_name(self, name): + + "Record the use of 'name' in a namespace." + + path = self.get_namespace_path() + init_item(self.names_used, path, set) + self.names_used[path].add(name) + + def set_module(self, name, module, leaf_module=None, hidden=False): + + """ + Set a module in the current namespace using the given 'name' and + corresponding 'module' object, with the 'leaf_module' being recorded + if different. If 'hidden' is a true value, the modules are recorded as + not necessarily being exposed by this module. This module is, however, + recorded as accessing the given modules and is thus dependent on them. + """ + + if name: + self.set_general_local(name, module and Reference("", module.name) or None) + if module: + if hidden: + self.imported_hidden.add(module) + if leaf_module and leaf_module is not module: + self.imported_hidden.add(leaf_module) + else: + self.imported.add(module) + module.accessing_modules.add(self.name) + if leaf_module and leaf_module is not module: + self.imported.add(leaf_module) + leaf_module.accessing_modules.add(self.name) + + def set_definition(self, name, kind): + + """ + Set the definition having the given 'name' and 'kind'. + + Definitions are set in the static namespace hierarchy, but they can also + be recorded for function locals. + """ + + if self.is_global(name): + print >>sys.stderr, "In %s, %s is defined as being global." % ( + self.get_namespace_path(), name) + + path = self.get_object_path(name) + self.set_object(path, kind) + + ref = self.get_object(path) + if ref.get_kind() == "": + print >>sys.stderr, "In %s, %s is defined more than once." % ( + self.get_namespace_path(), name) + + if not self.is_global(name) and self.in_function: + self.set_function_local(name, ref) + + def set_function_local(self, name, ref=None): + + "Set the local with the given 'name' and optional 'ref'." + + locals = self.function_locals[self.get_namespace_path()] + multiple = not ref or locals.has_key(name) and locals[name] != ref + locals[name] = multiple and Reference("") or ref + + def assign_general_local(self, name, name_ref): + + """ + Set for 'name' the given 'name_ref', recording the name for attribute + usage tracking. + """ + + self.set_general_local(name, name_ref) + self.assign_name(name, name_ref) + + def set_general_local(self, name, value=None): + + """ + Set the 'name' with optional 'value' in any kind of local namespace, + where the 'value' should be a reference if specified. + """ + + init_value = self.get_initialising_value(value) + + # Module global names. + + if self.is_global(name): + path = self.get_global_path(name) + self.set_object(path, init_value) + + # Function local names. + + elif self.in_function: + path = self.get_object_path(name) + self.set_function_local(name, init_value) + + # Other namespaces (classes). + + else: + path = self.get_object_path(name) + self.set_name(name, init_value) + + def set_name(self, name, ref=None): + + "Attach the 'name' with optional 'ref' to the current namespace." + + self.set_object(self.get_object_path(name), ref) + + def set_instance_attr(self, name, ref=None): + + """ + Add an instance attribute of the given 'name' to the current class, + using the optional 'ref'. + """ + + init_item(self.instance_attrs, self.in_class, set) + self.instance_attrs[self.in_class].add(name) + + if ref: + init_item(self.instance_attr_constants, self.in_class, dict) + self.instance_attr_constants[self.in_class][name] = ref + + def get_initialising_value(self, value): + + "Return a suitable initialiser reference for 'value'." + + if isinstance(value, (NameRef, AccessRef, InstanceRef)): # includes LiteralSequenceRef, ResolvedNameRef + return value.reference() + + # In general, invocations do not produce known results. However, the + # name initialisers are resolved once a module has been inspected. + + elif isinstance(value, InvocationRef): + return None + + else: + return value + + # Static, program-relative naming. + + def find_name(self, name): + + """ + Return the qualified name for the given 'name' used in the current + non-function namespace. + """ + + path = self.get_namespace_path() + ref = None + + if not self.in_function and name not in predefined_constants: + if self.in_class: + ref = self.get_object(self.get_object_path(name)) + if not ref: + ref = self.get_global_or_builtin(name) + + return ref + + def get_class(self, node): + + """ + Use the given 'node' to obtain the identity of a class. Return a + reference for the class. Unresolved dependencies are permitted and must + be resolved later. + """ + + ref = self._get_class(node) + return ref.has_kind(["", ""]) and ref or None + + def _get_class(self, node): + + """ + Use the given 'node' to find a class definition. Return a reference to + the class. + """ + + if isinstance(node, compiler.ast.Getattr): + + # Obtain the identity of the access target. + + ref = self._get_class(node.expr) + + # Where the target is a class or module, obtain the identity of the + # attribute. + + if ref.has_kind(["", ""]): + return None + else: + attrname = "%s.%s" % (ref.get_origin(), node.attrname) + return self.get_object(attrname) + + # Names can be module-level or built-in. + + elif isinstance(node, compiler.ast.Name): + + # Record usage of the name and attempt to identify it. + + self.record_name(node.name) + return self.get_global_or_builtin(node.name) + else: + return None + + def get_constant(self, name, value): + + "Return a constant reference for the given type 'name' and 'value'." + + ref = self.get_literal_builtin(name) + return self.get_constant_reference(ref, value) + + def get_literal_instance(self, n, name): + + "For node 'n', return a reference to an instance of 'name'." + + # Get a class reference. + + ref = self.get_literal_builtin(name) + + # Obtain the details of the literal itself. + # An alias to the type is generated for sequences. + + if name in ("dict", "list", "tuple"): + self.set_special_literal(name, ref) + return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) + + # Constant values are independently recorded. + + else: + return self.get_constant_reference(ref, n.value) + + def get_literal_builtin(self, name): + + "Return a reference for a built-in literal type of the given 'name'." + + ref = self.get_builtin(name) + true_origin = "__builtins__.%s.%s" % (name, name) + exposed_origin = "__builtins__.%s" % name + + # Obtain fully-imported built-in class references. + + if ref and ref.has_kind(""): + pass + + # Early-stage references need explicit references. + + elif ref: + ref = Reference("", true_origin) + + # Otherwise, the normal locations can be used. + + else: + ref = Reference("", true_origin, exposed_origin) + + return ref + + # Functions and invocations. + + def allocate_arguments(self, path, args): + + """ + Allocate temporary argument storage using current and maximum + requirements for the given 'path' and 'args'. + """ + + init_item(self.function_targets, path, lambda: [0, 0]) + t = self.function_targets[path] + t[0] += 1 + t[1] = max(t[0], t[1]) + + init_item(self.function_arguments, path, lambda: [0, 0]) + t = self.function_arguments[path] + t[0] += len(args) + 1 + t[1] = max(t[0], t[1]) + + def deallocate_arguments(self, path, args): + + "Deallocate temporary argument storage for the given 'path' and 'args'." + + self.function_targets[path][0] -= 1 + self.function_arguments[path][0] -= len(args) + 1 + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa internal_tests/branches.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/internal_tests/branches.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,627 @@ +#!/usr/bin/env python + +import branching + +names = [] + +# Equivalent to... +# +# a = ... +# a.p +# if ...: +# a = ... +# a.x +# else: +# ... +# a.q + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.new_branchpoint() # begin +bt.new_branch() # if ... +a2 = bt.assign_names(["a"]) # a = ... +ax = bt.use_attribute("a", "x") +bt.shelve_branch() +bt.new_branch() # else +bt.shelve_branch() +bt.merge_branches() # end +aq = bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([('p',), ('p', 'q')])}, \ + a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('q', 'x')])}, \ + a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ax) == [1], \ + bt.get_assignment_positions_for_branches("a", ax) +print bt.get_assignment_positions_for_branches("a", aq) == [0, 1], \ + bt.get_assignment_positions_for_branches("a", aq) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p +# if ...: +# a.x +# elif ...: +# a.y; a.z +# else: +# ... +# a.q + +bt = branching.BranchTracker() +a = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.new_branchpoint() # begin +bt.new_branch() # if ... +ax = bt.use_attribute("a", "x") +bt.shelve_branch() +bt.new_branch() # elif ... +ay = bt.use_attribute("a", "y") +az = bt.use_attribute("a", "z") +bt.shelve_branch() +bt.new_branch() # else +bt.shelve_branch() +bt.merge_branches() # end +bt.use_attribute("a", "q") + +print a.get_usage() == \ + {'a' : set([('p', 'q'), ('p', 'q', 'x'), ('p', 'q', 'y', 'z')])}, \ + a.get_usage() +print bt.get_assignment_positions_for_branches("a", ax) == [0], \ + bt.get_assignment_positions_for_branches("a", ax) +print bt.get_assignment_positions_for_branches("a", ay) == [0], \ + bt.get_assignment_positions_for_branches("a", ay) +print bt.get_assignment_positions_for_branches("a", az) == [0], \ + bt.get_assignment_positions_for_branches("a", az) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p +# while ...: +# a.x +# a.q + +bt = branching.BranchTracker() +a = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.new_branchpoint(True) # begin +bt.new_branch(True) # while ... +ax = bt.use_attribute("a", "x") +bt.resume_continuing_branches() +bt.shelve_branch(True) +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_broken_branches() +bt.use_attribute("a", "q") + +print a.get_usage() == \ + {'a' : set([('p', 'q'), ('p', 'q', 'x')])}, a.get_usage() +print bt.get_assignment_positions_for_branches("a", ax) == [0], \ + bt.get_assignment_positions_for_branches("a", ax) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p +# while ...: +# if ...: +# a.x +# else ...: +# a.y +# a.q + +bt = branching.BranchTracker() +a = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.new_branchpoint(True) # begin +bt.new_branch(True) # while ... +bt.new_branchpoint() # begin +bt.new_branch() # if ... +ax = bt.use_attribute("a", "x") +bt.shelve_branch() +bt.new_branch() +ay = bt.use_attribute("a", "y") +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_continuing_branches() +bt.shelve_branch(True) +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_broken_branches() +bt.use_attribute("a", "q") + +print a.get_usage() == \ + {'a' : set([('p', 'q'), ('p', 'q', 'x'), ('p', 'q', 'y')])}, \ + a.get_usage() +print bt.get_assignment_positions_for_branches("a", ax) == [0], \ + bt.get_assignment_positions_for_branches("a", ax) +print bt.get_assignment_positions_for_branches("a", ay) == [0], \ + bt.get_assignment_positions_for_branches("a", ay) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p +# while ...: +# if ...: +# a = ... +# a.x +# else ...: +# a.y +# a.q + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.new_branchpoint(True) # begin +bt.new_branch(True) # while ... +bt.new_branchpoint() # begin +bt.new_branch() # if ... +a2 = bt.assign_names(["a"]) # a = ... +ax = bt.use_attribute("a", "x") +bt.shelve_branch() +bt.new_branch() +ay = bt.use_attribute("a", "y") +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_continuing_branches() +bt.shelve_branch(True) +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_broken_branches() +bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([('p', 'q'), ('p', 'q', 'y'), ('p',)])}, a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('q', 'x')])}, a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ax) == [1], \ + bt.get_assignment_positions_for_branches("a", ax) +print bt.get_assignment_positions_for_branches("a", ay) == [0, 1], \ + bt.get_assignment_positions_for_branches("a", ay) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p +# while ...: +# if ...: +# a.y +# else ...: +# a = ... +# a.x +# a.q + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.new_branchpoint(True) # begin +bt.new_branch(True) # while ... +bt.new_branchpoint() # begin +bt.new_branch() # if ... +ay = bt.use_attribute("a", "y") +bt.shelve_branch() +bt.new_branch() +a2 = bt.assign_names(["a"]) # a = ... +ax = bt.use_attribute("a", "x") +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_continuing_branches() +bt.shelve_branch(True) +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_broken_branches() +bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([('p', 'q'), ('p', 'q', 'y'), ('p',)])}, a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('q', 'x')])}, a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ax) == [1], \ + bt.get_assignment_positions_for_branches("a", ax) +print bt.get_assignment_positions_for_branches("a", ay) == [0, 1], \ + bt.get_assignment_positions_for_branches("a", ay) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p +# while ...: +# a = ... +# a.x +# a.q + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +ap = bt.use_attribute("a", "p") +bt.new_branchpoint(True) # begin +bt.new_branch(True) # while ... +a2 = bt.assign_names(["a"]) # a = ... +ax = bt.use_attribute("a", "x") +bt.resume_continuing_branches() +bt.shelve_branch(True) +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.resume_broken_branches() +aq = bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([('p', 'q'), ('p',)])}, a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('q', 'x')])}, a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ax) == [1], \ + bt.get_assignment_positions_for_branches("a", ax) +print bt.get_assignment_positions_for_branches("a", ap) == [0], \ + bt.get_assignment_positions_for_branches("a", ap) +print bt.get_assignment_positions_for_branches("a", aq) == [0, 1], \ + bt.get_assignment_positions_for_branches("a", aq) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p +# while ...: +# if ...: +# break +# a.q +# a.r + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.new_branchpoint(True) # begin +bt.new_branch(True) # while ... +bt.new_branchpoint() # begin +bt.new_branch() # if ... +bt.suspend_broken_branch() # break +bt.shelve_branch() +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.use_attribute("a", "q") +bt.resume_continuing_branches() +bt.shelve_branch(True) +bt.merge_branches() # end +bt.resume_broken_branches() +bt.use_attribute("a", "r") + +print a1.get_usage() == \ + {'a' : set([('p', 'q', 'r'), ('p', 'r')])}, a1.get_usage() +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# a.p and a.q and a.r + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin +bt.new_branch() +bt.use_attribute("a", "p") +bt.new_branchpoint() # begin +bt.new_branch() +bt.use_attribute("a", "q") +bt.new_branchpoint() # begin +bt.new_branch() +bt.use_attribute("a", "r") +bt.shelve_branch() +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.shelve_branch() +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.shelve_branch() +bt.merge_branches() # end + +print a1.get_usage() == \ + {'a' : set([('p', 'q', 'r'), ('p', 'q'), ('p',)])}, a1.get_usage() +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# if ...: +# a.p +# return +# a.q + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin +bt.new_branch() # if ... +bt.use_attribute("a", "p") +bt.abandon_returning_branch() +bt.shelve_branch() +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([('p',), ('q',)])}, a1.get_usage() +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# try: +# if ...: +# a.p +# return +# a.q +# except: +# a.r + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin (try) +bt.new_branchpoint() # begin +bt.new_branch() # if ... +bt.use_attribute("a", "p") +bt.abandon_returning_branch() +bt.shelve_branch() # ... if +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.use_attribute("a", "q") +bt.resume_abandoned_branches() # except +bt.use_attribute("a", "r") +bt.shelve_branch() +bt.merge_branches() # end + +print a1.get_usage() == \ + {'a' : set([('p',), ('q', 'r')])}, a1.get_usage() +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# if ...: +# a.p +# a = ... +# if ...: +# a.q + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin +bt.new_branch() # if ... +ap = bt.use_attribute("a", "p") +bt.abandon_branch() +bt.shelve_branch() # ... if +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +a2 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin +bt.new_branch() # if ... +aq = bt.use_attribute("a", "q") +bt.abandon_branch() +bt.shelve_branch() # ... if +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end + +print a1.get_usage() == \ + {'a' : set([('p',), ()])}, a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('q',), ()])}, a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ap) == [0], \ + bt.get_assignment_positions_for_branches("a", ap) +print bt.get_assignment_positions_for_branches("a", aq) == [1], \ + bt.get_assignment_positions_for_branches("a", aq) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = {} +# a.p +# if ...: +# a = ... +# a.x +# else: +# ... +# a.q + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"], [":__builtins__.dict.dict"]) +ap = bt.use_attribute("a", "p") +bt.new_branchpoint() # begin +bt.new_branch() # if ... +a2 = bt.assign_names(["a"]) # a = ... +ax = bt.use_attribute("a", "x") +bt.shelve_branch() +bt.new_branch() # else +bt.shelve_branch() +bt.merge_branches() # end +aq = bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([('p',), ('p', 'q')])}, \ + a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('q', 'x')])}, \ + a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ap) == [0], \ + bt.get_assignment_positions_for_branches("a", ap) +print bt.get_assignment_positions_for_branches("a", ax) == [1], \ + bt.get_assignment_positions_for_branches("a", ax) +print bt.get_assignment_positions_for_branches("a", aq) == [0, 1], \ + bt.get_assignment_positions_for_branches("a", aq) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# if ...: +# a = ... +# a.x +# else: +# ... +# a.q + +bt = branching.BranchTracker() +bt.new_branchpoint() # begin +bt.new_branch() # if ... +a1 = bt.assign_names(["a"]) # a = ... +ax = bt.use_attribute("a", "x") +bt.shelve_branch() +bt.new_branch() # else +bt.shelve_branch() +bt.merge_branches() # end +aq = bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([('q', 'x')])}, \ + a1.get_usage() +print bt.get_assignment_positions_for_branches("a", aq) == [None, 0], \ + bt.get_assignment_positions_for_branches("a", aq) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# if ...: +# a = ... +# return +# a.q + +bt = branching.BranchTracker() +bt.new_branchpoint() # begin +bt.new_branch() # if ... +a1 = bt.assign_names(["a"]) +bt.abandon_returning_branch() +bt.shelve_branch() +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +aq = bt.use_attribute("a", "q") + +print a1.get_usage() == \ + {'a' : set([()])}, a1.get_usage() +print bt.get_assignment_positions_for_branches("a", aq) == [None], \ + bt.get_assignment_positions_for_branches("a", aq) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# try: +# if ...: +# a.p +# return +# a.q +# finally: +# a.r + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin +bt.new_branch() # if ... +bt.use_attribute("a", "p") +bt.abandon_returning_branch() +bt.shelve_branch() # ... if +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.use_attribute("a", "q") +branches = bt.resume_all_abandoned_branches() +bt.use_attribute("a", "r") +bt.restore_active_branches(branches) + +print a1.get_usage() == \ + {'a' : set([('p', 'r'), ('q', 'r')])}, a1.get_usage() +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# try: +# if ...: +# a = ... +# a.p +# return +# a.q +# finally: +# a.r + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin +bt.new_branch() # if ... +a2 = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.abandon_returning_branch() +bt.shelve_branch() # ... if +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +aq = bt.use_attribute("a", "q") +branches = bt.resume_all_abandoned_branches() +ar = bt.use_attribute("a", "r") +bt.restore_active_branches(branches) + +print a1.get_usage() == \ + {'a' : set([(), ('q', 'r')])}, a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('p', 'r')])}, a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ar) == [0, 1], \ + bt.get_assignment_positions_for_branches("a", ar) +names.append(bt.assignments["a"]) + +# Equivalent to... +# +# a = ... +# try: +# if ...: +# a = ... +# a.p +# return +# a.q +# except: +# a.r + +bt = branching.BranchTracker() +a1 = bt.assign_names(["a"]) +bt.new_branchpoint() # begin (try) +bt.new_branchpoint() # begin +bt.new_branch() # if ... +a2 = bt.assign_names(["a"]) +bt.use_attribute("a", "p") +bt.abandon_returning_branch() +bt.shelve_branch() # ... if +bt.new_branch() # (null) +bt.shelve_branch() +bt.merge_branches() # end +bt.use_attribute("a", "q") +bt.resume_abandoned_branches() # except +ar = bt.use_attribute("a", "r") +bt.shelve_branch() +bt.merge_branches() # end + +print a1.get_usage() == \ + {'a' : set([(), ('q', 'r')])}, a1.get_usage() +print a2.get_usage() == \ + {'a' : set([('p',)])}, a2.get_usage() +print bt.get_assignment_positions_for_branches("a", ar) == [0, 1], \ + bt.get_assignment_positions_for_branches("a", ar) +names.append(bt.assignments["a"]) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa internal_tests/references.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/internal_tests/references.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +from referencing import decode_reference, Reference + +def show_test(v1, v2): + print "%r %r %r" % (v1 == v2, v1, v2) + +# Compare decoded and constructed references. + +var1 = decode_reference("") +var2 = Reference("") +show_test(var1, var2) + +# Compare with var with superfluous origin. + +var3 = Reference("", "whatever") +show_test(var1, var3) + +# Compare with var and alias. + +var4 = Reference("", None, "attribute") +show_test(var1, var4) + +# Compare with var with superfluous origin and alias. + +var5 = Reference("", "whatever", "attribute") +show_test(var1, var5) +show_test(var5.get_origin(), None) + +# Compare vars with different aliases. + +var6 = Reference("", None, "other") +show_test(var4, var6) + +# Check aliased var. + +var7 = var1.alias("attribute") +show_test(var7, var4) + +# Check class references, firstly with someclass being identified as a class. + +cls1 = decode_reference("", "someclass") +cls2 = Reference("", "someclass") +show_test(cls1, cls2) + +# Check aliasing of class references. + +cls3 = cls1.alias("attribute") +cls4 = cls2.alias("other") +show_test(cls3, cls4) + +# Check other class references. + +cls5 = decode_reference(":someclass") +cls6 = Reference("", "someclass") +show_test(cls5, cls6) + +# Check aliasing again. + +cls7 = cls5.alias("attribute") +cls8 = cls6.alias("other") +show_test(cls7, cls8) + +# Check instance references. These do not make sense without an origin. + +inst1 = decode_reference(":someclass", "whatever") +inst2 = Reference("", "someclass") +show_test(inst1, inst2) + +# Check instantiation. + +inst3 = cls5.instance_of() +show_test(inst1, inst3) + +# Check modules. + +mod1 = decode_reference("somemodule") +mod2 = Reference("", "somemodule") +show_test(mod1, mod2) + +mod3 = decode_reference(":somemodule") +show_test(mod1, mod3) + +mod4 = decode_reference("", "somemodule") +show_test(mod1, mod4) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa lplc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lplc Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from errors import * +from os.path import abspath, exists, join, split +from time import time +import importer +import sys + +libdirs = [ + join(split(__file__)[0], "lib"), + "/usr/share/lichen/lib" + ] + +def load_module(filename, module_name): + for libdir in libdirs: + path = join(libdir, filename) + if exists(path): + return i.load_from_file(path, module_name) + return None + +def stopwatch(activity, now): + print >>sys.stderr, "%s took %.2f seconds" % (activity, time() - now) + return time() + +# Main program. + +if __name__ == "__main__": + args = sys.argv[2:] + path = libdirs + sys.path[:] + + filename = abspath(sys.argv[1]) + path.append(split(filename)[0]) + + verbose = "-v" in args + reset = "-r" in args + + # Load the program. + + try: + start = now = time() + + i = importer.Importer(path, "_cache", verbose) + m = i.initialise(filename, reset) + i.finalise() + + now = stopwatch("Inspection", now) + + # Report any errors. + + except ProcessingError, exc: + print exc + if "-tb" in args: + raise + elif "-exit" in args: + sys.exit(1) + + except KeyboardInterrupt: + if "-exit" in args: + sys.exit(2) + else: + raise + + else: + if "-exit" in args: + sys.exit(0) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa modules.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/modules.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,1229 @@ +#!/usr/bin/env python + +""" +Module abstractions. + +Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, + 2014, 2015, 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +from common import * +from encoders import decode_modifier_term, encode_modifiers, encode_usage +from referencing import decode_reference, Reference +import sys + +class BasicModule(CommonModule): + + "The basic module information." + + def __init__(self, name, importer): + CommonModule.__init__(self, name, importer) + + # Import machinery links. + + self.loaded = False + + # Module dependencies. + + self.imported = set() + self.imported_hidden = set() + self.imported_names = {} + self.revealed = set() + self.accessing_modules = set() + + # Global name information. + + self.objects = {} + self.special = {} + + # Class relationships. + + self.classes = {} + + # Attributes. + + self.class_attrs = {} + self.instance_attrs = {} + self.instance_attr_constants = {} + self.module_attrs = set() + + # Names used and missing. + + self.names_used = {} + self.names_missing = {} + self.name_references = {} # references to globals + + # Function details. + + self.function_parameters = {} + self.function_defaults = {} + self.function_locals = {} + self.scope_globals = {} + + # Invocation details. + + self.function_targets = {} + self.function_arguments = {} + + # Attribute usage at module and function levels. + + self.attr_usage = {} + self.name_initialisers = {} + + # General attribute access expressions. + + self.attr_accesses = {} + self.const_accesses = {} + + # Attribute accessor definition details. + + self.attr_accessors = {} + + # Assignment details for accesses. + + self.attr_access_modifiers = {} + + # Initialisation-related details. + + self.initialised_names = {} + self.aliased_names = {} + + def __repr__(self): + return "BasicModule(%r, %r)" % (self.name, self.importer) + + def resolve(self): + + "Resolve dependencies and complete definitions." + + self.resolve_class_bases() + self.check_special() + self.check_names_used() + self.resolve_members() + self.resolve_initialisers() + self.resolve_literals() + self.remove_redundant_accessors() + self.set_invocation_usage() + + # Propagate to the importer information needed in subsequent activities. + + self.propagate() + + # Derived information methods. + + def propagate(self): + + "Finalise and propagate module information." + + self.propagate_attrs() + self.propagate_name_references() + self.propagate_attr_accesses() + self.propagate_constants() + + def unpropagate(self): + + """ + Retract information from the importer including information about this + module derived by the importer. + """ + + del self.importer.all_module_attrs[self.name] + + for name in self.classes.keys(): + del self.importer.all_class_attrs[name] + del self.importer.all_instance_attrs[name] + del self.importer.all_combined_attrs[name] + del self.importer.all_instance_attr_constants[name] + + for name, bases in self.classes.items(): + for base in bases: + + # Get the identity of the class from the reference. + + base = base.get_origin() + + try: + self.importer.subclasses[base].remove(name) + except (KeyError, ValueError): + pass + + remove_items(self.importer.all_name_references, self.name_references) + remove_items(self.importer.all_initialised_names, self.initialised_names) + remove_items(self.importer.all_aliased_names, self.aliased_names) + remove_items(self.importer.all_attr_accesses, self.attr_accesses) + remove_items(self.importer.all_const_accesses, self.const_accesses) + remove_items(self.importer.all_attr_access_modifiers, self.attr_access_modifiers) + remove_items(self.importer.all_constants, self.constants) + remove_items(self.importer.all_constant_values, self.constant_values) + + # Remove this module's objects from the importer. Objects are + # automatically propagated when defined. + + for name, ref in self.objects.items(): + if ref.provided_by_module(self.name) or name in self.importer.hidden: + if ref.get_kind() != "": + del self.importer.objects[name] + + def resolve_class_bases(self): + + "Resolve all class bases since some of them may have been deferred." + + for name, bases in self.classes.items(): + resolved = [] + bad = [] + + for base in bases: + + # Resolve dependencies. + + if base.has_kind(""): + ref = self.importer.get_object(base.get_origin()) + else: + ref = base + + # Obtain the origin of the base class reference. + + if not ref or not ref.has_kind(""): + bad.append(base) + break + + resolved.append(ref) + + if bad: + print >>sys.stderr, "Bases of class %s were not classes." % (name, ", ".join(map(str, bad))) + else: + self.importer.classes[name] = self.classes[name] = resolved + + def propagate_attrs(self): + + "Derive attributes from the class and module member details." + + # Initialise class attribute records for all classes. + + for name in self.classes.keys(): + self.importer.all_class_attrs[name] = self.class_attrs[name] = {} + + # Separate the objects into module and class attributes. + + for name in self.objects.keys(): + if "." in name: + parent, attrname = name.rsplit(".", 1) + if self.classes.has_key(parent): + self.class_attrs[parent][attrname] = name + elif parent == self.name: + self.module_attrs.add(attrname) + + # Propagate the module attributes. + + self.importer.all_module_attrs[self.name] = self.module_attrs + + def propagate_name_references(self): + + "Propagate name references for the module." + + self.importer.all_name_references.update(self.name_references) + self.importer.all_initialised_names.update(self.initialised_names) + self.importer.all_aliased_names.update(self.aliased_names) + + def propagate_attr_accesses(self): + + "Propagate attribute accesses for the module." + + self.importer.all_attr_accesses.update(self.attr_accesses) + self.importer.all_const_accesses.update(self.const_accesses) + self.importer.all_attr_access_modifiers.update(self.attr_access_modifiers) + + def propagate_constants(self): + + "Propagate constant values and aliases for the module." + + self.importer.all_constants.update(self.constants) + self.importer.all_constant_values.update(self.constant_values) + + for name in self.classes.keys(): + self.importer.all_instance_attrs[name] = self.instance_attrs.get(name) or {} + self.importer.all_instance_attr_constants[name] = self.instance_attr_constants.get(name) or {} + + # Module-relative naming. + + def is_global(self, name): + + """ + Return whether 'name' is registered as a global in the current + namespace. + """ + + path = self.get_namespace_path() + return name in self.scope_globals.get(path, []) + + def get_globals(self): + + """ + Get the globals from this module, returning a dictionary mapping names + to references incorporating original definition details. + """ + + l = [] + for name, value in self.objects.items(): + parent, attrname = name.rsplit(".", 1) + if parent == self.name: + l.append((attrname, value)) + return dict(l) + + def get_global(self, name): + + """ + Get the global of the given 'name' from this module, returning a + reference incorporating the original definition details. + """ + + path = self.get_global_path(name) + return self.objects.get(path) + + # Name definition discovery. + + def get_global_or_builtin(self, name): + + """ + Return the object recorded the given 'name' from this module or from the + __builtins__ module. If no object has yet been defined, perhaps due to + circular module references, None is returned. + """ + + return self.get_global(name) or self.get_builtin(name) + + def get_builtin(self, name): + + "Return the object providing the given built-in 'name'." + + path = "__builtins__.%s" % name + + # Obtain __builtins__. + + module = self.ensure_builtins(path) + + # Attempt to find the named object within __builtins__. + + if module: + self.find_imported_name(name, module.name, module) + + # Return the path and any reference for the named object. + + return self.importer.get_object(path) + + def ensure_builtins(self, path): + + """ + If 'path' is a reference to an object within __builtins__, return the + __builtins__ module. + """ + + if path.split(".", 1)[0] == "__builtins__": + return self.importer.load("__builtins__", True, True) + else: + return None + + def get_object(self, path): + + """ + Get the details of an object with the given 'path', either from this + module or from the whole program. Return a tuple containing the path and + any object found. + """ + + if self.objects.has_key(path): + return self.objects[path] + else: + self.ensure_builtins(path) + return self.importer.get_object(path) + + def set_object(self, name, value=None): + + "Set an object with the given 'name' and the given 'value'." + + ref = decode_reference(value, name) + multiple = self.objects.has_key(name) and self.objects[name].get_kind() != ref.get_kind() + self.importer.objects[name] = self.objects[name] = multiple and ref.as_var() or ref + + # Special names. + + def get_special(self, name): + + "Return any stored value for the given special 'name'." + + return self.special.get(name) + + def set_special(self, name, value): + + """ + Set a special 'name' that merely tracks the use of an implicit object + 'value'. + """ + + self.special[name] = value + + def set_special_literal(self, name, ref): + + """ + Set a special name for the literal type 'name' having type 'ref'. Such + special names provide a way of referring to literal object types. + """ + + literal_name = "$L%s" % name + value = ResolvedNameRef(literal_name, ref) + self.set_special(literal_name, value) + + # Revealing modules by tracking name imports across modules. + + def set_imported_name(self, name, module_name, alias=None, path=None): + + """ + Record 'name' as being imported from the given 'module_name', employing + the given 'alias' in the local namespace if specified. + """ + + path = path or self.get_namespace_path() + init_item(self.imported_names, path, dict) + self.imported_names[path][alias or name] = (name, module_name) + + def get_imported_name(self, name, path): + + "Get details of any imported 'name' within the namespace at 'path'." + + if self.imported_names.has_key(path): + return self.imported_names[path].get(name) + else: + return None + + def find_imported_name(self, name, path, module=None): + + """ + Find details of the imported 'name' within the namespace at 'path', + starting within the given 'module' if indicated, or within this module + otherwise. + """ + + module = module or self + + # Obtain any module required by the name. + + name_modname = module.get_imported_name(name, path) + + if name_modname: + name, modname = name_modname + module = self._find_imported_name(name, modname, module) + + # Obtain the name from the final module, revealing it if appropriate. + + if module: + init_item(self.importer.revealing, module.name, set) + self.importer.set_revealing(module, name, self) + + def _find_imported_name(self, name, modname, module): + + """ + Traverse details for 'name' via 'modname' to the module providing the + name, tentatively revealing the module even if the module is not yet + loaded and cannot provide the details of the object recorded for the + name. + """ + + _name = name + + # Obtain any modules referenced by each required module. + + while True: + + # Get the module directly or traverse possibly-aliased names. + + module = self.get_module_direct(modname, True) + if not module: + top, module = self.get_module(modname, True) + name_modname = module.get_imported_name(_name, module.name) + if not name_modname: + break + else: + _name, modname = name_modname + + return module + + def reveal_referenced(self): + + """ + Reveal modules referenced by this module. + """ + + for path, names in self.imported_names.items(): + for alias, (name, modname) in names.items(): + module = self._find_imported_name(name, modname, self) + self.reveal_module(module) + + def reveal_module(self, module): + + """ + Reveal the given 'module', recording the revealed modules on this + module. + """ + + if module is not self: + self.importer.reveal_module(module) + self.revealed.add(module) + module.accessing_modules.add(self.name) + + # Module loading. + + def get_module_direct(self, modname, hidden=False): + + """ + Return 'modname' without traversing parent modules, keeping the module + 'hidden' if set to a true value, loading the module if not already + loaded. + """ + + return self.importer.get_module(modname, True) or self.importer.load(modname, hidden=hidden) + + def get_module(self, name, hidden=False): + + """ + Use the given 'name' to obtain the identity of a module. Return module + objects or None if the module cannot be found. This method is required + when aliases are used to refer to modules and where a module "path" does + not correspond to the actual module path. + + A tuple is returned containing the top or base module and the deepest or + leaf module involved. + """ + + path_so_far = [] + top = module = None + parts = name.split(".") + + for i, part in enumerate(parts): + path_so_far.append(part) + module_name = ".".join(path_so_far) + ref = self.get_object(module_name) + + # If no known object exists, attempt to load it. + + if not ref: + module = self.importer.load(module_name, True, hidden) + if not module: + return None + + # Rewrite the path to use the actual module details. + + path_so_far = module.name.split(".") + + # If the object exists and is not a module, stop. + + elif ref.has_kind(["", "", ""]): + return None + + else: + module = self.importer.get_module(ref.get_origin(), hidden) + + if not top: + top = module + + return top, module + +class CachedModule(BasicModule): + + "A cached module." + + def __repr__(self): + return "CachedModule(%r, %r)" % (self.name, self.importer) + + def resolve(self): + pass + + def to_cache(self, filename): + + "Not actually writing the module back to 'filename'." + + pass + + def from_cache(self, filename): + + """ + Read a module's details from the file with the given 'filename' as + described in the to_cache method of InspectedModule. + """ + + f = open(filename) + try: + self.filename = f.readline().rstrip() + + accessing_modules = f.readline().split(": ", 1)[-1].rstrip() + + module_names = f.readline().split(": ", 1)[-1].rstrip() + if module_names: + for module_name in module_names.split(", "): + self.imported.add(self.importer.load(module_name)) + + module_names = f.readline().split(": ", 1)[-1].rstrip() + if module_names: + for module_name in module_names.split(", "): + self.imported_hidden.add(self.importer.load(module_name, hidden=True)) + + module_names = f.readline().split(": ", 1)[-1].rstrip() + if module_names: + for module_name in module_names.split(", "): + module = self.importer.load(module_name, True, True) + self.reveal_module(module) + + f.readline() # (empty line) + + self._get_imported_names(f) + self._get_members(f) + self._get_class_relationships(f) + self._get_instance_attrs(f) + self._get_instance_attr_constants(f) + self.from_lines(f, self.names_used) # "names used:" + self.from_lines(f, self.names_missing) # "names missing:" + self._get_name_references(f) + self._get_initialised_names(f) + self._get_aliased_names(f) + self._get_function_parameters(f) + self._get_function_defaults(f) + self._get_function_locals(f) + self.from_lines(f, self.scope_globals) # "scope globals:" + self._get_function_targets(f) + self._get_function_arguments(f) + self._get_attribute_usage(f) + self._get_attr_accesses(f) + self._get_const_accesses(f) + self._get_attr_accessors(f) + self._get_attr_access_modifiers(f) + self._get_constant_literals(f) + self._get_constant_values(f) + + finally: + f.close() + + self.loaded = True + + def resolve(self): + self.propagate() + + def _get_imported_names(self, f): + f.readline() # "imported names:" + line = f.readline().rstrip() + while line: + path, alias_or_name, name, module_name = self._get_fields(line, 4) + init_item(self.imported_names, path, dict) + self.imported_names[path][alias_or_name] = (name, module_name) + line = f.readline().rstrip() + + def _get_members(self, f): + f.readline() # "members:" + line = f.readline().rstrip() + while line: + name, ref = line.split(" ", 1) + self.set_object(name, ref) + line = f.readline().rstrip() + + def _get_class_relationships(self, f): + f.readline() # "class relationships:" + line = f.readline().rstrip() + while line: + name, value = self._get_fields(line) + values = value and value.split(", ") or [] + self.importer.classes[name] = self.classes[name] = map(decode_reference, values) + self.importer.subclasses[name] = set() + line = f.readline().rstrip() + + def _get_instance_attrs(self, f): + f.readline() # "instance attributes:" + line = f.readline().rstrip() + while line: + name, value = self._get_fields(line) + self.importer.all_instance_attrs[name] = self.instance_attrs[name] = set(value and value.split(", ") or []) + line = f.readline().rstrip() + + def _get_instance_attr_constants(self, f): + f.readline() # "instance attribute constants:" + line = f.readline().rstrip() + while line: + name, attrname, ref = self._get_fields(line, 3) + init_item(self.instance_attr_constants, name, dict) + self.instance_attr_constants[name][attrname] = decode_reference(ref) + line = f.readline().rstrip() + + def _get_name_references(self, f): + f.readline() # "name references:" + line = f.readline().rstrip() + while line: + name, value = self._get_fields(line) + self.name_references[name] = value + line = f.readline().rstrip() + + def _get_initialised_names(self, f): + f.readline() # "initialised names:" + line = f.readline().rstrip() + while line: + name, version, value = self._get_fields(line, 3) + init_item(self.initialised_names, name, dict) + self.initialised_names[name][int(version)] = decode_reference(value) + line = f.readline().rstrip() + + def _get_aliased_names(self, f): + f.readline() # "aliased names:" + line = f.readline().rstrip() + while line: + name, version, original_name, attrnames, number = self._get_fields(line, 5) + init_item(self.aliased_names, name, dict) + if number == "{}": number = None + else: number = int(number) + self.aliased_names[name][int(version)] = (original_name, attrnames != "{}" and attrnames or None, number) + line = f.readline().rstrip() + + def _get_function_parameters(self, f): + f.readline() # "function parameters:" + line = f.readline().rstrip() + while line: + function, names = self._get_fields(line) + self.importer.function_parameters[function] = \ + self.function_parameters[function] = names and names.split(", ") or [] + line = f.readline().rstrip() + + def _get_function_defaults(self, f): + f.readline() # "function default parameters:" + line = f.readline().rstrip() + while line: + function, defaults = self._get_fields(line) + self.importer.function_defaults[function] = \ + self.function_defaults[function] = l = [] + if defaults != "{}": + for value in defaults.split(", "): + name, default = value.split("=") + default = decode_reference(default) + l.append((name, default)) + line = f.readline().rstrip() + + def _get_function_locals(self, f): + f.readline() # "function locals:" + line = f.readline().rstrip() + while line: + function, name, value = self._get_fields(line, 3) + init_item(self.function_locals, function, dict) + if name != "{}": + self.function_locals[function][name] = decode_reference(value) + line = f.readline().rstrip() + + def _get_function_targets(self, f): + f.readline() # "function targets:" + line = f.readline().rstrip() + while line: + function, n = self._get_fields(line) + self.importer.function_targets[function] = \ + self.function_targets[function] = int(n) + line = f.readline().rstrip() + + def _get_function_arguments(self, f): + f.readline() # "function arguments:" + line = f.readline().rstrip() + while line: + function, n = self._get_fields(line) + self.importer.function_arguments[function] = \ + self.function_arguments[function] = int(n) + line = f.readline().rstrip() + + def _get_attribute_usage(self, f): + f.readline() # "attribute usage:" + line = f.readline().rstrip() + while line: + unit, value = self._get_fields(line) + init_item(self.attr_usage, unit, dict) + self.usage_from_cache(value, self.attr_usage[unit]) + line = f.readline().rstrip() + + def _get_attr_accesses(self, f): + f.readline() # "attribute accesses:" + line = f.readline().rstrip() + while line: + name, value = self._get_fields(line) + self.attr_accesses[name] = set(value.split(", ")) + line = f.readline().rstrip() + + def _get_const_accesses(self, f): + f.readline() # "constant accesses:" + line = f.readline().rstrip() + while line: + name, original_name, attrnames, objpath, ref, remaining = self._get_fields(line, 6) + if attrnames == "{}": attrnames = None + init_item(self.const_accesses, name, dict) + self.const_accesses[name][(original_name, attrnames)] = (objpath, decode_reference(ref), remaining != "{}" and remaining or "") + line = f.readline().rstrip() + + def _get_attr_accessors(self, f): + f.readline() # "attribute access usage:" + line = f.readline().rstrip() + while line: + objpath, name, attrname, value = self._get_fields(line, 4) + if attrname == "{}": attrname = None + access = name, attrname + init_item(self.attr_accessors, objpath, dict) + init_item(self.attr_accessors[objpath], access, list) + positions = map(int, value.split(", ")) + self.attr_accessors[objpath][access].append(positions) + line = f.readline().rstrip() + + def _get_attr_access_modifiers(self, f): + f.readline() # "attribute access modifiers:" + line = f.readline().rstrip() + while line: + objpath, name, attrnames, value = self._get_fields(line, 4) + if name == "{}": name = None + if attrnames == "{}": attrnames = None + access = name, attrnames + init_item(self.attr_access_modifiers, objpath, dict) + init_item(self.attr_access_modifiers[objpath], access, list) + modifiers = [decode_modifier_term(s) for s in value] + self.attr_access_modifiers[objpath][access] = modifiers + line = f.readline().rstrip() + + def _get_constant_literals(self, f): + f.readline() # "constant literals:" + line = f.readline().rstrip() + last_path = None + n = None + while line: + path, constant = self._get_fields(line) + if path != last_path: + n = 0 + last_path = path + else: + n += 1 + init_item(self.constants, path, dict) + self.constants[path][eval(constant)] = n + line = f.readline().rstrip() + + def _get_constant_values(self, f): + f.readline() # "constant values:" + line = f.readline().rstrip() + while line: + name, value_type, value = self._get_fields(line, 3) + self.constant_values[name] = eval(value), value_type + line = f.readline().rstrip() + + # Generic parsing methods. + + def from_lines(self, f, d): + + "Read lines from 'f', populating 'd'." + + f.readline() # section heading + line = f.readline().rstrip() + while line: + name, value = self._get_fields(line) + d[name] = set(value and value.split(", ") or []) + line = f.readline().rstrip() + + def usage_from_cache(self, value, mapping): + + """ + Interpret the given 'value' containing name and usage information, + storing the information in the given 'mapping'. + """ + + local, usage = self._get_fields(value) + init_item(mapping, local, list) + self._usage_from_cache(mapping[local], usage) + + def _usage_from_cache(self, d, usage): + + # Interpret descriptions of each version of the name. + + all_usages = set() + for attrnames in usage.split("; "): + if attrnames == "{}": + all_attrnames = () + else: + # Decode attribute details for each usage description. + + all_attrnames = set() + for attrname_str in attrnames.split(", "): + all_attrnames.add(attrname_str) + + all_attrnames = list(all_attrnames) + all_attrnames.sort() + + all_usages.add(tuple(all_attrnames)) + + d.append(all_usages) + + def _get_fields(self, s, n=2): + result = s.split(" ", n-1) + if len(result) == n: + return result + else: + return tuple(result) + tuple([""] * (n - len(result))) + +class CacheWritingModule: + + """ + A mix-in providing cache-writing support, to be combined with BasicModule. + """ + + def to_cache(self, filename): + + """ + Write a cached representation of the inspected module with the following + format to the file having the given 'filename': + + filename + "accessed by:" accessing module names during this module's import + "imports:" imported module names + "hidden imports:" imported module names + "reveals:" revealed module names + (empty line) + "imported names:" + zero or more: qualified name " " name " " module name + (empty line) + "members:" + zero or more: qualified name " " reference + (empty line) + "class relationships:" + zero or more: qualified class name " " base class references + (empty line) + "instance attributes:" + zero or more: qualified class name " " instance attribute names + (empty line) + "instance attribute constants:" + zero or more: qualified class name " " attribute name " " reference + (empty line) + "names used:" + zero or more: qualified class/function/module name " " names + (empty line) + "names missing:" + zero or more: qualified class/function/module name " " names + (empty line) + "name references:" + zero or more: qualified name " " reference + (empty line) + "initialised names:" + zero or more: qualified name " " definition version " " reference + (empty line) + "aliased names:" + zero or more: qualified name " " definition version " " original name " " attribute names " " access number + (empty line) + "function parameters:" + zero or more: qualified function name " " parameter names + (empty line) + "function default parameters:" + zero or more: qualified function name " " parameter names with defaults + (empty line) + "function locals:" + zero or more: qualified function name " " local variable name " " reference + (empty line) + "scope globals:" + zero or more: qualified function name " " global variable names + (empty line) + "function targets:" + zero or more: qualified function name " " maximum number of targets allocated + (empty line) + "function arguments:" + zero or more: qualified function name " " maximum number of arguments allocated + (empty line) + "attribute usage:" + zero or more: qualified scope name " " local/global/qualified variable name " " usages + (empty line) + "attribute accesses:" + zero or more: qualified scope name " " attribute-chains + (empty line) + "constant accesses:" + zero or more: qualified function name " " attribute-chain " " reference " " remaining attribute-chain + (empty line) + "attribute access usage:" + zero or more: qualified function name " " local/global variable name " " attribute name " " definition versions + (empty line) + "attribute access modifiers:" + zero or more: qualified function name " " local/global variable name " " attribute name " " access modifiers + "constant literals:" + zero or more: qualified scope name " " constant literal + "constant values:" + zero or more: qualified name " " value type " " constant literal + + All collections of names are separated by ", " characters. + + References can be "", a module name, or one of "" or + "" followed optionally by a ":" character and a qualified + name. + + Parameter names with defaults are separated by ", " characters, with + each name followed by "=" and then followed by a reference. If "{}" is + indicated, no defaults are defined for the function. Similarly, function + locals may be indicated as "{}" meaning that there are no locals. + + All usages (attribute usage sets) are separated by "; " characters, with + the special string "{}" representing an empty set. + + Each usage is a collection of names separated by ", " characters, with + assigned attribute names suffixed with a "*" character. + + Each attribute-chain expression is a dot-separated chain of attribute + names, with assignments suffixed with a "*" character. + + Definition versions are separated by ", " characters and indicate the + name definition version associated with the access. + + Access modifiers are separated by ", " characters and indicate features + of each access, with multiple accesses described on a single line. + """ + + f = open(filename, "w") + try: + print >>f, self.filename + + accessing_modules = list(self.accessing_modules) + accessing_modules.sort() + print >>f, "accessed by:", ", ".join(accessing_modules) + + module_names = [m.name for m in self.imported] + module_names.sort() + print >>f, "imports:", ", ".join(module_names) + + module_names = [m.name for m in self.imported_hidden] + module_names.sort() + print >>f, "hidden imports:", ", ".join(module_names) + + module_names = [m.name for m in self.revealed] + module_names.sort() + print >>f, "reveals:", ", ".join(module_names) + + print >>f + print >>f, "imported names:" + paths = self.imported_names.keys() + paths.sort() + for path in paths: + names = self.imported_names[path].keys() + names.sort() + for alias_or_name in names: + name, modname = self.imported_names[path][alias_or_name] + print >>f, path, alias_or_name, name, modname + + print >>f + print >>f, "members:" + objects = self.objects.keys() + objects.sort() + for name in objects: + print >>f, name, self.objects[name] + + print >>f + print >>f, "class relationships:" + classes = self.classes.keys() + classes.sort() + for class_ in classes: + bases = self.classes[class_] + if bases: + print >>f, class_, ", ".join(map(str, bases)) + else: + print >>f, class_ + + self.to_lines(f, "instance attributes:", self.instance_attrs) + + print >>f + print >>f, "instance attribute constants:" + classes = self.instance_attr_constants.items() + classes.sort() + for name, attrs in classes: + attrs = attrs.items() + attrs.sort() + for attrname, ref in attrs: + print >>f, name, attrname, ref + + self.to_lines(f, "names used:", self.names_used) + self.to_lines(f, "names missing:", self.names_missing) + + print >>f + print >>f, "name references:" + refs = self.name_references.items() + refs.sort() + for name, ref in refs: + print >>f, name, ref + + print >>f + print >>f, "initialised names:" + assignments = self.initialised_names.items() + assignments.sort() + for name, refs in assignments: + versions = refs.items() + versions.sort() + for version, ref in versions: + print >>f, name, version, ref + + print >>f + print >>f, "aliased names:" + assignments = self.aliased_names.items() + assignments.sort() + for name, aliases in assignments: + versions = aliases.items() + versions.sort() + for version, alias in versions: + original_name, attrnames, number = alias + print >>f, name, version, original_name, attrnames or "{}", number is None and "{}" or number + + print >>f + print >>f, "function parameters:" + functions = self.function_parameters.keys() + functions.sort() + for function in functions: + print >>f, function, ", ".join(self.function_parameters[function]) + + print >>f + print >>f, "function default parameters:" + functions = self.function_defaults.keys() + functions.sort() + for function in functions: + parameters = self.function_defaults[function] + if parameters: + print >>f, function, ", ".join([("%s=%s" % (name, default)) for (name, default) in parameters]) + else: + print >>f, function, "{}" + + print >>f + print >>f, "function locals:" + functions = self.function_locals.keys() + functions.sort() + for function in functions: + names = self.function_locals[function].items() + if names: + names.sort() + for name, value in names: + print >>f, function, name, value + else: + print >>f, function, "{}" + + self.to_lines(f, "scope globals:", self.scope_globals) + + print >>f + print >>f, "function targets:" + functions = self.function_targets.keys() + functions.sort() + for function in functions: + print >>f, function, self.function_targets[function] + + print >>f + print >>f, "function arguments:" + functions = self.function_arguments.keys() + functions.sort() + for function in functions: + print >>f, function, self.function_arguments[function] + + print >>f + print >>f, "attribute usage:" + units = self.attr_usage.keys() + units.sort() + for unit in units: + d = self.attr_usage[unit] + self.usage_to_cache(d, f, unit) + + print >>f + print >>f, "attribute accesses:" + paths = self.attr_accesses.keys() + paths.sort() + for path in paths: + accesses = list(self.attr_accesses[path]) + accesses.sort() + print >>f, path, ", ".join(accesses) + + print >>f + print >>f, "constant accesses:" + paths = self.const_accesses.keys() + paths.sort() + for path in paths: + accesses = self.const_accesses[path].items() + accesses.sort() + for (original_name, attrnames), (objpath, ref, remaining_attrnames) in accesses: + print >>f, path, original_name, attrnames, objpath, ref, remaining_attrnames or "{}" + + print >>f + print >>f, "attribute access usage:" + paths = self.attr_accessors.keys() + paths.sort() + for path in paths: + all_accesses = self.attr_accessors[path].items() + all_accesses.sort() + for (name, attrname), accesses in all_accesses: + for positions in accesses: + positions = map(str, positions) + print >>f, path, name, attrname or "{}", ", ".join(positions) + + print >>f + print >>f, "attribute access modifiers:" + paths = self.attr_access_modifiers.keys() + paths.sort() + for path in paths: + all_accesses = self.attr_access_modifiers[path].items() + all_accesses.sort() + for (name, attrnames), modifiers in all_accesses: + print >>f, path, name or "{}", attrnames or "{}", encode_modifiers(modifiers) + + print >>f + print >>f, "constant literals:" + paths = self.constants.keys() + paths.sort() + for path in paths: + constants = [(v, k) for (k, v) in self.constants[path].items()] + constants.sort() + for n, constant in constants: + print >>f, path, repr(constant) + + print >>f + print >>f, "constant values:" + names = self.constant_values.keys() + names.sort() + for name in names: + value, value_type = self.constant_values[name] + print >>f, name, value_type, repr(value) + + finally: + f.close() + + def to_lines(self, f, heading, d): + + "Write lines to 'f' with the given 'heading', using 'd'." + + print >>f + print >>f, heading + keys = d.keys() + keys.sort() + for key in keys: + attrs = list(d[key]) + if attrs: + attrs.sort() + print >>f, key, ", ".join(attrs) + + def usage_to_cache(self, details, f, prefix): + + "Write the given namespace usage details to the cache." + + names = list(details.keys()) + if names: + names.sort() + for name in names: + if details[name]: + + # Produce descriptions for each version of the name. + + for version in details[name]: + all_usages = [] + for usage in version: + all_usages.append(encode_usage(usage)) + + print >>f, "%s %s %s" % (prefix, name, "; ".join(all_usages)) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 000000000000 -r 5366b587e3fa referencing.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/referencing.py Tue Aug 30 16:51:10 2016 +0200 @@ -0,0 +1,175 @@ +#!/usr/bin/env python + +""" +Reference abstractions. + +Copyright (C) 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +class Reference: + + "A reference abstraction." + + def __init__(self, kind, origin=None, name=None): + + """ + Initialise a reference using 'kind' to indicate the kind of object, + 'origin' to indicate the actual origin of a referenced object, and a + 'name' indicating an alias for the object in the program structure. + """ + + if isinstance(kind, Reference): + raise ValueError, (kind, origin) + self.kind = kind + self.origin = origin + self.name = name + + def __repr__(self): + return "Reference(%r, %r, %r)" % (self.kind, self.origin, self.name) + + def __str__(self): + + """ + Serialise the reference as '' or a description incorporating the + kind and origin. + """ + + if self.kind == "": + return self.kind + else: + return "%s:%s" % (self.kind, self.origin) + + def __hash__(self): + + "Hash instances using the kind and origin only." + + return hash((self.kind, self.get_origin())) + + def __cmp__(self, other): + + "Compare with 'other' using the kind and origin only." + + if isinstance(other, Reference): + return cmp((self.kind, self.get_origin()), (other.kind, other.get_origin())) + else: + return cmp(str(self), other) + + def get_name(self): + + "Return the name used for this reference." + + return self.name + + def get_origin(self): + + "Return the origin of the reference." + + return self.kind != "" and self.origin or None + + def get_kind(self): + + "Return the kind of object referenced." + + return self.kind + + def has_kind(self, kinds): + + """ + Return whether the reference describes an object from the given 'kinds', + where such kinds may be "", "", "", + "" or "". + """ + + if not isinstance(kinds, (list, tuple)): + kinds = [kinds] + return self.get_kind() in kinds + + def get_path(self): + + "Return the attribute names comprising the path to the origin." + + return self.get_origin().split(".") + + def static(self): + + "Return this reference if it refers to a static object, None otherwise." + + return not self.has_kind(["", ""]) and self or None + + def final(self): + + "Return a reference to either a static object or None." + + static = self.static() + return static and static.origin or None + + def instance_of(self): + + "Return a reference to an instance of the referenced class." + + return self.has_kind("") and Reference("", self.origin) or None + + def as_var(self): + + """ + Return a variable version of this reference. Any origin information is + discarded since variable references are deliberately ambiguous. + """ + + return Reference("", None, self.name) + + def provided_by_module(self, module_name): + + "Return whether the reference is provided by 'module_name'." + + path = self.origin + return not path or path.rsplit(".", 1)[0] == module_name + + def alias(self, name): + + "Alias this reference employing 'name'." + + return Reference(self.get_kind(), self.get_origin(), name) + +def decode_reference(s, name=None): + + "Decode 's', making a reference." + + if isinstance(s, Reference): + return s.alias(name) + + # Null value. + + elif not s: + return Reference("", None, name) + + # Kind and origin. + + elif ":" in s: + kind, origin = s.split(":") + return Reference(kind, origin, name) + + # Kind-only, origin is indicated name. + + elif s[0] == "<": + return Reference(s, name, name) + + # Module-only. + + else: + return Reference("", s, name) + +# vim: tabstop=4 expandtab shiftwidth=4