Lichen

inspector.py

938:799711337453
2021-06-29 Paul Boddie Renamed the string class to str, replacing the str function with the new_str function, this being invoked specially by the string instantiation function. As with the Unicode type renaming, a more general instantiation mechanism might permit the new_str function to be part of the functionality of the str or basestring classes.
     1 #!/usr/bin/env python     2      3 """     4 Inspect and obtain module structure.     5      6 Copyright (C) 2007-2019, 2021 Paul Boddie <paul@boddie.org.uk>     7      8 This program is free software; you can redistribute it and/or modify it under     9 the terms of the GNU General Public License as published by the Free Software    10 Foundation; either version 3 of the License, or (at your option) any later    11 version.    12     13 This program is distributed in the hope that it will be useful, but WITHOUT    14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    15 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    16 details.    17     18 You should have received a copy of the GNU General Public License along with    19 this program.  If not, see <http://www.gnu.org/licenses/>.    20 """    21     22 from branching import BranchTracker    23 from common import CommonModule, get_argnames, init_item, \    24                    predefined_constants, privileged_attributes    25 from modules import BasicModule, CacheWritingModule, InspectionNaming    26 from errors import InspectError    27 from referencing import Reference    28 from resolving import NameResolving    29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \    30                     LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \    31                     Result, VariableRef    32 import compiler    33 import sys    34     35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming):    36     37     "A module inspector."    38     39     def __init__(self, name, importer):    40     41         "Initialise the module with basic details."    42     43         BasicModule.__init__(self, name, importer)    44     45         self.in_class = False    46         self.in_conditional = False    47     48         # Accesses to global attributes.    49     50         self.global_attr_accesses = {}    51     52         # Usage tracking.    53     54         self.trackers = []    55         self.attr_accessor_branches = {}    56     57     def __repr__(self):    58         return "InspectedModule(%r, %r)" % (self.name, self.importer)    59     60     # Principal methods.    61     62     def parse(self, filename):    63     64         "Parse the file having the given 'filename'."    65     66         self.parse_file(filename)    67     68         # Inspect the module.    69     70         self.start_tracking_in_module()    71     72         # Detect and record imports and globals declared in the module.    73     74         self.process_structure(self.astnode)    75     76         # Set the class of the module after the definition has occurred.    77     78         ref = self.get_builtin("module")    79         self.set_name("__class__", ref)    80         self.set_name("__name__", self.get_constant("str", self.name).reference())    81         self.set_name("__file__", self.get_constant("str", filename).reference())    82     83         # Reserve a constant for the encoding.    84     85         if self.encoding:    86             self.get_constant("str", self.encoding)    87     88         # Get module-level attribute usage details.    89     90         self.stop_tracking_in_module()    91     92         # Collect external name references.    93     94         self.collect_names()    95     96     def complete(self):    97     98         "Complete the module inspection."    99    100         # Resolve names not definitively mapped to objects.   101    102         self.resolve()   103    104         # Propagate to the importer information needed in subsequent activities.   105    106         self.propagate()   107    108     # Accessory methods.   109    110     def collect_names(self):   111    112         "Collect the names used by each scope."   113    114         for path in self.names_used.keys():   115             self.collect_names_for_path(path)   116    117     def collect_names_for_path(self, path):   118    119         """   120         Collect the names used by the given 'path'. These are propagated to the   121         importer in advance of any dependency resolution.   122         """   123    124         names = self.names_used.get(path)   125         if not names:   126             return   127    128         in_function = self.function_locals.has_key(path)   129    130         for name in names:   131             if in_function and name in self.function_locals[path]:   132                 continue   133    134             key = "%s.%s" % (path, name)   135    136             # Find local definitions.   137    138             ref = self.get_resolved_object(key, True)   139             if ref:   140                 self.set_name_reference(key, ref)   141                 continue   142    143             # Find global.   144    145             ref = self.get_global(name)   146             if ref:   147                 self.set_name_reference(key, ref)   148                 continue   149    150             # Find presumed built-in definitions.   151    152             ref = self.get_builtin(name)   153             self.set_name_reference(key, ref)   154    155     def set_name_reference(self, path, ref):   156    157         "Map the given name 'path' to 'ref'."   158    159         self.importer.all_name_references[path] = self.name_references[path] = ref   160    161     # Module structure traversal.   162    163     def process_structure_node(self, n):   164    165         "Process the individual node 'n'."   166    167         path = self.get_namespace_path()   168    169         # Module global detection.   170    171         if isinstance(n, compiler.ast.Global):   172             self.process_global_node(n)   173    174         # Module import declarations.   175    176         elif isinstance(n, compiler.ast.From):   177             self.process_from_node(n)   178    179         elif isinstance(n, compiler.ast.Import):   180             self.process_import_node(n)   181    182         # Nodes using operator module functions.   183    184         elif isinstance(n, compiler.ast.Operator):   185             return self.process_operator_node(n)   186    187         elif isinstance(n, compiler.ast.AugAssign):   188             self.process_augassign_node(n)   189    190         elif isinstance(n, compiler.ast.Compare):   191             return self.process_compare_node(n)   192    193         elif isinstance(n, compiler.ast.Slice):   194             return self.process_slice_node(n)   195    196         elif isinstance(n, compiler.ast.Sliceobj):   197             return self.process_sliceobj_node(n)   198    199         elif isinstance(n, compiler.ast.Subscript):   200             return self.process_subscript_node(n)   201    202         # Namespaces within modules.   203    204         elif isinstance(n, compiler.ast.Class):   205             self.process_class_node(n)   206    207         elif isinstance(n, compiler.ast.Function):   208             self.process_function_node(n, n.name)   209    210         elif isinstance(n, compiler.ast.Lambda):   211             return self.process_lambda_node(n)   212    213         # Assignments.   214    215         elif isinstance(n, compiler.ast.Assign):   216    217             # Handle each assignment node.   218    219             for node in n.nodes:   220                 self.process_assignment_node(node, n.expr)   221    222         # Assignments within non-Assign nodes.   223    224         elif isinstance(n, compiler.ast.AssName):   225             raise InspectError("Name assignment appearing outside assignment statement.", path, n)   226    227         elif isinstance(n, compiler.ast.AssAttr):   228             raise InspectError("Attribute assignment appearing outside assignment statement.", path, n)   229    230         # Accesses.   231    232         elif isinstance(n, compiler.ast.Getattr):   233             return self.process_attribute_access(n)   234    235         # Name recording for later testing.   236    237         elif isinstance(n, compiler.ast.Name):   238             return self.process_name_node(n)   239    240         # Conditional statement tracking.   241    242         elif isinstance(n, compiler.ast.For):   243             self.process_for_node(n)   244    245         elif isinstance(n, compiler.ast.While):   246             self.process_while_node(n)   247    248         elif isinstance(n, compiler.ast.If):   249             self.process_if_node(n)   250    251         elif isinstance(n, (compiler.ast.And, compiler.ast.Or)):   252             return self.process_logical_node(n)   253    254         # Exception control-flow tracking.   255    256         elif isinstance(n, compiler.ast.TryExcept):   257             self.process_try_node(n)   258    259         elif isinstance(n, compiler.ast.TryFinally):   260             self.process_try_finally_node(n)   261    262         # Control-flow modification statements.   263    264         elif isinstance(n, compiler.ast.Break):   265             self.trackers[-1].suspend_broken_branch()   266    267         elif isinstance(n, compiler.ast.Continue):   268             self.trackers[-1].suspend_continuing_branch()   269    270         elif isinstance(n, compiler.ast.Raise):   271             self.process_structure(n)   272             self.trackers[-1].abandon_branch()   273    274         elif isinstance(n, compiler.ast.Return):   275             self.record_return_value(self.process_structure_node(n.value))   276             self.trackers[-1].abandon_returning_branch()   277    278         # Print statements.   279    280         elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)):   281             self.process_print_node(n)   282    283         # Invocations.   284    285         elif isinstance(n, compiler.ast.CallFunc):   286             return self.process_invocation_node(n)   287    288         # Constant usage.   289    290         elif isinstance(n, compiler.ast.Const):   291             return self.get_literal_instance(n)   292    293         elif isinstance(n, compiler.ast.Dict):   294             return self.get_literal_instance(n, "dict")   295    296         elif isinstance(n, compiler.ast.List):   297             return self.get_literal_instance(n, "list")   298    299         elif isinstance(n, compiler.ast.Tuple):   300             return self.get_literal_instance(n, "tuple")   301    302         # All other nodes are processed depth-first.   303    304         else:   305             self.process_structure(n)   306    307         # By default, no expression details are returned.   308    309         return None   310    311     # Specific node handling.   312    313     def process_assignment_node(self, n, expr):   314    315         "Process the individual node 'n' to be assigned the contents of 'expr'."   316    317         # Names and attributes are assigned the entire expression.   318    319         if isinstance(n, compiler.ast.AssName):   320             if n.name == "self":   321                 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n)   322    323             name_ref = expr and self.process_structure_node(expr)   324    325             # Name assignments populate either function namespaces or the   326             # general namespace hierarchy.   327    328             self.assign_general_local(n.name, name_ref)   329    330             # Record usage of the name.   331    332             self.record_name(n.name)   333    334         elif isinstance(n, compiler.ast.AssAttr):   335             if expr:   336                 expr = self.process_structure_node(expr)   337    338             in_assignment = self.in_assignment   339             self.in_assignment = True   340             self.process_attribute_access(n)   341             self.in_assignment = in_assignment   342    343         # Lists and tuples are matched against the expression and their   344         # items assigned to expression items.   345    346         elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)):   347             self.process_assignment_node_items(n, expr)   348    349         # Slices and subscripts are permitted within assignment nodes.   350    351         elif isinstance(n, compiler.ast.Slice):   352             self.process_slice_node(n, expr)   353    354         elif isinstance(n, compiler.ast.Subscript):   355             self.process_subscript_node(n, expr)   356    357     def process_attribute_access(self, n):   358    359         "Process the given attribute access node 'n'."   360    361         path = self.get_namespace_path()   362    363         # Test for access to special privileged attributes.   364    365         if isinstance(n, compiler.ast.Getattr) and \   366            n.attrname in privileged_attributes and not n.privileged:   367    368             raise InspectError("Attribute %s is accessed by an unprivileged operation." %   369                                n.attrname, path, n)   370    371         # Obtain any completed chain and return the reference to it.   372    373         name_ref = self.process_attribute_chain(n)   374    375         if self.have_access_expression(n):   376             return name_ref   377    378         # Where the start of the chain of attributes has been reached, determine   379         # the complete access.   380    381         # Given a non-access node, this chain can be handled in its entirety,   382         # either being name-based and thus an access rooted on a name, or being   383         # based on some other node and thus an anonymous access of some kind.   384    385         # Start with the the full attribute chain.   386    387         remaining = self.attrs   388         attrnames = ".".join(remaining)   389    390         # If the accessor cannot be identified, or where attributes   391         # remain in an attribute chain, record the anonymous accesses.   392    393         if not isinstance(name_ref, NameRef): # includes ResolvedNameRef   394    395             init_item(self.attr_accesses, path, set)   396             self.attr_accesses[path].add(attrnames)   397    398             self.record_access_details(None, attrnames, self.in_assignment,   399                 self.in_invocation)   400             del self.attrs[0]   401             return   402    403         # Name-based accesses will handle the first attribute in a   404         # chain.   405    406         else:   407             attrname = remaining[0]   408    409             # Attribute assignments are used to identify instance attributes.   410    411             if isinstance(n, compiler.ast.AssAttr) and \   412                 self.in_class and self.in_function and n.expr.name == "self":   413    414                 self.set_instance_attr(attrname)   415    416             # Record attribute usage using any name local to this namespace,   417             # if assigned in the namespace, or using an external name   418             # (presently just globals within classes).   419    420             name = self.get_name_for_tracking(name_ref.name, name_ref)   421             tracker = self.trackers[-1]   422    423             immediate_access = len(self.attrs) == 1   424             assignment = immediate_access and isinstance(n, compiler.ast.AssAttr)   425    426             # Record global-based chains for subsequent resolution.   427    428             if name_ref.is_global_name():   429                 self.record_global_access_details(name, attrnames)   430    431             # Make sure the name is being tracked: global names will not   432             # already be initialised in a branch and must be added   433             # explicitly.   434    435             if not tracker.have_name(name):   436                 tracker.assign_names([name])   437                 if self.in_function:   438                     self.scope_globals[path].add(name)   439    440             # Record attribute usage in the tracker, and record the branch   441             # information for the access.   442    443             branches = tracker.use_attribute(name, attrname,   444                 self.in_invocation is not None, assignment)   445    446             if not branches:   447                 raise InspectError("Name %s is accessed using %s before an assignment." % (   448                     name, attrname), path, n)   449    450             self.record_branches_for_access(branches, name, attrnames)   451             access_number = self.record_access_details(name, attrnames,   452                 self.in_assignment, self.in_invocation)   453    454             del self.attrs[0]   455             return AccessRef(name, attrnames, access_number)   456    457     def process_class_node(self, n):   458    459         "Process the given class node 'n'."   460    461         path = self.get_namespace_path()   462    463         # To avoid notions of class "versions" where the same definition   464         # might be parameterised with different state and be referenced   465         # elsewhere (as base classes, for example), classes in functions or   466         # conditions are forbidden.   467    468         if self.in_function or self.in_conditional:   469             print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % (   470                 path, n.name)   471             return   472    473         # Resolve base classes.   474    475         bases = []   476    477         for base in n.bases:   478             base_class = self.get_class(base)   479    480             if not base_class:   481                 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % (   482                     path, n.name, base)   483                 return   484             else:   485                 bases.append(base_class)   486    487         # Detect conflicting definitions. Such definitions cause conflicts in   488         # the storage of namespace-related information.   489    490         class_name = self.get_object_path(n.name)   491         ref = self.get_object(class_name, defer=False)   492    493         if ref and ref.static():   494             raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n)   495    496         # Record bases for the class and retain the class name.   497         # Note that the function class does not inherit from the object class.   498    499         if not bases and class_name != "__builtins__.core.object" and \   500                          class_name != "__builtins__.core.function":   501    502             ref = self.get_object("__builtins__.object")   503             bases.append(ref)   504    505         self.importer.classes[class_name] = self.classes[class_name] = bases   506         self.importer.subclasses[class_name] = set()   507         self.scope_globals[class_name] = set()   508    509         # Set the definition before entering the namespace rather than   510         # afterwards because methods may reference it. In normal Python,   511         # a class is not accessible until the definition is complete, but   512         # methods can generally reference it since upon being called the   513         # class will already exist.   514    515         self.set_definition(n.name, "<class>")   516    517         in_class = self.in_class   518         self.in_class = class_name   519         self.set_instance_attr("__class__", Reference("<class>", class_name))   520         self.enter_namespace(n.name)   521    522         # Do not provide the special instantiator attributes on the function   523         # class. Function instances provide these attributes.   524    525         if class_name != "__builtins__.core.function":   526    527             self.set_name("__fn__") # special instantiator attribute   528             self.set_name("__args__") # special instantiator attribute   529    530         # Provide leafname, parent and context attributes.   531    532         parent, leafname = class_name.rsplit(".", 1)   533         self.set_name("__name__", self.get_constant("str", leafname).reference())   534    535         if class_name != "__builtins__.core.function":   536             self.set_name("__parent__")   537    538         self.process_structure_node(n.code)   539         self.exit_namespace()   540         self.in_class = in_class   541    542     def process_from_node(self, n):   543    544         "Process the given node 'n', importing from another module."   545    546         path = self.get_namespace_path()   547    548         module_name, names = self.get_module_name(n)   549         if module_name == self.name:   550             raise InspectError("Cannot import from the current module.", path, n)   551    552         self.queue_module(module_name)   553    554         # Attempt to obtain the referenced objects.   555    556         for name, alias in n.names:   557             if name == "*":   558                 raise InspectError("Only explicitly specified names can be imported from modules.", path, n)   559    560             # Explicit names.   561    562             ref = self.import_name_from_module(name, module_name)   563             value = ResolvedNameRef(alias or name, ref)   564             self.set_general_local(alias or name, value)   565    566     def process_function_node(self, n, name):   567    568         """   569         Process the given function or lambda node 'n' with the given 'name'.   570         """   571    572         is_lambda = isinstance(n, compiler.ast.Lambda)   573    574         # Where a function is declared conditionally, use a separate name for   575         # the definition, and assign the definition to the stated name.   576    577         if (self.in_conditional or self.in_function) and not is_lambda:   578             original_name = name   579             name = self.get_lambda_name()   580         else:   581             original_name = None   582    583         # Detect conflicting definitions. Such definitions cause conflicts in   584         # the storage of namespace-related information.   585    586         function_name = self.get_object_path(name)   587         ref = self.get_object(function_name, defer=False)   588    589         if ref and ref.static():   590             raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n)   591    592         # Initialise argument and local records.   593    594         argnames = get_argnames(n.argnames)   595         is_method = self.in_class and not self.in_function   596    597         # Remove explicit "self" from method parameters.   598    599         if is_method and argnames and argnames[0] == "self":   600             del argnames[0]   601    602         # Convert .name entries in the parameters, provided this is a method.   603    604         l = []   605         attr_initialisers = []   606    607         for argname in argnames:   608             if argname[0] == ".":   609                 if not is_method:   610                     raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n)   611    612                 argname = argname[1:]   613                 attr_initialisers.append(argname)   614    615             if argname in l:   616                 raise InspectError("Argument name %s is repeated." % argname, function_name, n)   617    618             l.append(argname)   619    620         argnames = l   621    622         # Copy and propagate the parameters.   623    624         self.importer.function_parameters[function_name] = \   625             self.function_parameters[function_name] = argnames[:]   626    627         self.importer.function_attr_initialisers[function_name] = \   628             self.function_attr_initialisers[function_name] = attr_initialisers   629    630         # Define all arguments/parameters in the local namespace.   631    632         locals = \   633             self.importer.function_locals[function_name] = \   634             self.function_locals[function_name] = {}   635    636         # Insert "self" into method locals.   637    638         if is_method:   639             argnames.insert(0, "self")   640    641         # Define "self" in terms of the class if in a method.   642         # This does not diminish the need for type-narrowing in the deducer.   643    644         if argnames:   645             if self.in_class and not self.in_function and argnames[0] == "self":   646                 locals[argnames[0]] = Reference("<instance>", self.in_class)   647             else:   648                 locals[argnames[0]] = Reference("<var>")   649    650         for argname in argnames[1:]:   651             locals[argname] = Reference("<var>")   652    653         globals = self.scope_globals[function_name] = set()   654    655         # Process the defaults.   656    657         defaults = self.importer.function_defaults[function_name] = \   658                    self.function_defaults[function_name] = []   659    660         for argname, default in compiler.ast.get_defaults(n):   661             if argname[0] == ".":   662                 argname = argname[1:]   663    664             if default:   665    666                 # Obtain any reference for the default.   667    668                 name_ref = self.process_structure_node(default)   669                 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>")))   670    671         # Reset conditional tracking to focus on the function contents.   672    673         in_conditional = self.in_conditional   674         self.in_conditional = False   675    676         in_function = self.in_function   677         self.in_function = function_name   678    679         self.enter_namespace(name)   680    681         # Define a leafname attribute value for the function instance.   682    683         ref = self.get_builtin_class("str")   684         self.reserve_constant(function_name, name, ref.get_origin())   685    686         # Track attribute usage within the namespace.   687    688         path = self.get_namespace_path()   689         self.start_tracking(locals)   690    691         # Establish attributes for .name entries, provided this is a method.   692    693         for argname in attr_initialisers:   694             self.process_assignment_node(   695                     compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"),   696                     compiler.ast.Name(argname))   697    698         self.process_structure_node(n.code)   699         returns_value = self.stop_tracking()   700    701         # Record any null result.   702    703         is_initialiser = is_method and name == "__init__"   704    705         if not returns_value and not is_initialiser:   706             self.record_return_value(ResolvedNameRef("None", self.get_builtin("None")))   707    708         # Exit to the parent.   709    710         self.exit_namespace()   711    712         # Update flags.   713    714         self.in_function = in_function   715         self.in_conditional = in_conditional   716    717         # Define the function using the appropriate name.   718    719         self.set_definition(name, "<function>")   720    721         # Where a function is set conditionally, assign the name.   722    723         if original_name:   724             self.process_assignment_for_object(original_name, compiler.ast.Name(name))   725    726     def process_global_node(self, n):   727    728         """   729         Process the given "global" node 'n'.   730         """   731    732         path = self.get_namespace_path()   733    734         if path != self.name:   735             self.scope_globals[path].update(n.names)   736    737     def process_if_node(self, n):   738    739         """   740         Process the given "if" node 'n'.   741         """   742    743         tracker = self.trackers[-1]   744         tracker.new_branchpoint()   745    746         for test, body in n.tests:   747             self.process_structure_node(test)   748    749             tracker.new_branch()   750    751             in_conditional = self.in_conditional   752             self.in_conditional = True   753             self.process_structure_node(body)   754             self.in_conditional = in_conditional   755    756             tracker.shelve_branch()   757    758         # Maintain a branch for the else clause.   759    760         tracker.new_branch()   761         if n.else_:   762             self.process_structure_node(n.else_)   763         tracker.shelve_branch()   764    765         tracker.merge_branches()   766    767     def process_import_node(self, n):   768    769         "Process the given import node 'n'."   770    771         path = self.get_namespace_path()   772    773         # Load the mentioned module.   774    775         for name, alias in n.names:   776             if name == self.name:   777                 raise InspectError("Cannot import the current module.", path, n)   778    779             self.set_module(alias or name.split(".")[-1], name)   780             self.queue_module(name, True)   781    782     def process_invocation_node(self, n):   783    784         "Process the given invocation node 'n'."   785    786         path = self.get_namespace_path()   787    788         in_invocation = self.in_invocation   789         self.in_invocation = None   790    791         # Process the arguments.   792    793         keywords = set()   794    795         for arg in n.args:   796             self.process_structure_node(arg)   797             if isinstance(arg, compiler.ast.Keyword):   798                 keywords.add(arg.name)   799    800         keywords = list(keywords)   801         keywords.sort()   802    803         # Communicate to the invocation target expression that it forms the   804         # target of an invocation, potentially affecting attribute accesses.   805    806         self.in_invocation = len(n.args), keywords   807    808         # Process the expression, obtaining any identified reference.   809    810         name_ref = self.process_structure_node(n.node)   811         self.in_invocation = in_invocation   812    813         # Detect class invocations.   814    815         if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"):   816             return InstanceRef(name_ref.reference().instance_of())   817    818         elif isinstance(name_ref, (NameRef, AccessRef)):   819             return InvocationRef(name_ref)   820    821         # Provide a general reference to indicate that something is produced   822         # by the invocation, useful for retaining assignment expression   823         # details.   824    825         return VariableRef()   826    827     def process_lambda_node(self, n):   828    829         "Process the given lambda node 'n'."   830    831         name = self.get_lambda_name()   832         self.process_function_node(n, name)   833    834         origin = self.get_object_path(name)   835    836         if self.function_defaults.get(origin):   837             return None   838         else:   839             return ResolvedNameRef(name, Reference("<function>", origin))   840    841     def process_logical_node(self, n):   842    843         "Process the given operator node 'n'."   844    845         return self.process_operator_chain(n.nodes)   846    847     def process_name_node(self, n):   848    849         "Process the given name node 'n'."   850    851         path = self.get_namespace_path()   852    853         # Find predefined constant names before anything else.   854    855         if n.name in predefined_constants:   856             ref = self.get_builtin(n.name)   857             value = ResolvedNameRef(n.name, ref)   858             return value   859    860         # Special names that have already been identified.   861    862         if n.name.startswith("$"):   863             value = self.get_special(n.name)   864             if value:   865                 return value   866    867         # Special case for operator functions introduced through code   868         # transformations.   869    870         if n.name.startswith("$op"):   871    872             # Obtain the location of the actual function defined in the operator   873             # package.   874    875             op = n.name[len("$op"):]   876    877             # Attempt to get a reference.   878    879             ref = self.import_name_from_module(op, "operator")   880    881             # Record the imported name and provide the resolved name reference.   882    883             value = ResolvedNameRef(n.name, ref)   884             self.set_special(n.name, value)   885             return value   886    887         # Special case for sequence length testing.   888    889         elif n.name.startswith("$seq"):   890             op = n.name[len("$seq"):]   891             ref = self.import_name_from_module(op, "__builtins__.sequence")   892             value = ResolvedNameRef(n.name, ref)   893             self.set_special(n.name, value)   894             return value   895    896         # Special case for print operations.   897    898         elif n.name.startswith("$print"):   899    900             # Attempt to get a reference.   901    902             ref = self.get_builtin("print_")   903    904             # Record the imported name and provide the resolved name reference.   905    906             value = ResolvedNameRef(n.name, ref)   907             self.set_special(n.name, value)   908             return value   909    910         # Test for self usage, which is only allowed in methods.   911    912         if n.name == "self" and not (self.in_function and self.in_class):   913             raise InspectError("Use of self is only allowed in methods.", path, n)   914    915         # Record usage of the name.   916    917         self.record_name(n.name)   918    919         # Search for unknown names in non-function scopes immediately.   920         # Temporary names should not be re-used between scopes.   921         # External names in functions are resolved later.   922    923         ref = not n.name.startswith("$t") and self.find_name(n.name) or None   924    925         if ref:   926             self.record_name_access(n.name, True)   927             return ResolvedNameRef(n.name, ref, is_global=True)   928    929         # Explicitly-declared global names.   930    931         elif self.in_function and n.name in self.scope_globals[path]:   932             self.record_name_access(n.name, True)   933             return NameRef(n.name, is_global=True)   934    935         # Examine other names.   936    937         else:   938    939             # Check local names.   940    941             access_number = self.record_name_access(n.name)   942    943             # Local name.   944    945             if access_number is not None:   946                 return LocalNameRef(n.name, access_number)   947    948             # Possible global or built-in name.   949    950             else:   951                 self.record_name_access(n.name, True)   952                 return NameRef(n.name, is_global=True)   953    954     def record_name_access(self, name, is_global=False):   955    956         """   957         Record an access involving 'name' if the name is being tracked, using   958         'is_global' to indicate whether the name is global.   959         """   960    961         name = self.get_name_for_tracking(name, is_global=is_global)   962         branches = self.trackers[-1].tracking_name(name)   963         if branches:   964             self.record_branches_for_access(branches, name, None)   965             return self.record_access_details(name, None, self.in_assignment,   966                                               self.in_invocation)   967         return None   968    969     def process_operator_chain(self, nodes):   970    971         """   972         Process the given chain of 'nodes', processing each node or item.   973         Each node starts a new conditional region, effectively making a deeply-   974         nested collection of if-like statements.   975         """   976    977         results = []   978    979         tracker = self.trackers[-1]   980    981         for item in nodes:   982             tracker.new_branchpoint()   983             tracker.new_branch()   984             result = self.process_structure_node(item)   985             if result:   986                 results.append(result)   987    988         for item in nodes[:-1]:   989             tracker.shelve_branch()   990             tracker.new_branch()   991             tracker.shelve_branch()   992             tracker.merge_branches()   993    994         tracker.shelve_branch()   995         tracker.merge_branches()   996    997         return MultipleRef(results)   998    999     def process_try_node(self, n):  1000   1001         """  1002         Process the given "try...except" node 'n'.  1003         """  1004   1005         self.record_exception_handler()  1006   1007         tracker = self.trackers[-1]  1008         tracker.new_branchpoint()  1009   1010         self.process_structure_node(n.body)  1011   1012         for name, var, handler in n.handlers:  1013             if name is not None:  1014                 self.process_structure_node(name)  1015   1016             # Any abandoned branches from the body can now be resumed in a new  1017             # branch.  1018   1019             tracker.resume_abandoned_branches()  1020   1021             # Establish the local for the handler.  1022   1023             if var is not None:  1024                 self.process_assignment_node(var, None)  1025             if handler is not None:  1026                 self.process_structure_node(handler)  1027   1028             tracker.shelve_branch()  1029   1030         # The else clause maintains the usage from the body but without the  1031         # abandoned branches since they would never lead to the else clause  1032         # being executed.  1033   1034         if n.else_:  1035             tracker.new_branch()  1036             self.process_structure_node(n.else_)  1037             tracker.shelve_branch()  1038   1039         # Without an else clause, a null branch propagates the successful  1040         # outcome.  1041   1042         else:  1043             tracker.new_branch()  1044             tracker.shelve_branch()  1045   1046         tracker.merge_branches()  1047   1048     def process_try_finally_node(self, n):  1049   1050         """  1051         Process the given "try...finally" node 'n'.  1052         """  1053   1054         self.record_exception_handler()  1055   1056         tracker = self.trackers[-1]  1057         self.process_structure_node(n.body)  1058   1059         # Any abandoned branches from the body can now be resumed.  1060   1061         branches = tracker.resume_all_abandoned_branches()  1062         self.process_structure_node(n.final)  1063   1064         # At the end of the finally clause, abandoned branches are discarded.  1065   1066         tracker.restore_active_branches(branches)  1067   1068     def process_while_node(self, n):  1069   1070         "Process the given while node 'n'."  1071   1072         tracker = self.trackers[-1]  1073         tracker.new_branchpoint(loop_node=True)  1074   1075         # Evaluate any test or iterator outside the loop.  1076   1077         self.process_structure_node(n.test)  1078   1079         # Propagate attribute usage to branches.  1080   1081         tracker.new_branch(loop_node=True)  1082   1083         # Enter the loop.  1084   1085         in_conditional = self.in_conditional  1086         self.in_conditional = True  1087         self.process_structure_node(n.body)  1088         self.in_conditional = in_conditional  1089   1090         # Continuing branches are resumed before any test.  1091   1092         tracker.resume_continuing_branches()  1093   1094         # Evaluate any continuation test within the body.  1095   1096         self.process_structure_node(n.test)  1097   1098         tracker.shelve_branch(loop_node=True)  1099   1100         # Support the non-looping condition.  1101   1102         tracker.new_branch()  1103         tracker.shelve_branch()  1104   1105         tracker.merge_branches()  1106   1107         # Evaluate any else clause outside branches.  1108   1109         if n.else_:  1110             self.process_structure_node(n.else_)  1111   1112         # Connect broken branches to the code after any loop.  1113   1114         tracker.resume_broken_branches()  1115   1116     # Branch tracking methods.  1117   1118     def start_tracking(self, names):  1119   1120         """  1121         Start tracking attribute usage for names in the current namespace,  1122         immediately registering the given 'names'.  1123         """  1124   1125         path = self.get_namespace_path()  1126         parent = self.trackers[-1]  1127         tracker = BranchTracker()  1128         self.trackers.append(tracker)  1129   1130         # Record the given names established as new branches.  1131   1132         tracker.assign_names(names)  1133   1134     def assign_name(self, name, name_ref):  1135   1136         "Assign to 'name' the given 'name_ref' in the current namespace."  1137   1138         name = self.get_name_for_tracking(name)  1139         self.trackers[-1].assign_names([name], [name_ref])  1140   1141     def stop_tracking(self):  1142   1143         """  1144         Stop tracking attribute usage, recording computed usage for the current  1145         namespace. Indicate whether a value is always returned from the  1146         namespace.  1147         """  1148   1149         path = self.get_namespace_path()  1150         tracker = self.trackers.pop()  1151         self.record_assignments_for_access(tracker)  1152   1153         self.attr_usage[path] = tracker.get_all_usage()  1154         self.name_initialisers[path] = tracker.get_all_values()  1155   1156         return tracker.returns_value()  1157   1158     def start_tracking_in_module(self):  1159   1160         "Start tracking attribute usage in the module."  1161   1162         tracker = BranchTracker()  1163         self.trackers.append(tracker)  1164   1165     def stop_tracking_in_module(self):  1166   1167         "Stop tracking attribute usage in the module."  1168   1169         tracker = self.trackers[0]  1170         self.record_assignments_for_access(tracker)  1171         self.attr_usage[self.name] = tracker.get_all_usage()  1172         self.name_initialisers[self.name] = tracker.get_all_values()  1173   1174     def record_assignments_for_access(self, tracker):  1175   1176         """  1177         For the current path, use the given 'tracker' to record assignment  1178         version information for attribute accesses.  1179         """  1180   1181         path = self.get_path_for_access()  1182   1183         if not self.attr_accessor_branches.has_key(path):  1184             return  1185   1186         init_item(self.attr_accessors, path, dict)  1187         attr_accessors = self.attr_accessors[path]  1188   1189         # Obtain the branches applying during each access.  1190   1191         for access, all_branches in self.attr_accessor_branches[path].items():  1192             name, attrnames = access  1193             init_item(attr_accessors, access, list)  1194   1195             # Obtain the assignments applying to each branch.  1196   1197             for branches in all_branches:  1198                 positions = tracker.get_assignment_positions_for_branches(name, branches)  1199   1200                 # Detect missing name information.  1201   1202                 if None in positions:  1203                     globals = self.global_attr_accesses.get(path)  1204                     accesses = globals and globals.get(name)  1205                     if not accesses:  1206                         print >>sys.stderr, "In %s, %s may not be defined when used." % (  1207                             self.get_namespace_path(), name)  1208                     positions.remove(None)  1209   1210                 attr_accessors[access].append(positions)  1211   1212     def record_branches_for_access(self, branches, name, attrnames):  1213   1214         """  1215         Record the given 'branches' for an access involving the given 'name' and  1216         'attrnames'.  1217         """  1218   1219         access = name, attrnames  1220         path = self.get_path_for_access()  1221   1222         init_item(self.attr_accessor_branches, path, dict)  1223         attr_accessor_branches = self.attr_accessor_branches[path]  1224   1225         init_item(attr_accessor_branches, access, list)  1226         attr_accessor_branches[access].append(branches)  1227   1228     def record_access_details(self, name, attrnames, assignment, invocation):  1229   1230         """  1231         For the given 'name' and 'attrnames', record an access indicating  1232         whether an 'assignment' or an 'invocation' is occurring.  1233   1234         These details correspond to accesses otherwise recorded by the attribute  1235         accessor and attribute access dictionaries.  1236         """  1237   1238         access = name, attrnames  1239         path = self.get_path_for_access()  1240   1241         init_item(self.attr_access_modifiers, path, dict)  1242         init_item(self.attr_access_modifiers[path], access, list)  1243   1244         access_number = len(self.attr_access_modifiers[path][access])  1245         self.attr_access_modifiers[path][access].append((assignment, invocation))  1246         return access_number  1247   1248     def record_global_access_details(self, name, attrnames):  1249   1250         """  1251         Record details of a global access via the given 'name' involving the  1252         indicated 'attrnames'.  1253         """  1254   1255         path = self.get_namespace_path()  1256   1257         init_item(self.global_attr_accesses, path, dict)  1258         init_item(self.global_attr_accesses[path], name, set)  1259         self.global_attr_accesses[path][name].add(attrnames)  1260   1261     # Namespace modification.  1262   1263     def record_name(self, name):  1264   1265         "Record the use of 'name' in a namespace."  1266   1267         path = self.get_namespace_path()  1268         init_item(self.names_used, path, set)  1269         self.names_used[path].add(name)  1270   1271     def set_module(self, name, module_name):  1272   1273         """  1274         Set a module in the current namespace using the given 'name' associated  1275         with the corresponding 'module_name'.  1276         """  1277   1278         if name:  1279             self.set_general_local(name, Reference("<module>", module_name))  1280   1281     def set_definition(self, name, kind):  1282   1283         """  1284         Set the definition having the given 'name' and 'kind'.  1285   1286         Definitions are set in the static namespace hierarchy, but they can also  1287         be recorded for function locals.  1288         """  1289   1290         if self.is_global(name):  1291             print >>sys.stderr, "In %s, %s is defined as being global." % (  1292                 self.get_namespace_path(), name)  1293   1294         path = self.get_object_path(name)  1295         self.set_object(path, kind)  1296   1297         ref = self.get_object(path)  1298         if ref.get_kind() == "<var>":  1299             print >>sys.stderr, "In %s, %s is defined more than once." % (  1300                 self.get_namespace_path(), name)  1301   1302         if not self.is_global(name) and self.in_function:  1303             self.set_function_local(name, ref)  1304   1305     def set_function_local(self, name, ref=None):  1306   1307         "Set the local with the given 'name' and optional 'ref'."  1308   1309         path = self.get_namespace_path()  1310         locals = self.function_locals[path]  1311         used = self.names_used.get(path)  1312   1313         if not locals.has_key(name) and used and name in used:  1314             raise InspectError("Name %s assigned locally but used previously." % name, path)  1315   1316         multiple = not ref or locals.has_key(name) and locals[name] != ref  1317         locals[name] = multiple and Reference("<var>") or ref  1318   1319     def assign_general_local(self, name, name_ref):  1320   1321         """  1322         Set for 'name' the given 'name_ref', recording the name for attribute  1323         usage tracking.  1324         """  1325   1326         self.set_general_local(name, name_ref)  1327         self.assign_name(name, name_ref)  1328   1329     def set_general_local(self, name, value=None):  1330   1331         """  1332         Set the 'name' with optional 'value' in any kind of local namespace,  1333         where the 'value' should be a reference if specified.  1334         """  1335   1336         init_value = self.get_initialising_value(value)  1337   1338         # Module global names.  1339   1340         if self.is_global(name):  1341             path = self.get_global_path(name)  1342             self.set_object(path, init_value)  1343   1344         # Function local names.  1345   1346         elif self.in_function:  1347             self.set_function_local(name, init_value)  1348   1349         # Other namespaces (classes).  1350   1351         else:  1352             self.set_name(name, init_value)  1353   1354     def set_name(self, name, ref=None):  1355   1356         "Attach the 'name' with optional 'ref' to the current namespace."  1357   1358         self.set_object(self.get_object_path(name), ref)  1359   1360     def set_instance_attr(self, name, ref=None):  1361   1362         """  1363         Add an instance attribute of the given 'name' to the current class,  1364         using the optional 'ref'.  1365         """  1366   1367         self._set_instance_attr(self.in_class, name, ref)  1368   1369     def _set_instance_attr(self, path, name, ref=None):  1370   1371         init_item(self.instance_attrs, path, set)  1372         self.instance_attrs[path].add(name)  1373   1374         if ref:  1375             init_item(self.instance_attr_constants, path, dict)  1376             self.instance_attr_constants[path][name] = ref  1377   1378     def get_initialising_value(self, value):  1379   1380         "Return a suitable initialiser reference for 'value'."  1381   1382         if isinstance(value, Result):  1383             return value.reference()  1384         else:  1385             return value  1386   1387     # Static, program-relative naming.  1388   1389     def find_name(self, name):  1390   1391         """  1392         Return the qualified name for the given 'name' used in the current  1393         non-function namespace.  1394         """  1395   1396         path = self.get_namespace_path()  1397         ref = None  1398   1399         if not self.in_function and name not in predefined_constants:  1400             if self.in_class:  1401                 ref = self.get_object(self.get_object_path(name), False)  1402             if not ref:  1403                 ref = self.get_global_or_builtin(name)  1404   1405         return ref  1406   1407     def get_class(self, node):  1408   1409         """  1410         Use the given 'node' to obtain the identity of a class. Return a  1411         reference for the class. Unresolved dependencies are permitted and must  1412         be resolved later.  1413         """  1414   1415         ref = self._get_class(node)  1416         return ref.has_kind(["<class>", "<depends>"]) and ref or None  1417   1418     def _get_class(self, node):  1419   1420         """  1421         Use the given 'node' to find a class definition. Return a reference to  1422         the class.  1423         """  1424   1425         if isinstance(node, compiler.ast.Getattr):  1426   1427             # Obtain the identity of the access target.  1428   1429             ref = self._get_class(node.expr)  1430   1431             # Where the target is a class or module, obtain the identity of the  1432             # attribute.  1433   1434             if ref.has_kind(["<function>", "<var>"]):  1435                 return None  1436             else:  1437                 attrname = "%s.%s" % (ref.get_origin(), node.attrname)  1438                 return self.get_object(attrname)  1439   1440         # Names can be module-level or built-in.  1441   1442         elif isinstance(node, compiler.ast.Name):  1443   1444             # Record usage of the name and attempt to identify it.  1445   1446             self.record_name(node.name)  1447             return self.find_name(node.name)  1448         else:  1449             return None  1450   1451     def get_constant(self, name, value):  1452   1453         "Return a constant reference for the given type 'name' and 'value'."  1454   1455         ref = self.get_builtin_class(name)  1456         return self.get_constant_reference(ref, value)  1457   1458     def get_literal_instance(self, n, name=None):  1459   1460         """  1461         For node 'n', return a reference to an instance of 'name', or if 'name'  1462         is not specified, deduce the type from the value.  1463         """  1464   1465         # Handle stray None constants (Sliceobj seems to produce them).  1466   1467         if name == "NoneType":  1468             return self.process_name_node(compiler.ast.Name("None"))  1469   1470         # Obtain the details of the literal itself.  1471         # An alias to the type is generated for sequences.  1472   1473         if name in ("dict", "list", "tuple"):  1474             ref = self.get_builtin_class(name)  1475             self.set_special_literal(name, ref)  1476             return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef)  1477   1478         # Constant values are independently recorded.  1479   1480         else:  1481             value, typename, encoding = self.get_constant_value(n.value, n.literals)  1482             ref = self.get_builtin_class(typename)  1483             return self.get_constant_reference(ref, value, encoding)  1484   1485     # Special names.  1486   1487     def get_special(self, name):  1488   1489         "Return any stored value for the given special 'name'."  1490   1491         value = self.special.get(name)  1492         if value:  1493             ref, paths = value  1494         else:  1495             ref = None  1496         return ref  1497   1498     def set_special(self, name, value):  1499   1500         """  1501         Set a special 'name' that merely tracks the use of an implicit object  1502         'value'.  1503         """  1504   1505         if not self.special.has_key(name):  1506             paths = set()  1507             self.special[name] = value, paths  1508         else:  1509             _ref, paths = self.special[name]  1510   1511         paths.add(self.get_namespace_path())  1512   1513     def set_special_literal(self, name, ref):  1514   1515         """  1516         Set a special name for the literal type 'name' having type 'ref'. Such  1517         special names provide a way of referring to literal object types.  1518         """  1519   1520         literal_name = "$L%s" % name  1521         value = ResolvedNameRef(literal_name, ref)  1522         self.set_special(literal_name, value)  1523   1524     # Exceptions.  1525   1526     def record_exception_handler(self):  1527   1528         "Record the current namespace as employing an exception handler."  1529   1530         self.exception_namespaces.add(self.get_namespace_path())  1531   1532     # Return values.  1533   1534     def record_return_value(self, expr):  1535   1536         "Record the given return 'expr'."  1537   1538         path = self.get_namespace_path()  1539         l = init_item(self.return_values, path, list)  1540         l.append(expr)  1541         if not self.importer.all_return_values.has_key(path):  1542             self.importer.all_return_values[path] = l  1543   1544 # vim: tabstop=4 expandtab shiftwidth=4