Lichen

inspector.py

934:2989aab1b4f7
5 months ago Paul Boddie Renamed the utf8string class to unicode, eliminating the unicode function. This means that the simple case of merely returning an object if it is already a Unicode object no longer occurs when using the unicode callable, but such behaviour might be better supported with more general customised instantiation functionality.
     1 #!/usr/bin/env python     2      3 """     4 Inspect and obtain module structure.     5      6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017,     7               2018, 2019 Paul Boddie <paul@boddie.org.uk>     8      9 This program is free software; you can redistribute it and/or modify it under    10 the terms of the GNU General Public License as published by the Free Software    11 Foundation; either version 3 of the License, or (at your option) any later    12 version.    13     14 This program is distributed in the hope that it will be useful, but WITHOUT    15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS    16 FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more    17 details.    18     19 You should have received a copy of the GNU General Public License along with    20 this program.  If not, see <http://www.gnu.org/licenses/>.    21 """    22     23 from branching import BranchTracker    24 from common import CommonModule, get_argnames, init_item, \    25                    predefined_constants, privileged_attributes    26 from modules import BasicModule, CacheWritingModule, InspectionNaming    27 from errors import InspectError    28 from referencing import Reference    29 from resolving import NameResolving    30 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \    31                     LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \    32                     Result, VariableRef    33 import compiler    34 import sys    35     36 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming):    37     38     "A module inspector."    39     40     def __init__(self, name, importer):    41     42         "Initialise the module with basic details."    43     44         BasicModule.__init__(self, name, importer)    45     46         self.in_class = False    47         self.in_conditional = False    48     49         # Accesses to global attributes.    50     51         self.global_attr_accesses = {}    52     53         # Usage tracking.    54     55         self.trackers = []    56         self.attr_accessor_branches = {}    57     58     def __repr__(self):    59         return "InspectedModule(%r, %r)" % (self.name, self.importer)    60     61     # Principal methods.    62     63     def parse(self, filename):    64     65         "Parse the file having the given 'filename'."    66     67         self.parse_file(filename)    68     69         # Inspect the module.    70     71         self.start_tracking_in_module()    72     73         # Detect and record imports and globals declared in the module.    74     75         self.process_structure(self.astnode)    76     77         # Set the class of the module after the definition has occurred.    78     79         ref = self.get_builtin("module")    80         self.set_name("__class__", ref)    81         self.set_name("__name__", self.get_constant("string", self.name).reference())    82         self.set_name("__file__", self.get_constant("string", filename).reference())    83     84         # Reserve a constant for the encoding.    85     86         if self.encoding:    87             self.get_constant("string", self.encoding)    88     89         # Get module-level attribute usage details.    90     91         self.stop_tracking_in_module()    92     93         # Collect external name references.    94     95         self.collect_names()    96     97     def complete(self):    98     99         "Complete the module inspection."   100    101         # Resolve names not definitively mapped to objects.   102    103         self.resolve()   104    105         # Propagate to the importer information needed in subsequent activities.   106    107         self.propagate()   108    109     # Accessory methods.   110    111     def collect_names(self):   112    113         "Collect the names used by each scope."   114    115         for path in self.names_used.keys():   116             self.collect_names_for_path(path)   117    118     def collect_names_for_path(self, path):   119    120         """   121         Collect the names used by the given 'path'. These are propagated to the   122         importer in advance of any dependency resolution.   123         """   124    125         names = self.names_used.get(path)   126         if not names:   127             return   128    129         in_function = self.function_locals.has_key(path)   130    131         for name in names:   132             if in_function and name in self.function_locals[path]:   133                 continue   134    135             key = "%s.%s" % (path, name)   136    137             # Find local definitions.   138    139             ref = self.get_resolved_object(key, True)   140             if ref:   141                 self.set_name_reference(key, ref)   142                 continue   143    144             # Find global.   145    146             ref = self.get_global(name)   147             if ref:   148                 self.set_name_reference(key, ref)   149                 continue   150    151             # Find presumed built-in definitions.   152    153             ref = self.get_builtin(name)   154             self.set_name_reference(key, ref)   155    156     def set_name_reference(self, path, ref):   157    158         "Map the given name 'path' to 'ref'."   159    160         self.importer.all_name_references[path] = self.name_references[path] = ref   161    162     # Module structure traversal.   163    164     def process_structure_node(self, n):   165    166         "Process the individual node 'n'."   167    168         path = self.get_namespace_path()   169    170         # Module global detection.   171    172         if isinstance(n, compiler.ast.Global):   173             self.process_global_node(n)   174    175         # Module import declarations.   176    177         elif isinstance(n, compiler.ast.From):   178             self.process_from_node(n)   179    180         elif isinstance(n, compiler.ast.Import):   181             self.process_import_node(n)   182    183         # Nodes using operator module functions.   184    185         elif isinstance(n, compiler.ast.Operator):   186             return self.process_operator_node(n)   187    188         elif isinstance(n, compiler.ast.AugAssign):   189             self.process_augassign_node(n)   190    191         elif isinstance(n, compiler.ast.Compare):   192             return self.process_compare_node(n)   193    194         elif isinstance(n, compiler.ast.Slice):   195             return self.process_slice_node(n)   196    197         elif isinstance(n, compiler.ast.Sliceobj):   198             return self.process_sliceobj_node(n)   199    200         elif isinstance(n, compiler.ast.Subscript):   201             return self.process_subscript_node(n)   202    203         # Namespaces within modules.   204    205         elif isinstance(n, compiler.ast.Class):   206             self.process_class_node(n)   207    208         elif isinstance(n, compiler.ast.Function):   209             self.process_function_node(n, n.name)   210    211         elif isinstance(n, compiler.ast.Lambda):   212             return self.process_lambda_node(n)   213    214         # Assignments.   215    216         elif isinstance(n, compiler.ast.Assign):   217    218             # Handle each assignment node.   219    220             for node in n.nodes:   221                 self.process_assignment_node(node, n.expr)   222    223         # Assignments within non-Assign nodes.   224    225         elif isinstance(n, compiler.ast.AssName):   226             raise InspectError("Name assignment appearing outside assignment statement.", path, n)   227    228         elif isinstance(n, compiler.ast.AssAttr):   229             raise InspectError("Attribute assignment appearing outside assignment statement.", path, n)   230    231         # Accesses.   232    233         elif isinstance(n, compiler.ast.Getattr):   234             return self.process_attribute_access(n)   235    236         # Name recording for later testing.   237    238         elif isinstance(n, compiler.ast.Name):   239             return self.process_name_node(n)   240    241         # Conditional statement tracking.   242    243         elif isinstance(n, compiler.ast.For):   244             self.process_for_node(n)   245    246         elif isinstance(n, compiler.ast.While):   247             self.process_while_node(n)   248    249         elif isinstance(n, compiler.ast.If):   250             self.process_if_node(n)   251    252         elif isinstance(n, (compiler.ast.And, compiler.ast.Or)):   253             return self.process_logical_node(n)   254    255         # Exception control-flow tracking.   256    257         elif isinstance(n, compiler.ast.TryExcept):   258             self.process_try_node(n)   259    260         elif isinstance(n, compiler.ast.TryFinally):   261             self.process_try_finally_node(n)   262    263         # Control-flow modification statements.   264    265         elif isinstance(n, compiler.ast.Break):   266             self.trackers[-1].suspend_broken_branch()   267    268         elif isinstance(n, compiler.ast.Continue):   269             self.trackers[-1].suspend_continuing_branch()   270    271         elif isinstance(n, compiler.ast.Raise):   272             self.process_structure(n)   273             self.trackers[-1].abandon_branch()   274    275         elif isinstance(n, compiler.ast.Return):   276             self.record_return_value(self.process_structure_node(n.value))   277             self.trackers[-1].abandon_returning_branch()   278    279         # Print statements.   280    281         elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)):   282             self.process_print_node(n)   283    284         # Invocations.   285    286         elif isinstance(n, compiler.ast.CallFunc):   287             return self.process_invocation_node(n)   288    289         # Constant usage.   290    291         elif isinstance(n, compiler.ast.Const):   292             return self.get_literal_instance(n)   293    294         elif isinstance(n, compiler.ast.Dict):   295             return self.get_literal_instance(n, "dict")   296    297         elif isinstance(n, compiler.ast.List):   298             return self.get_literal_instance(n, "list")   299    300         elif isinstance(n, compiler.ast.Tuple):   301             return self.get_literal_instance(n, "tuple")   302    303         # All other nodes are processed depth-first.   304    305         else:   306             self.process_structure(n)   307    308         # By default, no expression details are returned.   309    310         return None   311    312     # Specific node handling.   313    314     def process_assignment_node(self, n, expr):   315    316         "Process the individual node 'n' to be assigned the contents of 'expr'."   317    318         # Names and attributes are assigned the entire expression.   319    320         if isinstance(n, compiler.ast.AssName):   321             if n.name == "self":   322                 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n)   323    324             name_ref = expr and self.process_structure_node(expr)   325    326             # Name assignments populate either function namespaces or the   327             # general namespace hierarchy.   328    329             self.assign_general_local(n.name, name_ref)   330    331             # Record usage of the name.   332    333             self.record_name(n.name)   334    335         elif isinstance(n, compiler.ast.AssAttr):   336             if expr:   337                 expr = self.process_structure_node(expr)   338    339             in_assignment = self.in_assignment   340             self.in_assignment = True   341             self.process_attribute_access(n)   342             self.in_assignment = in_assignment   343    344         # Lists and tuples are matched against the expression and their   345         # items assigned to expression items.   346    347         elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)):   348             self.process_assignment_node_items(n, expr)   349    350         # Slices and subscripts are permitted within assignment nodes.   351    352         elif isinstance(n, compiler.ast.Slice):   353             self.process_slice_node(n, expr)   354    355         elif isinstance(n, compiler.ast.Subscript):   356             self.process_subscript_node(n, expr)   357    358     def process_attribute_access(self, n):   359    360         "Process the given attribute access node 'n'."   361    362         path = self.get_namespace_path()   363    364         # Test for access to special privileged attributes.   365    366         if isinstance(n, compiler.ast.Getattr) and \   367            n.attrname in privileged_attributes and not n.privileged:   368    369             raise InspectError("Attribute %s is accessed by an unprivileged operation." %   370                                n.attrname, path, n)   371    372         # Obtain any completed chain and return the reference to it.   373    374         name_ref = self.process_attribute_chain(n)   375    376         if self.have_access_expression(n):   377             return name_ref   378    379         # Where the start of the chain of attributes has been reached, determine   380         # the complete access.   381    382         # Given a non-access node, this chain can be handled in its entirety,   383         # either being name-based and thus an access rooted on a name, or being   384         # based on some other node and thus an anonymous access of some kind.   385    386         # Start with the the full attribute chain.   387    388         remaining = self.attrs   389         attrnames = ".".join(remaining)   390    391         # If the accessor cannot be identified, or where attributes   392         # remain in an attribute chain, record the anonymous accesses.   393    394         if not isinstance(name_ref, NameRef): # includes ResolvedNameRef   395    396             init_item(self.attr_accesses, path, set)   397             self.attr_accesses[path].add(attrnames)   398    399             self.record_access_details(None, attrnames, self.in_assignment,   400                 self.in_invocation)   401             del self.attrs[0]   402             return   403    404         # Name-based accesses will handle the first attribute in a   405         # chain.   406    407         else:   408             attrname = remaining[0]   409    410             # Attribute assignments are used to identify instance attributes.   411    412             if isinstance(n, compiler.ast.AssAttr) and \   413                 self.in_class and self.in_function and n.expr.name == "self":   414    415                 self.set_instance_attr(attrname)   416    417             # Record attribute usage using any name local to this namespace,   418             # if assigned in the namespace, or using an external name   419             # (presently just globals within classes).   420    421             name = self.get_name_for_tracking(name_ref.name, name_ref)   422             tracker = self.trackers[-1]   423    424             immediate_access = len(self.attrs) == 1   425             assignment = immediate_access and isinstance(n, compiler.ast.AssAttr)   426    427             # Record global-based chains for subsequent resolution.   428    429             if name_ref.is_global_name():   430                 self.record_global_access_details(name, attrnames)   431    432             # Make sure the name is being tracked: global names will not   433             # already be initialised in a branch and must be added   434             # explicitly.   435    436             if not tracker.have_name(name):   437                 tracker.assign_names([name])   438                 if self.in_function:   439                     self.scope_globals[path].add(name)   440    441             # Record attribute usage in the tracker, and record the branch   442             # information for the access.   443    444             branches = tracker.use_attribute(name, attrname,   445                 self.in_invocation is not None, assignment)   446    447             if not branches:   448                 raise InspectError("Name %s is accessed using %s before an assignment." % (   449                     name, attrname), path, n)   450    451             self.record_branches_for_access(branches, name, attrnames)   452             access_number = self.record_access_details(name, attrnames,   453                 self.in_assignment, self.in_invocation)   454    455             del self.attrs[0]   456             return AccessRef(name, attrnames, access_number)   457    458     def process_class_node(self, n):   459    460         "Process the given class node 'n'."   461    462         path = self.get_namespace_path()   463    464         # To avoid notions of class "versions" where the same definition   465         # might be parameterised with different state and be referenced   466         # elsewhere (as base classes, for example), classes in functions or   467         # conditions are forbidden.   468    469         if self.in_function or self.in_conditional:   470             print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % (   471                 path, n.name)   472             return   473    474         # Resolve base classes.   475    476         bases = []   477    478         for base in n.bases:   479             base_class = self.get_class(base)   480    481             if not base_class:   482                 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % (   483                     path, n.name, base)   484                 return   485             else:   486                 bases.append(base_class)   487    488         # Detect conflicting definitions. Such definitions cause conflicts in   489         # the storage of namespace-related information.   490    491         class_name = self.get_object_path(n.name)   492         ref = self.get_object(class_name, defer=False)   493    494         if ref and ref.static():   495             raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n)   496    497         # Record bases for the class and retain the class name.   498         # Note that the function class does not inherit from the object class.   499    500         if not bases and class_name != "__builtins__.core.object" and \   501                          class_name != "__builtins__.core.function":   502    503             ref = self.get_object("__builtins__.object")   504             bases.append(ref)   505    506         self.importer.classes[class_name] = self.classes[class_name] = bases   507         self.importer.subclasses[class_name] = set()   508         self.scope_globals[class_name] = set()   509    510         # Set the definition before entering the namespace rather than   511         # afterwards because methods may reference it. In normal Python,   512         # a class is not accessible until the definition is complete, but   513         # methods can generally reference it since upon being called the   514         # class will already exist.   515    516         self.set_definition(n.name, "<class>")   517    518         in_class = self.in_class   519         self.in_class = class_name   520         self.set_instance_attr("__class__", Reference("<class>", class_name))   521         self.enter_namespace(n.name)   522    523         # Do not provide the special instantiator attributes on the function   524         # class. Function instances provide these attributes.   525    526         if class_name != "__builtins__.core.function":   527    528             self.set_name("__fn__") # special instantiator attribute   529             self.set_name("__args__") # special instantiator attribute   530    531         # Provide leafname, parent and context attributes.   532    533         parent, leafname = class_name.rsplit(".", 1)   534         self.set_name("__name__", self.get_constant("string", leafname).reference())   535    536         if class_name != "__builtins__.core.function":   537             self.set_name("__parent__")   538    539         self.process_structure_node(n.code)   540         self.exit_namespace()   541         self.in_class = in_class   542    543     def process_from_node(self, n):   544    545         "Process the given node 'n', importing from another module."   546    547         path = self.get_namespace_path()   548    549         module_name, names = self.get_module_name(n)   550         if module_name == self.name:   551             raise InspectError("Cannot import from the current module.", path, n)   552    553         self.queue_module(module_name)   554    555         # Attempt to obtain the referenced objects.   556    557         for name, alias in n.names:   558             if name == "*":   559                 raise InspectError("Only explicitly specified names can be imported from modules.", path, n)   560    561             # Explicit names.   562    563             ref = self.import_name_from_module(name, module_name)   564             value = ResolvedNameRef(alias or name, ref)   565             self.set_general_local(alias or name, value)   566    567     def process_function_node(self, n, name):   568    569         """   570         Process the given function or lambda node 'n' with the given 'name'.   571         """   572    573         is_lambda = isinstance(n, compiler.ast.Lambda)   574    575         # Where a function is declared conditionally, use a separate name for   576         # the definition, and assign the definition to the stated name.   577    578         if (self.in_conditional or self.in_function) and not is_lambda:   579             original_name = name   580             name = self.get_lambda_name()   581         else:   582             original_name = None   583    584         # Detect conflicting definitions. Such definitions cause conflicts in   585         # the storage of namespace-related information.   586    587         function_name = self.get_object_path(name)   588         ref = self.get_object(function_name, defer=False)   589    590         if ref and ref.static():   591             raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n)   592    593         # Initialise argument and local records.   594    595         argnames = get_argnames(n.argnames)   596         is_method = self.in_class and not self.in_function   597    598         # Remove explicit "self" from method parameters.   599    600         if is_method and argnames and argnames[0] == "self":   601             del argnames[0]   602    603         # Convert .name entries in the parameters, provided this is a method.   604    605         l = []   606         attr_initialisers = []   607    608         for argname in argnames:   609             if argname[0] == ".":   610                 if not is_method:   611                     raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n)   612    613                 argname = argname[1:]   614                 attr_initialisers.append(argname)   615    616             if argname in l:   617                 raise InspectError("Argument name %s is repeated." % argname, function_name, n)   618    619             l.append(argname)   620    621         argnames = l   622    623         # Copy and propagate the parameters.   624    625         self.importer.function_parameters[function_name] = \   626             self.function_parameters[function_name] = argnames[:]   627    628         self.importer.function_attr_initialisers[function_name] = \   629             self.function_attr_initialisers[function_name] = attr_initialisers   630    631         # Define all arguments/parameters in the local namespace.   632    633         locals = \   634             self.importer.function_locals[function_name] = \   635             self.function_locals[function_name] = {}   636    637         # Insert "self" into method locals.   638    639         if is_method:   640             argnames.insert(0, "self")   641    642         # Define "self" in terms of the class if in a method.   643         # This does not diminish the need for type-narrowing in the deducer.   644    645         if argnames:   646             if self.in_class and not self.in_function and argnames[0] == "self":   647                 locals[argnames[0]] = Reference("<instance>", self.in_class)   648             else:   649                 locals[argnames[0]] = Reference("<var>")   650    651         for argname in argnames[1:]:   652             locals[argname] = Reference("<var>")   653    654         globals = self.scope_globals[function_name] = set()   655    656         # Process the defaults.   657    658         defaults = self.importer.function_defaults[function_name] = \   659                    self.function_defaults[function_name] = []   660    661         for argname, default in compiler.ast.get_defaults(n):   662             if argname[0] == ".":   663                 argname = argname[1:]   664    665             if default:   666    667                 # Obtain any reference for the default.   668    669                 name_ref = self.process_structure_node(default)   670                 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>")))   671    672         # Reset conditional tracking to focus on the function contents.   673    674         in_conditional = self.in_conditional   675         self.in_conditional = False   676    677         in_function = self.in_function   678         self.in_function = function_name   679    680         self.enter_namespace(name)   681    682         # Define a leafname attribute value for the function instance.   683    684         ref = self.get_builtin_class("string")   685         self.reserve_constant(function_name, name, ref.get_origin())   686    687         # Track attribute usage within the namespace.   688    689         path = self.get_namespace_path()   690         self.start_tracking(locals)   691    692         # Establish attributes for .name entries, provided this is a method.   693    694         for argname in attr_initialisers:   695             self.process_assignment_node(   696                     compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"),   697                     compiler.ast.Name(argname))   698    699         self.process_structure_node(n.code)   700         returns_value = self.stop_tracking()   701    702         # Record any null result.   703    704         is_initialiser = is_method and name == "__init__"   705    706         if not returns_value and not is_initialiser:   707             self.record_return_value(ResolvedNameRef("None", self.get_builtin("None")))   708    709         # Exit to the parent.   710    711         self.exit_namespace()   712    713         # Update flags.   714    715         self.in_function = in_function   716         self.in_conditional = in_conditional   717    718         # Define the function using the appropriate name.   719    720         self.set_definition(name, "<function>")   721    722         # Where a function is set conditionally, assign the name.   723    724         if original_name:   725             self.process_assignment_for_object(original_name, compiler.ast.Name(name))   726    727     def process_global_node(self, n):   728    729         """   730         Process the given "global" node 'n'.   731         """   732    733         path = self.get_namespace_path()   734    735         if path != self.name:   736             self.scope_globals[path].update(n.names)   737    738     def process_if_node(self, n):   739    740         """   741         Process the given "if" node 'n'.   742         """   743    744         tracker = self.trackers[-1]   745         tracker.new_branchpoint()   746    747         for test, body in n.tests:   748             self.process_structure_node(test)   749    750             tracker.new_branch()   751    752             in_conditional = self.in_conditional   753             self.in_conditional = True   754             self.process_structure_node(body)   755             self.in_conditional = in_conditional   756    757             tracker.shelve_branch()   758    759         # Maintain a branch for the else clause.   760    761         tracker.new_branch()   762         if n.else_:   763             self.process_structure_node(n.else_)   764         tracker.shelve_branch()   765    766         tracker.merge_branches()   767    768     def process_import_node(self, n):   769    770         "Process the given import node 'n'."   771    772         path = self.get_namespace_path()   773    774         # Load the mentioned module.   775    776         for name, alias in n.names:   777             if name == self.name:   778                 raise InspectError("Cannot import the current module.", path, n)   779    780             self.set_module(alias or name.split(".")[-1], name)   781             self.queue_module(name, True)   782    783     def process_invocation_node(self, n):   784    785         "Process the given invocation node 'n'."   786    787         path = self.get_namespace_path()   788    789         in_invocation = self.in_invocation   790         self.in_invocation = None   791    792         # Process the arguments.   793    794         keywords = set()   795    796         for arg in n.args:   797             self.process_structure_node(arg)   798             if isinstance(arg, compiler.ast.Keyword):   799                 keywords.add(arg.name)   800    801         keywords = list(keywords)   802         keywords.sort()   803    804         # Communicate to the invocation target expression that it forms the   805         # target of an invocation, potentially affecting attribute accesses.   806    807         self.in_invocation = len(n.args), keywords   808    809         # Process the expression, obtaining any identified reference.   810    811         name_ref = self.process_structure_node(n.node)   812         self.in_invocation = in_invocation   813    814         # Detect class invocations.   815    816         if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"):   817             return InstanceRef(name_ref.reference().instance_of())   818    819         elif isinstance(name_ref, (NameRef, AccessRef)):   820             return InvocationRef(name_ref)   821    822         # Provide a general reference to indicate that something is produced   823         # by the invocation, useful for retaining assignment expression   824         # details.   825    826         return VariableRef()   827    828     def process_lambda_node(self, n):   829    830         "Process the given lambda node 'n'."   831    832         name = self.get_lambda_name()   833         self.process_function_node(n, name)   834    835         origin = self.get_object_path(name)   836    837         if self.function_defaults.get(origin):   838             return None   839         else:   840             return ResolvedNameRef(name, Reference("<function>", origin))   841    842     def process_logical_node(self, n):   843    844         "Process the given operator node 'n'."   845    846         return self.process_operator_chain(n.nodes)   847    848     def process_name_node(self, n):   849    850         "Process the given name node 'n'."   851    852         path = self.get_namespace_path()   853    854         # Find predefined constant names before anything else.   855    856         if n.name in predefined_constants:   857             ref = self.get_builtin(n.name)   858             value = ResolvedNameRef(n.name, ref)   859             return value   860    861         # Special names that have already been identified.   862    863         if n.name.startswith("$"):   864             value = self.get_special(n.name)   865             if value:   866                 return value   867    868         # Special case for operator functions introduced through code   869         # transformations.   870    871         if n.name.startswith("$op"):   872    873             # Obtain the location of the actual function defined in the operator   874             # package.   875    876             op = n.name[len("$op"):]   877    878             # Attempt to get a reference.   879    880             ref = self.import_name_from_module(op, "operator")   881    882             # Record the imported name and provide the resolved name reference.   883    884             value = ResolvedNameRef(n.name, ref)   885             self.set_special(n.name, value)   886             return value   887    888         # Special case for sequence length testing.   889    890         elif n.name.startswith("$seq"):   891             op = n.name[len("$seq"):]   892             ref = self.import_name_from_module(op, "__builtins__.sequence")   893             value = ResolvedNameRef(n.name, ref)   894             self.set_special(n.name, value)   895             return value   896    897         # Special case for print operations.   898    899         elif n.name.startswith("$print"):   900    901             # Attempt to get a reference.   902    903             ref = self.get_builtin("print_")   904    905             # Record the imported name and provide the resolved name reference.   906    907             value = ResolvedNameRef(n.name, ref)   908             self.set_special(n.name, value)   909             return value   910    911         # Test for self usage, which is only allowed in methods.   912    913         if n.name == "self" and not (self.in_function and self.in_class):   914             raise InspectError("Use of self is only allowed in methods.", path, n)   915    916         # Record usage of the name.   917    918         self.record_name(n.name)   919    920         # Search for unknown names in non-function scopes immediately.   921         # Temporary names should not be re-used between scopes.   922         # External names in functions are resolved later.   923    924         ref = not n.name.startswith("$t") and self.find_name(n.name) or None   925    926         if ref:   927             self.record_name_access(n.name, True)   928             return ResolvedNameRef(n.name, ref, is_global=True)   929    930         # Explicitly-declared global names.   931    932         elif self.in_function and n.name in self.scope_globals[path]:   933             self.record_name_access(n.name, True)   934             return NameRef(n.name, is_global=True)   935    936         # Examine other names.   937    938         else:   939    940             # Check local names.   941    942             access_number = self.record_name_access(n.name)   943    944             # Local name.   945    946             if access_number is not None:   947                 return LocalNameRef(n.name, access_number)   948    949             # Possible global or built-in name.   950    951             else:   952                 self.record_name_access(n.name, True)   953                 return NameRef(n.name, is_global=True)   954    955     def record_name_access(self, name, is_global=False):   956    957         """   958         Record an access involving 'name' if the name is being tracked, using   959         'is_global' to indicate whether the name is global.   960         """   961    962         name = self.get_name_for_tracking(name, is_global=is_global)   963         branches = self.trackers[-1].tracking_name(name)   964         if branches:   965             self.record_branches_for_access(branches, name, None)   966             return self.record_access_details(name, None, self.in_assignment,   967                                               self.in_invocation)   968         return None   969    970     def process_operator_chain(self, nodes):   971    972         """   973         Process the given chain of 'nodes', processing each node or item.   974         Each node starts a new conditional region, effectively making a deeply-   975         nested collection of if-like statements.   976         """   977    978         results = []   979    980         tracker = self.trackers[-1]   981    982         for item in nodes:   983             tracker.new_branchpoint()   984             tracker.new_branch()   985             result = self.process_structure_node(item)   986             if result:   987                 results.append(result)   988    989         for item in nodes[:-1]:   990             tracker.shelve_branch()   991             tracker.new_branch()   992             tracker.shelve_branch()   993             tracker.merge_branches()   994    995         tracker.shelve_branch()   996         tracker.merge_branches()   997    998         return MultipleRef(results)   999   1000     def process_try_node(self, n):  1001   1002         """  1003         Process the given "try...except" node 'n'.  1004         """  1005   1006         self.record_exception_handler()  1007   1008         tracker = self.trackers[-1]  1009         tracker.new_branchpoint()  1010   1011         self.process_structure_node(n.body)  1012   1013         for name, var, handler in n.handlers:  1014             if name is not None:  1015                 self.process_structure_node(name)  1016   1017             # Any abandoned branches from the body can now be resumed in a new  1018             # branch.  1019   1020             tracker.resume_abandoned_branches()  1021   1022             # Establish the local for the handler.  1023   1024             if var is not None:  1025                 self.process_assignment_node(var, None)  1026             if handler is not None:  1027                 self.process_structure_node(handler)  1028   1029             tracker.shelve_branch()  1030   1031         # The else clause maintains the usage from the body but without the  1032         # abandoned branches since they would never lead to the else clause  1033         # being executed.  1034   1035         if n.else_:  1036             tracker.new_branch()  1037             self.process_structure_node(n.else_)  1038             tracker.shelve_branch()  1039   1040         # Without an else clause, a null branch propagates the successful  1041         # outcome.  1042   1043         else:  1044             tracker.new_branch()  1045             tracker.shelve_branch()  1046   1047         tracker.merge_branches()  1048   1049     def process_try_finally_node(self, n):  1050   1051         """  1052         Process the given "try...finally" node 'n'.  1053         """  1054   1055         self.record_exception_handler()  1056   1057         tracker = self.trackers[-1]  1058         self.process_structure_node(n.body)  1059   1060         # Any abandoned branches from the body can now be resumed.  1061   1062         branches = tracker.resume_all_abandoned_branches()  1063         self.process_structure_node(n.final)  1064   1065         # At the end of the finally clause, abandoned branches are discarded.  1066   1067         tracker.restore_active_branches(branches)  1068   1069     def process_while_node(self, n):  1070   1071         "Process the given while node 'n'."  1072   1073         tracker = self.trackers[-1]  1074         tracker.new_branchpoint(loop_node=True)  1075   1076         # Evaluate any test or iterator outside the loop.  1077   1078         self.process_structure_node(n.test)  1079   1080         # Propagate attribute usage to branches.  1081   1082         tracker.new_branch(loop_node=True)  1083   1084         # Enter the loop.  1085   1086         in_conditional = self.in_conditional  1087         self.in_conditional = True  1088         self.process_structure_node(n.body)  1089         self.in_conditional = in_conditional  1090   1091         # Continuing branches are resumed before any test.  1092   1093         tracker.resume_continuing_branches()  1094   1095         # Evaluate any continuation test within the body.  1096   1097         self.process_structure_node(n.test)  1098   1099         tracker.shelve_branch(loop_node=True)  1100   1101         # Support the non-looping condition.  1102   1103         tracker.new_branch()  1104         tracker.shelve_branch()  1105   1106         tracker.merge_branches()  1107   1108         # Evaluate any else clause outside branches.  1109   1110         if n.else_:  1111             self.process_structure_node(n.else_)  1112   1113         # Connect broken branches to the code after any loop.  1114   1115         tracker.resume_broken_branches()  1116   1117     # Branch tracking methods.  1118   1119     def start_tracking(self, names):  1120   1121         """  1122         Start tracking attribute usage for names in the current namespace,  1123         immediately registering the given 'names'.  1124         """  1125   1126         path = self.get_namespace_path()  1127         parent = self.trackers[-1]  1128         tracker = BranchTracker()  1129         self.trackers.append(tracker)  1130   1131         # Record the given names established as new branches.  1132   1133         tracker.assign_names(names)  1134   1135     def assign_name(self, name, name_ref):  1136   1137         "Assign to 'name' the given 'name_ref' in the current namespace."  1138   1139         name = self.get_name_for_tracking(name)  1140         self.trackers[-1].assign_names([name], [name_ref])  1141   1142     def stop_tracking(self):  1143   1144         """  1145         Stop tracking attribute usage, recording computed usage for the current  1146         namespace. Indicate whether a value is always returned from the  1147         namespace.  1148         """  1149   1150         path = self.get_namespace_path()  1151         tracker = self.trackers.pop()  1152         self.record_assignments_for_access(tracker)  1153   1154         self.attr_usage[path] = tracker.get_all_usage()  1155         self.name_initialisers[path] = tracker.get_all_values()  1156   1157         return tracker.returns_value()  1158   1159     def start_tracking_in_module(self):  1160   1161         "Start tracking attribute usage in the module."  1162   1163         tracker = BranchTracker()  1164         self.trackers.append(tracker)  1165   1166     def stop_tracking_in_module(self):  1167   1168         "Stop tracking attribute usage in the module."  1169   1170         tracker = self.trackers[0]  1171         self.record_assignments_for_access(tracker)  1172         self.attr_usage[self.name] = tracker.get_all_usage()  1173         self.name_initialisers[self.name] = tracker.get_all_values()  1174   1175     def record_assignments_for_access(self, tracker):  1176   1177         """  1178         For the current path, use the given 'tracker' to record assignment  1179         version information for attribute accesses.  1180         """  1181   1182         path = self.get_path_for_access()  1183   1184         if not self.attr_accessor_branches.has_key(path):  1185             return  1186   1187         init_item(self.attr_accessors, path, dict)  1188         attr_accessors = self.attr_accessors[path]  1189   1190         # Obtain the branches applying during each access.  1191   1192         for access, all_branches in self.attr_accessor_branches[path].items():  1193             name, attrnames = access  1194             init_item(attr_accessors, access, list)  1195   1196             # Obtain the assignments applying to each branch.  1197   1198             for branches in all_branches:  1199                 positions = tracker.get_assignment_positions_for_branches(name, branches)  1200   1201                 # Detect missing name information.  1202   1203                 if None in positions:  1204                     globals = self.global_attr_accesses.get(path)  1205                     accesses = globals and globals.get(name)  1206                     if not accesses:  1207                         print >>sys.stderr, "In %s, %s may not be defined when used." % (  1208                             self.get_namespace_path(), name)  1209                     positions.remove(None)  1210   1211                 attr_accessors[access].append(positions)  1212   1213     def record_branches_for_access(self, branches, name, attrnames):  1214   1215         """  1216         Record the given 'branches' for an access involving the given 'name' and  1217         'attrnames'.  1218         """  1219   1220         access = name, attrnames  1221         path = self.get_path_for_access()  1222   1223         init_item(self.attr_accessor_branches, path, dict)  1224         attr_accessor_branches = self.attr_accessor_branches[path]  1225   1226         init_item(attr_accessor_branches, access, list)  1227         attr_accessor_branches[access].append(branches)  1228   1229     def record_access_details(self, name, attrnames, assignment, invocation):  1230   1231         """  1232         For the given 'name' and 'attrnames', record an access indicating  1233         whether an 'assignment' or an 'invocation' is occurring.  1234   1235         These details correspond to accesses otherwise recorded by the attribute  1236         accessor and attribute access dictionaries.  1237         """  1238   1239         access = name, attrnames  1240         path = self.get_path_for_access()  1241   1242         init_item(self.attr_access_modifiers, path, dict)  1243         init_item(self.attr_access_modifiers[path], access, list)  1244   1245         access_number = len(self.attr_access_modifiers[path][access])  1246         self.attr_access_modifiers[path][access].append((assignment, invocation))  1247         return access_number  1248   1249     def record_global_access_details(self, name, attrnames):  1250   1251         """  1252         Record details of a global access via the given 'name' involving the  1253         indicated 'attrnames'.  1254         """  1255   1256         path = self.get_namespace_path()  1257   1258         init_item(self.global_attr_accesses, path, dict)  1259         init_item(self.global_attr_accesses[path], name, set)  1260         self.global_attr_accesses[path][name].add(attrnames)  1261   1262     # Namespace modification.  1263   1264     def record_name(self, name):  1265   1266         "Record the use of 'name' in a namespace."  1267   1268         path = self.get_namespace_path()  1269         init_item(self.names_used, path, set)  1270         self.names_used[path].add(name)  1271   1272     def set_module(self, name, module_name):  1273   1274         """  1275         Set a module in the current namespace using the given 'name' associated  1276         with the corresponding 'module_name'.  1277         """  1278   1279         if name:  1280             self.set_general_local(name, Reference("<module>", module_name))  1281   1282     def set_definition(self, name, kind):  1283   1284         """  1285         Set the definition having the given 'name' and 'kind'.  1286   1287         Definitions are set in the static namespace hierarchy, but they can also  1288         be recorded for function locals.  1289         """  1290   1291         if self.is_global(name):  1292             print >>sys.stderr, "In %s, %s is defined as being global." % (  1293                 self.get_namespace_path(), name)  1294   1295         path = self.get_object_path(name)  1296         self.set_object(path, kind)  1297   1298         ref = self.get_object(path)  1299         if ref.get_kind() == "<var>":  1300             print >>sys.stderr, "In %s, %s is defined more than once." % (  1301                 self.get_namespace_path(), name)  1302   1303         if not self.is_global(name) and self.in_function:  1304             self.set_function_local(name, ref)  1305   1306     def set_function_local(self, name, ref=None):  1307   1308         "Set the local with the given 'name' and optional 'ref'."  1309   1310         path = self.get_namespace_path()  1311         locals = self.function_locals[path]  1312         used = self.names_used.get(path)  1313   1314         if not locals.has_key(name) and used and name in used:  1315             raise InspectError("Name %s assigned locally but used previously." % name, path)  1316   1317         multiple = not ref or locals.has_key(name) and locals[name] != ref  1318         locals[name] = multiple and Reference("<var>") or ref  1319   1320     def assign_general_local(self, name, name_ref):  1321   1322         """  1323         Set for 'name' the given 'name_ref', recording the name for attribute  1324         usage tracking.  1325         """  1326   1327         self.set_general_local(name, name_ref)  1328         self.assign_name(name, name_ref)  1329   1330     def set_general_local(self, name, value=None):  1331   1332         """  1333         Set the 'name' with optional 'value' in any kind of local namespace,  1334         where the 'value' should be a reference if specified.  1335         """  1336   1337         init_value = self.get_initialising_value(value)  1338   1339         # Module global names.  1340   1341         if self.is_global(name):  1342             path = self.get_global_path(name)  1343             self.set_object(path, init_value)  1344   1345         # Function local names.  1346   1347         elif self.in_function:  1348             self.set_function_local(name, init_value)  1349   1350         # Other namespaces (classes).  1351   1352         else:  1353             self.set_name(name, init_value)  1354   1355     def set_name(self, name, ref=None):  1356   1357         "Attach the 'name' with optional 'ref' to the current namespace."  1358   1359         self.set_object(self.get_object_path(name), ref)  1360   1361     def set_instance_attr(self, name, ref=None):  1362   1363         """  1364         Add an instance attribute of the given 'name' to the current class,  1365         using the optional 'ref'.  1366         """  1367   1368         self._set_instance_attr(self.in_class, name, ref)  1369   1370     def _set_instance_attr(self, path, name, ref=None):  1371   1372         init_item(self.instance_attrs, path, set)  1373         self.instance_attrs[path].add(name)  1374   1375         if ref:  1376             init_item(self.instance_attr_constants, path, dict)  1377             self.instance_attr_constants[path][name] = ref  1378   1379     def get_initialising_value(self, value):  1380   1381         "Return a suitable initialiser reference for 'value'."  1382   1383         if isinstance(value, Result):  1384             return value.reference()  1385         else:  1386             return value  1387   1388     # Static, program-relative naming.  1389   1390     def find_name(self, name):  1391   1392         """  1393         Return the qualified name for the given 'name' used in the current  1394         non-function namespace.  1395         """  1396   1397         path = self.get_namespace_path()  1398         ref = None  1399   1400         if not self.in_function and name not in predefined_constants:  1401             if self.in_class:  1402                 ref = self.get_object(self.get_object_path(name), False)  1403             if not ref:  1404                 ref = self.get_global_or_builtin(name)  1405   1406         return ref  1407   1408     def get_class(self, node):  1409   1410         """  1411         Use the given 'node' to obtain the identity of a class. Return a  1412         reference for the class. Unresolved dependencies are permitted and must  1413         be resolved later.  1414         """  1415   1416         ref = self._get_class(node)  1417         return ref.has_kind(["<class>", "<depends>"]) and ref or None  1418   1419     def _get_class(self, node):  1420   1421         """  1422         Use the given 'node' to find a class definition. Return a reference to  1423         the class.  1424         """  1425   1426         if isinstance(node, compiler.ast.Getattr):  1427   1428             # Obtain the identity of the access target.  1429   1430             ref = self._get_class(node.expr)  1431   1432             # Where the target is a class or module, obtain the identity of the  1433             # attribute.  1434   1435             if ref.has_kind(["<function>", "<var>"]):  1436                 return None  1437             else:  1438                 attrname = "%s.%s" % (ref.get_origin(), node.attrname)  1439                 return self.get_object(attrname)  1440   1441         # Names can be module-level or built-in.  1442   1443         elif isinstance(node, compiler.ast.Name):  1444   1445             # Record usage of the name and attempt to identify it.  1446   1447             self.record_name(node.name)  1448             return self.find_name(node.name)  1449         else:  1450             return None  1451   1452     def get_constant(self, name, value):  1453   1454         "Return a constant reference for the given type 'name' and 'value'."  1455   1456         ref = self.get_builtin_class(name)  1457         return self.get_constant_reference(ref, value)  1458   1459     def get_literal_instance(self, n, name=None):  1460   1461         """  1462         For node 'n', return a reference to an instance of 'name', or if 'name'  1463         is not specified, deduce the type from the value.  1464         """  1465   1466         # Handle stray None constants (Sliceobj seems to produce them).  1467   1468         if name == "NoneType":  1469             return self.process_name_node(compiler.ast.Name("None"))  1470   1471         # Obtain the details of the literal itself.  1472         # An alias to the type is generated for sequences.  1473   1474         if name in ("dict", "list", "tuple"):  1475             ref = self.get_builtin_class(name)  1476             self.set_special_literal(name, ref)  1477             return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef)  1478   1479         # Constant values are independently recorded.  1480   1481         else:  1482             value, typename, encoding = self.get_constant_value(n.value, n.literals)  1483             ref = self.get_builtin_class(typename)  1484             return self.get_constant_reference(ref, value, encoding)  1485   1486     # Special names.  1487   1488     def get_special(self, name):  1489   1490         "Return any stored value for the given special 'name'."  1491   1492         value = self.special.get(name)  1493         if value:  1494             ref, paths = value  1495         else:  1496             ref = None  1497         return ref  1498   1499     def set_special(self, name, value):  1500   1501         """  1502         Set a special 'name' that merely tracks the use of an implicit object  1503         'value'.  1504         """  1505   1506         if not self.special.has_key(name):  1507             paths = set()  1508             self.special[name] = value, paths  1509         else:  1510             _ref, paths = self.special[name]  1511   1512         paths.add(self.get_namespace_path())  1513   1514     def set_special_literal(self, name, ref):  1515   1516         """  1517         Set a special name for the literal type 'name' having type 'ref'. Such  1518         special names provide a way of referring to literal object types.  1519         """  1520   1521         literal_name = "$L%s" % name  1522         value = ResolvedNameRef(literal_name, ref)  1523         self.set_special(literal_name, value)  1524   1525     # Exceptions.  1526   1527     def record_exception_handler(self):  1528   1529         "Record the current namespace as employing an exception handler."  1530   1531         self.exception_namespaces.add(self.get_namespace_path())  1532   1533     # Return values.  1534   1535     def record_return_value(self, expr):  1536   1537         "Record the given return 'expr'."  1538   1539         path = self.get_namespace_path()  1540         l = init_item(self.return_values, path, list)  1541         l.append(expr)  1542         if not self.importer.all_return_values.has_key(path):  1543             self.importer.all_return_values[path] = l  1544   1545 # vim: tabstop=4 expandtab shiftwidth=4