1.1 --- a/inspector.py Sat Sep 03 22:38:21 2016 +0200
1.2 +++ b/inspector.py Sat Sep 03 22:43:44 2016 +0200
1.3 @@ -21,48 +21,23 @@
1.4 """
1.5
1.6 from branching import BranchTracker
1.7 -from common import *
1.8 -from modules import *
1.9 -from os import listdir
1.10 -from os.path import extsep, split, splitext
1.11 +from common import get_argnames, init_item, predefined_constants
1.12 +from modules import BasicModule, CacheWritingModule
1.13 from referencing import Reference
1.14 +from resolving import NameResolving
1.15 +from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \
1.16 + LocalNameRef, NameRef, ResolvedNameRef
1.17 import compiler
1.18 import sys
1.19
1.20 -class AccessRef(Result):
1.21 -
1.22 - """
1.23 - A reference to an attribute access that is generally only returned from a
1.24 - processed access for possible initialiser resolution for assignments.
1.25 - """
1.26 -
1.27 - def __init__(self, original_name, attrnames, number):
1.28 - self.original_name = original_name
1.29 - self.attrnames = attrnames
1.30 - self.number = number
1.31 -
1.32 - def reference(self):
1.33 - return None
1.34 -
1.35 - def __repr__(self):
1.36 - return "AccessRef(%r, %r, %r)" % (self.original_name, self.attrnames, self.number)
1.37 -
1.38 -class InvocationRef(Result):
1.39 -
1.40 - "An invocation of a name reference."
1.41 -
1.42 - def __init__(self, name_ref):
1.43 - self.name_ref = name_ref
1.44 -
1.45 - def __repr__(self):
1.46 - return "InvocationRef(%r)" % self.name_ref
1.47 -
1.48 -class InspectedModule(BasicModule, CacheWritingModule):
1.49 +class InspectedModule(BasicModule, CacheWritingModule, NameResolving):
1.50
1.51 "A module inspector."
1.52
1.53 def __init__(self, name, importer):
1.54 BasicModule.__init__(self, name, importer)
1.55 + NameResolving.__init__(self)
1.56 +
1.57 self.in_class = False
1.58 self.in_conditional = False
1.59 self.global_attr_accesses = {}
1.60 @@ -105,340 +80,21 @@
1.61
1.62 self.stop_tracking_in_module()
1.63
1.64 - # Check names used and resolve them.
1.65 -
1.66 - self.register_submodules()
1.67 - self.loaded = True
1.68 -
1.69 - def register_submodules(self):
1.70 -
1.71 - "For package modules add submodule details."
1.72 -
1.73 - if splitext(split(self.filename)[1])[0] == "__init__":
1.74 - for subname in listdir(split(self.filename)[0]):
1.75 - name, ext = splitext(subname)
1.76 -
1.77 - # Add modules whose names are not already defined as globals.
1.78 -
1.79 - if ext == ("%spy" % extsep) and name != "__init__" and not self.get_global(name):
1.80 - module_name = self.get_global_path(name)
1.81 - top, submodule = self.get_module(module_name, True)
1.82 - self.set_module(name, submodule, hidden=True)
1.83 -
1.84 - def check_special(self):
1.85 -
1.86 - "Check special names."
1.87 -
1.88 - for name, value in self.special.items():
1.89 - if value.has_kind("<depends>"):
1.90 - self.find_imported_name(name, self.name)
1.91 - self.special[name] = self.get_object(value.get_origin())
1.92 -
1.93 - def check_names_used(self):
1.94 -
1.95 - "Check the names used by each function."
1.96 -
1.97 - for path in self.names_used.keys():
1.98 - self.check_names_used_for_path(path)
1.99 -
1.100 - def check_names_used_for_path(self, path):
1.101 -
1.102 - "Check the names used by the given 'path'."
1.103 + def complete(self):
1.104
1.105 - names = self.names_used.get(path)
1.106 - if not names:
1.107 - return
1.108 -
1.109 - in_function = self.function_locals.has_key(path)
1.110 -
1.111 - for name in names:
1.112 - if name in predefined_constants or in_function and name in self.function_locals[path]:
1.113 - continue
1.114 -
1.115 - # Resolve names that have been imported locally.
1.116 -
1.117 - self.find_imported_name(name, path)
1.118 -
1.119 - # Find local definitions.
1.120 -
1.121 - key = "%s.%s" % (path, name)
1.122 - ref = self.get_object(key)
1.123 - if ref:
1.124 - self.name_references[key] = ref.final() or key
1.125 - self.resolve_accesses(path, name, ref)
1.126 - continue
1.127 -
1.128 - # Resolve names that have been imported globally.
1.129 -
1.130 - self.find_imported_name(name, self.name)
1.131 -
1.132 - # Find global or built-in definitions.
1.133 -
1.134 - ref = self.get_global_or_builtin(name)
1.135 - objpath = ref and (ref.final() or ref.get_name())
1.136 - if objpath:
1.137 - self.name_references[key] = objpath
1.138 - self.resolve_accesses(path, name, ref)
1.139 - continue
1.140 -
1.141 - print >>sys.stderr, "Name not recognised: %s in %s" % (name, path)
1.142 - init_item(self.names_missing, path, set)
1.143 - self.names_missing[path].add(name)
1.144 -
1.145 - def resolve_members(self):
1.146 + "Complete the module inspection."
1.147
1.148 - "Resolve any members referring to deferred references."
1.149 -
1.150 - for name, ref in self.objects.items():
1.151 - if ref.has_kind("<depends>"):
1.152 - ref = self.get_object(ref.get_origin())
1.153 - ref = ref.alias(name)
1.154 - self.importer.objects[name] = self.objects[name] = ref
1.155 -
1.156 - def resolve_accesses(self, path, name, ref):
1.157 -
1.158 - """
1.159 - Resolve any unresolved accesses in the function at the given 'path'
1.160 - for the given 'name' corresponding to the indicated 'ref'. Note that
1.161 - this mechanism cannot resolve things like inherited methods because
1.162 - they are not recorded as program objects in their inherited locations.
1.163 - """
1.164 -
1.165 - attr_accesses = self.global_attr_accesses.get(path)
1.166 - all_attrnames = attr_accesses and attr_accesses.get(name)
1.167 -
1.168 - if not all_attrnames:
1.169 - return
1.170 -
1.171 - # Insist on constant accessors.
1.172 -
1.173 - if not ref.has_kind(["<class>", "<module>"]):
1.174 - return
1.175 -
1.176 - found_attrnames = set()
1.177 -
1.178 - for attrnames in all_attrnames:
1.179 -
1.180 - # Start with the resolved name, adding attributes.
1.181 -
1.182 - attrs = ref.get_path()
1.183 - remaining = attrnames.split(".")
1.184 - last_ref = ref
1.185 -
1.186 - # Add each component, testing for a constant object.
1.187 + # Resolve names not definitively mapped to objects.
1.188
1.189 - while remaining:
1.190 - attrname = remaining[0]
1.191 - attrs.append(attrname)
1.192 - del remaining[0]
1.193 -
1.194 - # Find any constant object reference.
1.195 -
1.196 - attr_ref = self.get_object(".".join(attrs))
1.197 -
1.198 - # Non-constant accessors terminate the traversal.
1.199 -
1.200 - if not attr_ref.has_kind(["<class>", "<module>", "<function>"]):
1.201 -
1.202 - # Provide the furthest constant accessor unless the final
1.203 - # access can be resolved.
1.204 -
1.205 - if remaining:
1.206 - remaining.insert(0, attrs.pop())
1.207 - else:
1.208 - last_ref = attr_ref
1.209 - break
1.210 -
1.211 - # A module may expose an attribute imported from a hidden
1.212 - # module.
1.213 -
1.214 - elif last_ref.has_kind("<module>"):
1.215 - module, leaf_module = self.get_module(last_ref.get_origin())
1.216 - self.find_imported_name(attrname, module.name, module)
1.217 -
1.218 - # Follow any reference to a constant object.
1.219 - # Where the given name refers to an object in another location,
1.220 - # switch to the other location in order to be able to test its
1.221 - # attributes.
1.222 -
1.223 - last_ref = attr_ref
1.224 - attrs = attr_ref.get_path()
1.225 -
1.226 - # Record a constant accessor only if an object was found
1.227 - # that is different from the namespace involved.
1.228 -
1.229 - if last_ref:
1.230 - objpath = ".".join(attrs)
1.231 - if objpath != path:
1.232 + self.resolve()
1.233
1.234 - # Establish a constant access.
1.235 -
1.236 - init_item(self.const_accesses, path, dict)
1.237 - self.const_accesses[path][(name, attrnames)] = (objpath, last_ref, ".".join(remaining))
1.238 -
1.239 - if len(attrs) > 1:
1.240 - found_attrnames.add(attrs[1])
1.241 -
1.242 - # Remove any usage records for the name.
1.243 -
1.244 - if found_attrnames:
1.245 -
1.246 - # NOTE: Should be only one name version.
1.247 -
1.248 - versions = []
1.249 - for version in self.attr_usage[path][name]:
1.250 - new_usage = set()
1.251 - for usage in version:
1.252 - if found_attrnames.intersection(usage):
1.253 - new_usage.add(tuple(set(usage).difference(found_attrnames)))
1.254 - else:
1.255 - new_usage.add(usage)
1.256 - versions.append(new_usage)
1.257 -
1.258 - self.attr_usage[path][name] = versions
1.259 -
1.260 - def resolve_initialisers(self):
1.261 -
1.262 - "Resolve initialiser values for names."
1.263 -
1.264 - # Get the initialisers in each namespace.
1.265 -
1.266 - for path, name_initialisers in self.name_initialisers.items():
1.267 - const_accesses = self.const_accesses.get(path)
1.268 -
1.269 - # Resolve values for each name in a scope.
1.270 + # Define the invocation requirements in each namespace.
1.271
1.272 - for name, values in name_initialisers.items():
1.273 - if path == self.name:
1.274 - assigned_path = name
1.275 - else:
1.276 - assigned_path = "%s.%s" % (path, name)
1.277 -
1.278 - initialised_names = {}
1.279 - aliased_names = {}
1.280 -
1.281 - for i, name_ref in enumerate(values):
1.282 -
1.283 - # Unwrap invocations.
1.284 -
1.285 - if isinstance(name_ref, InvocationRef):
1.286 - invocation = True
1.287 - name_ref = name_ref.name_ref
1.288 - else:
1.289 - invocation = False
1.290 -
1.291 - # Obtain a usable reference from names or constants.
1.292 -
1.293 - if isinstance(name_ref, ResolvedNameRef):
1.294 - if not name_ref.reference():
1.295 - continue
1.296 - ref = name_ref.reference()
1.297 -
1.298 - # Obtain a reference from instances.
1.299 -
1.300 - elif isinstance(name_ref, InstanceRef):
1.301 - if not name_ref.reference():
1.302 - continue
1.303 - ref = name_ref.reference()
1.304 -
1.305 - # Resolve accesses that employ constants.
1.306 -
1.307 - elif isinstance(name_ref, AccessRef):
1.308 - ref = None
1.309 -
1.310 - if const_accesses:
1.311 - resolved_access = const_accesses.get((name_ref.original_name, name_ref.attrnames))
1.312 - if resolved_access:
1.313 - objpath, ref, remaining_attrnames = resolved_access
1.314 - if remaining_attrnames:
1.315 - ref = None
1.316 + self.set_invocation_usage()
1.317
1.318 - # Accesses that do not employ constants cannot be resolved,
1.319 - # but they may be resolvable later.
1.320 -
1.321 - if not ref:
1.322 - if not invocation:
1.323 - aliased_names[i] = name_ref.original_name, name_ref.attrnames, name_ref.number
1.324 - continue
1.325 -
1.326 - # Attempt to resolve a plain name reference.
1.327 -
1.328 - elif isinstance(name_ref, LocalNameRef):
1.329 - key = "%s.%s" % (path, name_ref.name)
1.330 - origin = self.name_references.get(key)
1.331 -
1.332 - # Accesses that do not refer to known static objects
1.333 - # cannot be resolved, but they may be resolvable later.
1.334 -
1.335 - if not origin:
1.336 - if not invocation:
1.337 - aliased_names[i] = name_ref.name, None, name_ref.number
1.338 - continue
1.339 -
1.340 - ref = self.get_object(origin)
1.341 - if not ref:
1.342 - continue
1.343 -
1.344 - elif isinstance(name_ref, NameRef):
1.345 - key = "%s.%s" % (path, name_ref.name)
1.346 - origin = self.name_references.get(key)
1.347 - if not origin:
1.348 - continue
1.349 -
1.350 - ref = self.get_object(origin)
1.351 - if not ref:
1.352 - continue
1.353 -
1.354 - else:
1.355 - continue
1.356 -
1.357 - # Convert class invocations to instances.
1.358 + # Propagate to the importer information needed in subsequent activities.
1.359
1.360 - if invocation:
1.361 - ref = ref.has_kind("<class>") and ref.instance_of() or None
1.362 -
1.363 - if ref:
1.364 - initialised_names[i] = ref
1.365 -
1.366 - if initialised_names:
1.367 - self.initialised_names[assigned_path] = initialised_names
1.368 - if aliased_names:
1.369 - self.aliased_names[assigned_path] = aliased_names
1.370 -
1.371 - def resolve_literals(self):
1.372 -
1.373 - "Resolve constant value types."
1.374 -
1.375 - # Get the constants defined in each namespace.
1.376 -
1.377 - for path, constants in self.constants.items():
1.378 - for constant, n in constants.items():
1.379 - objpath = "%s.$c%d" % (path, n)
1.380 - _constant, value_type = self.constant_values[objpath]
1.381 - self.initialised_names[objpath] = {0 : Reference("<instance>", value_type)}
1.382 -
1.383 - # Get the literals defined in each namespace.
1.384 -
1.385 - for path, literals in self.literals.items():
1.386 - for n in range(0, literals):
1.387 - objpath = "%s.$C%d" % (path, n)
1.388 - value_type = self.literal_types[objpath]
1.389 - self.initialised_names[objpath] = {0 : Reference("<instance>", value_type)}
1.390 -
1.391 - def remove_redundant_accessors(self):
1.392 -
1.393 - "Remove now-redundant modifier and accessor information."
1.394 -
1.395 - for path, const_accesses in self.const_accesses.items():
1.396 - accesses = self.attr_accessors.get(path)
1.397 - modifiers = self.attr_access_modifiers.get(path)
1.398 - if not accesses:
1.399 - continue
1.400 - for access in const_accesses.keys():
1.401 - if accesses.has_key(access):
1.402 - del accesses[access]
1.403 - if modifiers and modifiers.has_key(access):
1.404 - del modifiers[access]
1.405 + self.propagate()
1.406
1.407 def set_invocation_usage(self):
1.408
1.409 @@ -761,8 +417,8 @@
1.410 base_class = self.get_class(base)
1.411
1.412 if not base_class:
1.413 - print >>sys.stderr, "In %s, class %s has unidentifiable base classes." % (
1.414 - path, n.name)
1.415 + print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % (
1.416 + path, n.name, base)
1.417 return
1.418 else:
1.419 bases.append(base_class)
1.420 @@ -804,73 +460,23 @@
1.421
1.422 path = self.get_namespace_path()
1.423
1.424 - modname, names = self.get_module_name(n)
1.425 -
1.426 - # Load the mentioned module.
1.427 + module_name, names = self.get_module_name(n)
1.428 + if module_name == self.name:
1.429 + raise InspectError("Cannot import from the current module.", path, n)
1.430
1.431 - top, module = self.get_module(modname, True)
1.432 - self.set_module(None, module, hidden=True)
1.433 -
1.434 - if not module:
1.435 - print >>sys.stderr, "In %s, from statement importing from %s failed." % (
1.436 - path, modname)
1.437 + self.importer.queue_module(module_name, self)
1.438
1.439 # Attempt to obtain the referenced objects.
1.440
1.441 for name, alias in n.names:
1.442 -
1.443 - # NOTE: Package submodules are not implicitly imported.
1.444 -
1.445 if name == "*":
1.446 - if module:
1.447 -
1.448 - # Warn about a circular import that probably doesn't find
1.449 - # the names.
1.450 -
1.451 - if not module.loaded:
1.452 - print >>sys.stderr, "In %s, from statement performs circular import %s of %s." % (
1.453 - path, modname)
1.454 -
1.455 - for name, value in module.get_globals().items():
1.456 - if name != "__name__":
1.457 - value = ResolvedNameRef(name, value)
1.458 - self.set_general_local(name, value)
1.459 - self.set_imported_name(name, modname)
1.460 - break
1.461 + raise InspectError("Only explicitly specified names can be imported from modules.", path, n)
1.462
1.463 # Explicit names.
1.464
1.465 - ref = self.import_name_from_module(name, modname, module, alias)
1.466 + ref = self.import_name_from_module(name, module_name)
1.467 value = ResolvedNameRef(alias or name, ref)
1.468 self.set_general_local(alias or name, value)
1.469 - self.set_imported_name(name, modname, alias)
1.470 -
1.471 - def import_name_from_module(self, name, modname, module, alias=None):
1.472 -
1.473 - """
1.474 - Import 'name' from the module having the given 'modname', with 'module'
1.475 - having been obtained for the module name, using 'alias' for the imported
1.476 - name in the current namespace.
1.477 - """
1.478 -
1.479 - path = self.get_namespace_path()
1.480 -
1.481 - if module and module.get_global(name):
1.482 - value = module.get_global(name)
1.483 -
1.484 - # Warn about an import that fails to provide a name, perhaps due
1.485 - # to a circular import.
1.486 -
1.487 - if not value:
1.488 - print >>sys.stderr, "In %s, from statement cannot import %s from %s%s." % (
1.489 - path, name, modname, not module.loaded and "(circular import)")
1.490 -
1.491 - return value
1.492 -
1.493 - # Record the name as a dependency.
1.494 -
1.495 - else:
1.496 - return Reference("<depends>", "%s.%s" % (modname, name))
1.497
1.498 def process_function_node(self, n, name):
1.499
1.500 @@ -1031,16 +637,13 @@
1.501 # Load the mentioned module.
1.502
1.503 for name, alias in n.names:
1.504 - module, leaf_module = self.get_module(name, alias)
1.505 + if name == self.name:
1.506 + raise InspectError("Cannot import the current module.", path, n)
1.507 + if not alias and len(n.names) > 1:
1.508 + raise InspectError("Imported modules must be aliased unless a simple module is imported.", path, n)
1.509
1.510 - if not module:
1.511 - print >>sys.stderr, "In %s, import statement importing from %s failed." % (
1.512 - path, name)
1.513 - if module and not module.loaded:
1.514 - print >>sys.stderr, "In %s, import statement performs circular import of %s." % (
1.515 - path, name)
1.516 -
1.517 - self.set_module(alias or name.split(".")[0], module, leaf_module)
1.518 + self.set_module(alias or name.split(".")[0], name)
1.519 + self.importer.queue_module(name, self)
1.520
1.521 def process_invocation_node(self, n):
1.522
1.523 @@ -1124,22 +727,12 @@
1.524
1.525 op = n.name[len("$op"):]
1.526
1.527 - # Access the operator module.
1.528 -
1.529 - top, module = self.get_module("operator", True)
1.530 - self.set_module(None, module, hidden=True)
1.531 -
1.532 - # Link the operation to the operator module definition in this
1.533 - # module.
1.534 -
1.535 - self.set_imported_name(op, "operator", n.name, self.name)
1.536 -
1.537 # Attempt to get a reference.
1.538
1.539 - ref = self.import_name_from_module(op, "operator", module)
1.540 - ref = self.get_object("operator.%s" % op) or ref
1.541 + ref = self.import_name_from_module(op, "operator")
1.542
1.543 # Record the imported name and provide the resolved name reference.
1.544 + # NOTE: Maybe use a different class.
1.545
1.546 value = ResolvedNameRef(n.name, ref)
1.547 self.set_special(n.name, value)
1.548 @@ -1491,29 +1084,15 @@
1.549 init_item(self.names_used, path, set)
1.550 self.names_used[path].add(name)
1.551
1.552 - def set_module(self, name, module, leaf_module=None, hidden=False):
1.553 + def set_module(self, name, module_name):
1.554
1.555 """
1.556 - Set a module in the current namespace using the given 'name' and
1.557 - corresponding 'module' object, with the 'leaf_module' being recorded
1.558 - if different. If 'hidden' is a true value, the modules are recorded as
1.559 - not necessarily being exposed by this module. This module is, however,
1.560 - recorded as accessing the given modules and is thus dependent on them.
1.561 + Set a module in the current namespace using the given 'name' associated
1.562 + with the corresponding 'module_name'.
1.563 """
1.564
1.565 if name:
1.566 - self.set_general_local(name, module and Reference("<module>", module.name) or None)
1.567 - if module:
1.568 - if hidden:
1.569 - self.imported_hidden.add(module)
1.570 - if leaf_module and leaf_module is not module:
1.571 - self.imported_hidden.add(leaf_module)
1.572 - else:
1.573 - self.imported.add(module)
1.574 - module.accessing_modules.add(self.name)
1.575 - if leaf_module and leaf_module is not module:
1.576 - self.imported.add(leaf_module)
1.577 - leaf_module.accessing_modules.add(self.name)
1.578 + self.set_general_local(name, Reference("<module>", module_name))
1.579
1.580 def set_definition(self, name, kind):
1.581
1.582 @@ -1688,16 +1267,16 @@
1.583
1.584 "Return a constant reference for the given type 'name' and 'value'."
1.585
1.586 - ref = self.get_literal_builtin(name)
1.587 + ref = self.get_builtin_class(name)
1.588 return self.get_constant_reference(ref, value)
1.589
1.590 def get_literal_instance(self, n, name):
1.591
1.592 "For node 'n', return a reference to an instance of 'name'."
1.593
1.594 - # Get a class reference.
1.595 + # Get a reference to the built-in class.
1.596
1.597 - ref = self.get_literal_builtin(name)
1.598 + ref = self.get_builtin_class(name)
1.599
1.600 # Obtain the details of the literal itself.
1.601 # An alias to the type is generated for sequences.
1.602 @@ -1711,31 +1290,6 @@
1.603 else:
1.604 return self.get_constant_reference(ref, n.value)
1.605
1.606 - def get_literal_builtin(self, name):
1.607 -
1.608 - "Return a reference for a built-in literal type of the given 'name'."
1.609 -
1.610 - ref = self.get_builtin(name)
1.611 - true_origin = "__builtins__.%s.%s" % (name, name)
1.612 - exposed_origin = "__builtins__.%s" % name
1.613 -
1.614 - # Obtain fully-imported built-in class references.
1.615 -
1.616 - if ref and ref.has_kind("<class>"):
1.617 - pass
1.618 -
1.619 - # Early-stage references need explicit references.
1.620 -
1.621 - elif ref:
1.622 - ref = Reference("<class>", true_origin)
1.623 -
1.624 - # Otherwise, the normal locations can be used.
1.625 -
1.626 - else:
1.627 - ref = Reference("<class>", true_origin, exposed_origin)
1.628 -
1.629 - return ref
1.630 -
1.631 # Functions and invocations.
1.632
1.633 def allocate_arguments(self, path, args):