1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017, 2018 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \ 31 Result, VariableRef 32 import compiler 33 import sys 34 35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 36 37 "A module inspector." 38 39 def __init__(self, name, importer): 40 41 "Initialise the module with basic details." 42 43 BasicModule.__init__(self, name, importer) 44 45 self.in_class = False 46 self.in_conditional = False 47 48 # Accesses to global attributes. 49 50 self.global_attr_accesses = {} 51 52 # Usage tracking. 53 54 self.trackers = [] 55 self.attr_accessor_branches = {} 56 57 def __repr__(self): 58 return "InspectedModule(%r, %r)" % (self.name, self.importer) 59 60 # Principal methods. 61 62 def parse(self, filename): 63 64 "Parse the file having the given 'filename'." 65 66 self.parse_file(filename) 67 68 # Inspect the module. 69 70 self.start_tracking_in_module() 71 72 # Detect and record imports and globals declared in the module. 73 74 self.process_structure(self.astnode) 75 76 # Set the class of the module after the definition has occurred. 77 78 ref = self.get_builtin("module") 79 self.set_name("__class__", ref) 80 self.set_name("__name__", self.get_constant("string", self.name).reference()) 81 self.set_name("__file__", self.get_constant("string", filename).reference()) 82 83 # Reserve a constant for the encoding. 84 85 if self.encoding: 86 self.get_constant("string", self.encoding) 87 88 # Get module-level attribute usage details. 89 90 self.stop_tracking_in_module() 91 92 # Collect external name references. 93 94 self.collect_names() 95 96 def complete(self): 97 98 "Complete the module inspection." 99 100 # Resolve names not definitively mapped to objects. 101 102 self.resolve() 103 104 # Propagate to the importer information needed in subsequent activities. 105 106 self.propagate() 107 108 # Accessory methods. 109 110 def collect_names(self): 111 112 "Collect the names used by each scope." 113 114 for path in self.names_used.keys(): 115 self.collect_names_for_path(path) 116 117 def collect_names_for_path(self, path): 118 119 """ 120 Collect the names used by the given 'path'. These are propagated to the 121 importer in advance of any dependency resolution. 122 """ 123 124 names = self.names_used.get(path) 125 if not names: 126 return 127 128 in_function = self.function_locals.has_key(path) 129 130 for name in names: 131 if in_function and name in self.function_locals[path]: 132 continue 133 134 key = "%s.%s" % (path, name) 135 136 # Find local definitions. 137 138 ref = self.get_resolved_object(key, True) 139 if ref: 140 self.set_name_reference(key, ref) 141 continue 142 143 # Find global. 144 145 ref = self.get_global(name) 146 if ref: 147 self.set_name_reference(key, ref) 148 continue 149 150 # Find presumed built-in definitions. 151 152 ref = self.get_builtin(name) 153 self.set_name_reference(key, ref) 154 155 def set_name_reference(self, path, ref): 156 157 "Map the given name 'path' to 'ref'." 158 159 self.importer.all_name_references[path] = self.name_references[path] = ref 160 161 # Module structure traversal. 162 163 def process_structure_node(self, n): 164 165 "Process the individual node 'n'." 166 167 path = self.get_namespace_path() 168 169 # Module global detection. 170 171 if isinstance(n, compiler.ast.Global): 172 self.process_global_node(n) 173 174 # Module import declarations. 175 176 elif isinstance(n, compiler.ast.From): 177 self.process_from_node(n) 178 179 elif isinstance(n, compiler.ast.Import): 180 self.process_import_node(n) 181 182 # Nodes using operator module functions. 183 184 elif isinstance(n, compiler.ast.Operator): 185 return self.process_operator_node(n) 186 187 elif isinstance(n, compiler.ast.AugAssign): 188 self.process_augassign_node(n) 189 190 elif isinstance(n, compiler.ast.Compare): 191 return self.process_compare_node(n) 192 193 elif isinstance(n, compiler.ast.Slice): 194 return self.process_slice_node(n) 195 196 elif isinstance(n, compiler.ast.Sliceobj): 197 return self.process_sliceobj_node(n) 198 199 elif isinstance(n, compiler.ast.Subscript): 200 return self.process_subscript_node(n) 201 202 # Namespaces within modules. 203 204 elif isinstance(n, compiler.ast.Class): 205 self.process_class_node(n) 206 207 elif isinstance(n, compiler.ast.Function): 208 self.process_function_node(n, n.name) 209 210 elif isinstance(n, compiler.ast.Lambda): 211 return self.process_lambda_node(n) 212 213 # Assignments. 214 215 elif isinstance(n, compiler.ast.Assign): 216 217 # Handle each assignment node. 218 219 for node in n.nodes: 220 self.process_assignment_node(node, n.expr) 221 222 # Assignments within non-Assign nodes. 223 224 elif isinstance(n, compiler.ast.AssName): 225 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 226 227 elif isinstance(n, compiler.ast.AssAttr): 228 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 229 230 # Accesses. 231 232 elif isinstance(n, compiler.ast.Getattr): 233 return self.process_attribute_access(n) 234 235 # Name recording for later testing. 236 237 elif isinstance(n, compiler.ast.Name): 238 return self.process_name_node(n) 239 240 # Conditional statement tracking. 241 242 elif isinstance(n, compiler.ast.For): 243 self.process_for_node(n) 244 245 elif isinstance(n, compiler.ast.While): 246 self.process_while_node(n) 247 248 elif isinstance(n, compiler.ast.If): 249 self.process_if_node(n) 250 251 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 252 return self.process_logical_node(n) 253 254 # Exception control-flow tracking. 255 256 elif isinstance(n, compiler.ast.TryExcept): 257 self.process_try_node(n) 258 259 elif isinstance(n, compiler.ast.TryFinally): 260 self.process_try_finally_node(n) 261 262 # Control-flow modification statements. 263 264 elif isinstance(n, compiler.ast.Break): 265 self.trackers[-1].suspend_broken_branch() 266 267 elif isinstance(n, compiler.ast.Continue): 268 self.trackers[-1].suspend_continuing_branch() 269 270 elif isinstance(n, compiler.ast.Raise): 271 self.process_structure(n) 272 self.trackers[-1].abandon_branch() 273 274 elif isinstance(n, compiler.ast.Return): 275 self.record_return_value(self.process_structure_node(n.value)) 276 self.trackers[-1].abandon_returning_branch() 277 278 # Print statements. 279 280 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 281 self.process_print_node(n) 282 283 # Invocations. 284 285 elif isinstance(n, compiler.ast.CallFunc): 286 return self.process_invocation_node(n) 287 288 # Constant usage. 289 290 elif isinstance(n, compiler.ast.Const): 291 return self.get_literal_instance(n) 292 293 elif isinstance(n, compiler.ast.Dict): 294 return self.get_literal_instance(n, "dict") 295 296 elif isinstance(n, compiler.ast.List): 297 return self.get_literal_instance(n, "list") 298 299 elif isinstance(n, compiler.ast.Tuple): 300 return self.get_literal_instance(n, "tuple") 301 302 # All other nodes are processed depth-first. 303 304 else: 305 self.process_structure(n) 306 307 # By default, no expression details are returned. 308 309 return None 310 311 # Specific node handling. 312 313 def process_assignment_node(self, n, expr): 314 315 "Process the individual node 'n' to be assigned the contents of 'expr'." 316 317 # Names and attributes are assigned the entire expression. 318 319 if isinstance(n, compiler.ast.AssName): 320 if n.name == "self": 321 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 322 323 name_ref = expr and self.process_structure_node(expr) 324 325 # Name assignments populate either function namespaces or the 326 # general namespace hierarchy. 327 328 self.assign_general_local(n.name, name_ref) 329 330 # Record usage of the name. 331 332 self.record_name(n.name) 333 334 elif isinstance(n, compiler.ast.AssAttr): 335 if expr: 336 expr = self.process_structure_node(expr) 337 338 in_assignment = self.in_assignment 339 self.in_assignment = True 340 self.process_attribute_access(n) 341 self.in_assignment = in_assignment 342 343 # Lists and tuples are matched against the expression and their 344 # items assigned to expression items. 345 346 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 347 self.process_assignment_node_items(n, expr) 348 349 # Slices and subscripts are permitted within assignment nodes. 350 351 elif isinstance(n, compiler.ast.Slice): 352 self.process_slice_node(n, expr) 353 354 elif isinstance(n, compiler.ast.Subscript): 355 self.process_subscript_node(n, expr) 356 357 def process_attribute_access(self, n): 358 359 "Process the given attribute access node 'n'." 360 361 # Obtain any completed chain and return the reference to it. 362 363 name_ref = self.process_attribute_chain(n) 364 365 if self.have_access_expression(n): 366 return name_ref 367 368 # Where the start of the chain of attributes has been reached, determine 369 # the complete access. 370 371 # Given a non-access node, this chain can be handled in its entirety, 372 # either being name-based and thus an access rooted on a name, or being 373 # based on some other node and thus an anonymous access of some kind. 374 375 path = self.get_namespace_path() 376 377 # Start with the the full attribute chain. 378 379 remaining = self.attrs 380 attrnames = ".".join(remaining) 381 382 # If the accessor cannot be identified, or where attributes 383 # remain in an attribute chain, record the anonymous accesses. 384 385 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 386 387 init_item(self.attr_accesses, path, set) 388 self.attr_accesses[path].add(attrnames) 389 390 self.record_access_details(None, attrnames, self.in_assignment, 391 self.in_invocation) 392 del self.attrs[0] 393 return 394 395 # Name-based accesses will handle the first attribute in a 396 # chain. 397 398 else: 399 attrname = remaining[0] 400 401 # Attribute assignments are used to identify instance attributes. 402 403 if isinstance(n, compiler.ast.AssAttr) and \ 404 self.in_class and self.in_function and n.expr.name == "self": 405 406 self.set_instance_attr(attrname) 407 408 # Record attribute usage using any name local to this namespace, 409 # if assigned in the namespace, or using an external name 410 # (presently just globals within classes). 411 412 name = self.get_name_for_tracking(name_ref.name, name_ref) 413 tracker = self.trackers[-1] 414 415 immediate_access = len(self.attrs) == 1 416 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 417 418 # Record global-based chains for subsequent resolution. 419 420 if name_ref.is_global_name(): 421 self.record_global_access_details(name, attrnames) 422 423 # Make sure the name is being tracked: global names will not 424 # already be initialised in a branch and must be added 425 # explicitly. 426 427 if not tracker.have_name(name): 428 tracker.assign_names([name]) 429 if self.in_function: 430 self.scope_globals[path].add(name) 431 432 # Record attribute usage in the tracker, and record the branch 433 # information for the access. 434 435 branches = tracker.use_attribute(name, attrname, 436 self.in_invocation is not None, assignment) 437 438 if not branches: 439 raise InspectError("Name %s is accessed using %s before an assignment." % ( 440 name, attrname), path, n) 441 442 self.record_branches_for_access(branches, name, attrnames) 443 access_number = self.record_access_details(name, attrnames, 444 self.in_assignment, self.in_invocation) 445 446 del self.attrs[0] 447 return AccessRef(name, attrnames, access_number) 448 449 def process_class_node(self, n): 450 451 "Process the given class node 'n'." 452 453 path = self.get_namespace_path() 454 455 # To avoid notions of class "versions" where the same definition 456 # might be parameterised with different state and be referenced 457 # elsewhere (as base classes, for example), classes in functions or 458 # conditions are forbidden. 459 460 if self.in_function or self.in_conditional: 461 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 462 path, n.name) 463 return 464 465 # Resolve base classes. 466 467 bases = [] 468 469 for base in n.bases: 470 base_class = self.get_class(base) 471 472 if not base_class: 473 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 474 path, n.name, base) 475 return 476 else: 477 bases.append(base_class) 478 479 # Detect conflicting definitions. Such definitions cause conflicts in 480 # the storage of namespace-related information. 481 482 class_name = self.get_object_path(n.name) 483 ref = self.get_object(class_name, defer=False) 484 485 if ref and ref.static(): 486 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 487 488 # Record bases for the class and retain the class name. 489 # Note that the function class does not inherit from the object class. 490 491 if not bases and class_name != "__builtins__.core.object" and \ 492 class_name != "__builtins__.core.function": 493 494 ref = self.get_object("__builtins__.object") 495 bases.append(ref) 496 497 self.importer.classes[class_name] = self.classes[class_name] = bases 498 self.importer.subclasses[class_name] = set() 499 self.scope_globals[class_name] = set() 500 501 # Set the definition before entering the namespace rather than 502 # afterwards because methods may reference it. In normal Python, 503 # a class is not accessible until the definition is complete, but 504 # methods can generally reference it since upon being called the 505 # class will already exist. 506 507 self.set_definition(n.name, "<class>") 508 509 in_class = self.in_class 510 self.in_class = class_name 511 self.set_instance_attr("__class__", Reference("<class>", class_name)) 512 self.enter_namespace(n.name) 513 514 # Do not provide the special instantiator attributes on the function 515 # class. Function instances provide these attributes. 516 517 if class_name != "__builtins__.core.function": 518 519 self.set_name("__fn__") # special instantiator attribute 520 self.set_name("__args__") # special instantiator attribute 521 522 # Provide leafname, parent and context attributes. 523 524 parent, leafname = class_name.rsplit(".", 1) 525 self.set_name("__name__", self.get_constant("string", leafname).reference()) 526 527 if class_name != "__builtins__.core.function": 528 self.set_name("__parent__") 529 530 self.process_structure_node(n.code) 531 self.exit_namespace() 532 self.in_class = in_class 533 534 def process_from_node(self, n): 535 536 "Process the given node 'n', importing from another module." 537 538 path = self.get_namespace_path() 539 540 module_name, names = self.get_module_name(n) 541 if module_name == self.name: 542 raise InspectError("Cannot import from the current module.", path, n) 543 544 self.queue_module(module_name) 545 546 # Attempt to obtain the referenced objects. 547 548 for name, alias in n.names: 549 if name == "*": 550 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 551 552 # Explicit names. 553 554 ref = self.import_name_from_module(name, module_name) 555 value = ResolvedNameRef(alias or name, ref) 556 self.set_general_local(alias or name, value) 557 558 def process_function_node(self, n, name): 559 560 """ 561 Process the given function or lambda node 'n' with the given 'name'. 562 """ 563 564 is_lambda = isinstance(n, compiler.ast.Lambda) 565 566 # Where a function is declared conditionally, use a separate name for 567 # the definition, and assign the definition to the stated name. 568 569 if (self.in_conditional or self.in_function) and not is_lambda: 570 original_name = name 571 name = self.get_lambda_name() 572 else: 573 original_name = None 574 575 # Detect conflicting definitions. Such definitions cause conflicts in 576 # the storage of namespace-related information. 577 578 function_name = self.get_object_path(name) 579 ref = self.get_object(function_name, defer=False) 580 581 if ref and ref.static(): 582 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 583 584 # Initialise argument and local records. 585 586 argnames = get_argnames(n.argnames) 587 is_method = self.in_class and not self.in_function 588 589 # Remove explicit "self" from method parameters. 590 591 if is_method and argnames and argnames[0] == "self": 592 del argnames[0] 593 594 # Convert .name entries in the parameters, provided this is a method. 595 596 l = [] 597 attr_initialisers = [] 598 599 for argname in argnames: 600 if argname[0] == ".": 601 if not is_method: 602 raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n) 603 604 argname = argname[1:] 605 attr_initialisers.append(argname) 606 607 l.append(argname) 608 609 argnames = l 610 611 # Copy and propagate the parameters. 612 613 self.importer.function_parameters[function_name] = \ 614 self.function_parameters[function_name] = argnames[:] 615 616 self.importer.function_attr_initialisers[function_name] = \ 617 self.function_attr_initialisers[function_name] = attr_initialisers 618 619 # Define all arguments/parameters in the local namespace. 620 621 locals = \ 622 self.importer.function_locals[function_name] = \ 623 self.function_locals[function_name] = {} 624 625 # Insert "self" into method locals. 626 627 if is_method: 628 argnames.insert(0, "self") 629 630 # Define "self" in terms of the class if in a method. 631 # This does not diminish the need for type-narrowing in the deducer. 632 633 if argnames: 634 if self.in_class and not self.in_function and argnames[0] == "self": 635 locals[argnames[0]] = Reference("<instance>", self.in_class) 636 else: 637 locals[argnames[0]] = Reference("<var>") 638 639 for argname in argnames[1:]: 640 locals[argname] = Reference("<var>") 641 642 globals = self.scope_globals[function_name] = set() 643 644 # Process the defaults. 645 646 defaults = self.importer.function_defaults[function_name] = \ 647 self.function_defaults[function_name] = [] 648 649 for argname, default in compiler.ast.get_defaults(n): 650 if default: 651 652 # Obtain any reference for the default. 653 654 name_ref = self.process_structure_node(default) 655 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 656 657 # Reset conditional tracking to focus on the function contents. 658 659 in_conditional = self.in_conditional 660 self.in_conditional = False 661 662 in_function = self.in_function 663 self.in_function = function_name 664 665 self.enter_namespace(name) 666 667 # Define a leafname attribute value for the function instance. 668 669 ref = self.get_builtin_class("string") 670 self.reserve_constant(function_name, name, ref.get_origin()) 671 672 # Track attribute usage within the namespace. 673 674 path = self.get_namespace_path() 675 self.start_tracking(locals) 676 677 # Establish attributes for .name entries, provided this is a method. 678 679 for argname in attr_initialisers: 680 self.process_assignment_node( 681 compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"), 682 compiler.ast.Name(argname)) 683 684 self.process_structure_node(n.code) 685 returns_value = self.stop_tracking() 686 687 # Record any null result. 688 689 is_initialiser = is_method and name == "__init__" 690 691 if not returns_value and not is_initialiser: 692 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 693 694 # Exit to the parent. 695 696 self.exit_namespace() 697 698 # Update flags. 699 700 self.in_function = in_function 701 self.in_conditional = in_conditional 702 703 # Define the function using the appropriate name. 704 705 self.set_definition(name, "<function>") 706 707 # Where a function is set conditionally, assign the name. 708 709 if original_name: 710 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 711 712 def process_global_node(self, n): 713 714 """ 715 Process the given "global" node 'n'. 716 """ 717 718 path = self.get_namespace_path() 719 720 if path != self.name: 721 self.scope_globals[path].update(n.names) 722 723 def process_if_node(self, n): 724 725 """ 726 Process the given "if" node 'n'. 727 """ 728 729 tracker = self.trackers[-1] 730 tracker.new_branchpoint() 731 732 for test, body in n.tests: 733 self.process_structure_node(test) 734 735 tracker.new_branch() 736 737 in_conditional = self.in_conditional 738 self.in_conditional = True 739 self.process_structure_node(body) 740 self.in_conditional = in_conditional 741 742 tracker.shelve_branch() 743 744 # Maintain a branch for the else clause. 745 746 tracker.new_branch() 747 if n.else_: 748 self.process_structure_node(n.else_) 749 tracker.shelve_branch() 750 751 tracker.merge_branches() 752 753 def process_import_node(self, n): 754 755 "Process the given import node 'n'." 756 757 path = self.get_namespace_path() 758 759 # Load the mentioned module. 760 761 for name, alias in n.names: 762 if name == self.name: 763 raise InspectError("Cannot import the current module.", path, n) 764 765 self.set_module(alias or name.split(".")[-1], name) 766 self.queue_module(name, True) 767 768 def process_invocation_node(self, n): 769 770 "Process the given invocation node 'n'." 771 772 path = self.get_namespace_path() 773 774 in_invocation = self.in_invocation 775 self.in_invocation = None 776 777 # Process the arguments. 778 779 keywords = set() 780 781 for arg in n.args: 782 self.process_structure_node(arg) 783 if isinstance(arg, compiler.ast.Keyword): 784 keywords.add(arg.name) 785 786 keywords = list(keywords) 787 keywords.sort() 788 789 # Communicate to the invocation target expression that it forms the 790 # target of an invocation, potentially affecting attribute accesses. 791 792 self.in_invocation = len(n.args), keywords 793 794 # Process the expression, obtaining any identified reference. 795 796 name_ref = self.process_structure_node(n.node) 797 self.in_invocation = in_invocation 798 799 # Detect class invocations. 800 801 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 802 return InstanceRef(name_ref.reference().instance_of()) 803 804 elif isinstance(name_ref, (NameRef, AccessRef)): 805 return InvocationRef(name_ref) 806 807 # Provide a general reference to indicate that something is produced 808 # by the invocation, useful for retaining assignment expression 809 # details. 810 811 return VariableRef() 812 813 def process_lambda_node(self, n): 814 815 "Process the given lambda node 'n'." 816 817 name = self.get_lambda_name() 818 self.process_function_node(n, name) 819 820 origin = self.get_object_path(name) 821 822 if self.function_defaults.get(origin): 823 return None 824 else: 825 return ResolvedNameRef(name, Reference("<function>", origin)) 826 827 def process_logical_node(self, n): 828 829 "Process the given operator node 'n'." 830 831 return self.process_operator_chain(n.nodes) 832 833 def process_name_node(self, n): 834 835 "Process the given name node 'n'." 836 837 path = self.get_namespace_path() 838 839 # Find predefined constant names before anything else. 840 841 if n.name in predefined_constants: 842 ref = self.get_builtin(n.name) 843 value = ResolvedNameRef(n.name, ref) 844 return value 845 846 # Special names that have already been identified. 847 848 if n.name.startswith("$"): 849 value = self.get_special(n.name) 850 if value: 851 return value 852 853 # Special case for operator functions introduced through code 854 # transformations. 855 856 if n.name.startswith("$op"): 857 858 # Obtain the location of the actual function defined in the operator 859 # package. 860 861 op = n.name[len("$op"):] 862 863 # Attempt to get a reference. 864 865 ref = self.import_name_from_module(op, "operator") 866 867 # Record the imported name and provide the resolved name reference. 868 869 value = ResolvedNameRef(n.name, ref) 870 self.set_special(n.name, value) 871 return value 872 873 # Special case for print operations. 874 875 elif n.name.startswith("$print"): 876 877 # Attempt to get a reference. 878 879 ref = self.get_builtin("print_") 880 881 # Record the imported name and provide the resolved name reference. 882 883 value = ResolvedNameRef(n.name, ref) 884 self.set_special(n.name, value) 885 return value 886 887 # Test for self usage, which is only allowed in methods. 888 889 if n.name == "self" and not (self.in_function and self.in_class): 890 raise InspectError("Use of self is only allowed in methods.", path, n) 891 892 # Record usage of the name. 893 894 self.record_name(n.name) 895 896 # Search for unknown names in non-function scopes immediately. 897 # Temporary names should not be re-used between scopes. 898 # External names in functions are resolved later. 899 900 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 901 902 if ref: 903 self.record_name_access(n.name, True) 904 return ResolvedNameRef(n.name, ref, is_global=True) 905 906 # Explicitly-declared global names. 907 908 elif self.in_function and n.name in self.scope_globals[path]: 909 self.record_name_access(n.name, True) 910 return NameRef(n.name, is_global=True) 911 912 # Examine other names. 913 914 else: 915 916 # Check local names. 917 918 access_number = self.record_name_access(n.name) 919 920 # Local name. 921 922 if access_number is not None: 923 return LocalNameRef(n.name, access_number) 924 925 # Possible global or built-in name. 926 927 else: 928 self.record_name_access(n.name, True) 929 return NameRef(n.name, is_global=True) 930 931 def record_name_access(self, name, is_global=False): 932 933 """ 934 Record an access involving 'name' if the name is being tracked, using 935 'is_global' to indicate whether the name is global. 936 """ 937 938 name = self.get_name_for_tracking(name, is_global=is_global) 939 branches = self.trackers[-1].tracking_name(name) 940 if branches: 941 self.record_branches_for_access(branches, name, None) 942 return self.record_access_details(name, None, self.in_assignment, 943 self.in_invocation) 944 return None 945 946 def process_operator_chain(self, nodes): 947 948 """ 949 Process the given chain of 'nodes', processing each node or item. 950 Each node starts a new conditional region, effectively making a deeply- 951 nested collection of if-like statements. 952 """ 953 954 results = [] 955 956 tracker = self.trackers[-1] 957 958 for item in nodes: 959 tracker.new_branchpoint() 960 tracker.new_branch() 961 result = self.process_structure_node(item) 962 if result: 963 results.append(result) 964 965 for item in nodes[:-1]: 966 tracker.shelve_branch() 967 tracker.new_branch() 968 tracker.shelve_branch() 969 tracker.merge_branches() 970 971 tracker.shelve_branch() 972 tracker.merge_branches() 973 974 return MultipleRef(results) 975 976 def process_try_node(self, n): 977 978 """ 979 Process the given "try...except" node 'n'. 980 """ 981 982 self.record_exception_handler() 983 984 tracker = self.trackers[-1] 985 tracker.new_branchpoint() 986 987 self.process_structure_node(n.body) 988 989 for name, var, handler in n.handlers: 990 if name is not None: 991 self.process_structure_node(name) 992 993 # Any abandoned branches from the body can now be resumed in a new 994 # branch. 995 996 tracker.resume_abandoned_branches() 997 998 # Establish the local for the handler. 999 1000 if var is not None: 1001 self.process_assignment_node(var, None) 1002 if handler is not None: 1003 self.process_structure_node(handler) 1004 1005 tracker.shelve_branch() 1006 1007 # The else clause maintains the usage from the body but without the 1008 # abandoned branches since they would never lead to the else clause 1009 # being executed. 1010 1011 if n.else_: 1012 tracker.new_branch() 1013 self.process_structure_node(n.else_) 1014 tracker.shelve_branch() 1015 1016 # Without an else clause, a null branch propagates the successful 1017 # outcome. 1018 1019 else: 1020 tracker.new_branch() 1021 tracker.shelve_branch() 1022 1023 tracker.merge_branches() 1024 1025 def process_try_finally_node(self, n): 1026 1027 """ 1028 Process the given "try...finally" node 'n'. 1029 """ 1030 1031 self.record_exception_handler() 1032 1033 tracker = self.trackers[-1] 1034 self.process_structure_node(n.body) 1035 1036 # Any abandoned branches from the body can now be resumed. 1037 1038 branches = tracker.resume_all_abandoned_branches() 1039 self.process_structure_node(n.final) 1040 1041 # At the end of the finally clause, abandoned branches are discarded. 1042 1043 tracker.restore_active_branches(branches) 1044 1045 def process_while_node(self, n): 1046 1047 "Process the given while node 'n'." 1048 1049 tracker = self.trackers[-1] 1050 tracker.new_branchpoint(loop_node=True) 1051 1052 # Evaluate any test or iterator outside the loop. 1053 1054 self.process_structure_node(n.test) 1055 1056 # Propagate attribute usage to branches. 1057 1058 tracker.new_branch(loop_node=True) 1059 1060 # Enter the loop. 1061 1062 in_conditional = self.in_conditional 1063 self.in_conditional = True 1064 self.process_structure_node(n.body) 1065 self.in_conditional = in_conditional 1066 1067 # Continuing branches are resumed before any test. 1068 1069 tracker.resume_continuing_branches() 1070 1071 # Evaluate any continuation test within the body. 1072 1073 self.process_structure_node(n.test) 1074 1075 tracker.shelve_branch(loop_node=True) 1076 1077 # Support the non-looping condition. 1078 1079 tracker.new_branch() 1080 tracker.shelve_branch() 1081 1082 tracker.merge_branches() 1083 1084 # Evaluate any else clause outside branches. 1085 1086 if n.else_: 1087 self.process_structure_node(n.else_) 1088 1089 # Connect broken branches to the code after any loop. 1090 1091 tracker.resume_broken_branches() 1092 1093 # Branch tracking methods. 1094 1095 def start_tracking(self, names): 1096 1097 """ 1098 Start tracking attribute usage for names in the current namespace, 1099 immediately registering the given 'names'. 1100 """ 1101 1102 path = self.get_namespace_path() 1103 parent = self.trackers[-1] 1104 tracker = BranchTracker() 1105 self.trackers.append(tracker) 1106 1107 # Record the given names established as new branches. 1108 1109 tracker.assign_names(names) 1110 1111 def assign_name(self, name, name_ref): 1112 1113 "Assign to 'name' the given 'name_ref' in the current namespace." 1114 1115 name = self.get_name_for_tracking(name) 1116 self.trackers[-1].assign_names([name], [name_ref]) 1117 1118 def stop_tracking(self): 1119 1120 """ 1121 Stop tracking attribute usage, recording computed usage for the current 1122 namespace. Indicate whether a value is always returned from the 1123 namespace. 1124 """ 1125 1126 path = self.get_namespace_path() 1127 tracker = self.trackers.pop() 1128 self.record_assignments_for_access(tracker) 1129 1130 self.attr_usage[path] = tracker.get_all_usage() 1131 self.name_initialisers[path] = tracker.get_all_values() 1132 1133 return tracker.returns_value() 1134 1135 def start_tracking_in_module(self): 1136 1137 "Start tracking attribute usage in the module." 1138 1139 tracker = BranchTracker() 1140 self.trackers.append(tracker) 1141 1142 def stop_tracking_in_module(self): 1143 1144 "Stop tracking attribute usage in the module." 1145 1146 tracker = self.trackers[0] 1147 self.record_assignments_for_access(tracker) 1148 self.attr_usage[self.name] = tracker.get_all_usage() 1149 self.name_initialisers[self.name] = tracker.get_all_values() 1150 1151 def record_assignments_for_access(self, tracker): 1152 1153 """ 1154 For the current path, use the given 'tracker' to record assignment 1155 version information for attribute accesses. 1156 """ 1157 1158 path = self.get_path_for_access() 1159 1160 if not self.attr_accessor_branches.has_key(path): 1161 return 1162 1163 init_item(self.attr_accessors, path, dict) 1164 attr_accessors = self.attr_accessors[path] 1165 1166 # Obtain the branches applying during each access. 1167 1168 for access, all_branches in self.attr_accessor_branches[path].items(): 1169 name, attrnames = access 1170 init_item(attr_accessors, access, list) 1171 1172 # Obtain the assignments applying to each branch. 1173 1174 for branches in all_branches: 1175 positions = tracker.get_assignment_positions_for_branches(name, branches) 1176 1177 # Detect missing name information. 1178 1179 if None in positions: 1180 globals = self.global_attr_accesses.get(path) 1181 accesses = globals and globals.get(name) 1182 if not accesses: 1183 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1184 self.get_namespace_path(), name) 1185 positions.remove(None) 1186 1187 attr_accessors[access].append(positions) 1188 1189 def record_branches_for_access(self, branches, name, attrnames): 1190 1191 """ 1192 Record the given 'branches' for an access involving the given 'name' and 1193 'attrnames'. 1194 """ 1195 1196 access = name, attrnames 1197 path = self.get_path_for_access() 1198 1199 init_item(self.attr_accessor_branches, path, dict) 1200 attr_accessor_branches = self.attr_accessor_branches[path] 1201 1202 init_item(attr_accessor_branches, access, list) 1203 attr_accessor_branches[access].append(branches) 1204 1205 def record_access_details(self, name, attrnames, assignment, invocation): 1206 1207 """ 1208 For the given 'name' and 'attrnames', record an access indicating 1209 whether an 'assignment' or an 'invocation' is occurring. 1210 1211 These details correspond to accesses otherwise recorded by the attribute 1212 accessor and attribute access dictionaries. 1213 """ 1214 1215 access = name, attrnames 1216 path = self.get_path_for_access() 1217 1218 init_item(self.attr_access_modifiers, path, dict) 1219 init_item(self.attr_access_modifiers[path], access, list) 1220 1221 access_number = len(self.attr_access_modifiers[path][access]) 1222 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1223 return access_number 1224 1225 def record_global_access_details(self, name, attrnames): 1226 1227 """ 1228 Record details of a global access via the given 'name' involving the 1229 indicated 'attrnames'. 1230 """ 1231 1232 path = self.get_namespace_path() 1233 1234 init_item(self.global_attr_accesses, path, dict) 1235 init_item(self.global_attr_accesses[path], name, set) 1236 self.global_attr_accesses[path][name].add(attrnames) 1237 1238 # Namespace modification. 1239 1240 def record_name(self, name): 1241 1242 "Record the use of 'name' in a namespace." 1243 1244 path = self.get_namespace_path() 1245 init_item(self.names_used, path, set) 1246 self.names_used[path].add(name) 1247 1248 def set_module(self, name, module_name): 1249 1250 """ 1251 Set a module in the current namespace using the given 'name' associated 1252 with the corresponding 'module_name'. 1253 """ 1254 1255 if name: 1256 self.set_general_local(name, Reference("<module>", module_name)) 1257 1258 def set_definition(self, name, kind): 1259 1260 """ 1261 Set the definition having the given 'name' and 'kind'. 1262 1263 Definitions are set in the static namespace hierarchy, but they can also 1264 be recorded for function locals. 1265 """ 1266 1267 if self.is_global(name): 1268 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1269 self.get_namespace_path(), name) 1270 1271 path = self.get_object_path(name) 1272 self.set_object(path, kind) 1273 1274 ref = self.get_object(path) 1275 if ref.get_kind() == "<var>": 1276 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1277 self.get_namespace_path(), name) 1278 1279 if not self.is_global(name) and self.in_function: 1280 self.set_function_local(name, ref) 1281 1282 def set_function_local(self, name, ref=None): 1283 1284 "Set the local with the given 'name' and optional 'ref'." 1285 1286 path = self.get_namespace_path() 1287 locals = self.function_locals[path] 1288 used = self.names_used.get(path) 1289 1290 if not locals.has_key(name) and used and name in used: 1291 raise InspectError("Name %s assigned locally but used previously." % name, path) 1292 1293 multiple = not ref or locals.has_key(name) and locals[name] != ref 1294 locals[name] = multiple and Reference("<var>") or ref 1295 1296 def assign_general_local(self, name, name_ref): 1297 1298 """ 1299 Set for 'name' the given 'name_ref', recording the name for attribute 1300 usage tracking. 1301 """ 1302 1303 self.set_general_local(name, name_ref) 1304 self.assign_name(name, name_ref) 1305 1306 def set_general_local(self, name, value=None): 1307 1308 """ 1309 Set the 'name' with optional 'value' in any kind of local namespace, 1310 where the 'value' should be a reference if specified. 1311 """ 1312 1313 init_value = self.get_initialising_value(value) 1314 1315 # Module global names. 1316 1317 if self.is_global(name): 1318 path = self.get_global_path(name) 1319 self.set_object(path, init_value) 1320 1321 # Function local names. 1322 1323 elif self.in_function: 1324 self.set_function_local(name, init_value) 1325 1326 # Other namespaces (classes). 1327 1328 else: 1329 self.set_name(name, init_value) 1330 1331 def set_name(self, name, ref=None): 1332 1333 "Attach the 'name' with optional 'ref' to the current namespace." 1334 1335 self.set_object(self.get_object_path(name), ref) 1336 1337 def set_instance_attr(self, name, ref=None): 1338 1339 """ 1340 Add an instance attribute of the given 'name' to the current class, 1341 using the optional 'ref'. 1342 """ 1343 1344 self._set_instance_attr(self.in_class, name, ref) 1345 1346 def _set_instance_attr(self, path, name, ref=None): 1347 1348 init_item(self.instance_attrs, path, set) 1349 self.instance_attrs[path].add(name) 1350 1351 if ref: 1352 init_item(self.instance_attr_constants, path, dict) 1353 self.instance_attr_constants[path][name] = ref 1354 1355 def get_initialising_value(self, value): 1356 1357 "Return a suitable initialiser reference for 'value'." 1358 1359 if isinstance(value, Result): 1360 return value.reference() 1361 else: 1362 return value 1363 1364 # Static, program-relative naming. 1365 1366 def find_name(self, name): 1367 1368 """ 1369 Return the qualified name for the given 'name' used in the current 1370 non-function namespace. 1371 """ 1372 1373 path = self.get_namespace_path() 1374 ref = None 1375 1376 if not self.in_function and name not in predefined_constants: 1377 if self.in_class: 1378 ref = self.get_object(self.get_object_path(name), False) 1379 if not ref: 1380 ref = self.get_global_or_builtin(name) 1381 1382 return ref 1383 1384 def get_class(self, node): 1385 1386 """ 1387 Use the given 'node' to obtain the identity of a class. Return a 1388 reference for the class. Unresolved dependencies are permitted and must 1389 be resolved later. 1390 """ 1391 1392 ref = self._get_class(node) 1393 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1394 1395 def _get_class(self, node): 1396 1397 """ 1398 Use the given 'node' to find a class definition. Return a reference to 1399 the class. 1400 """ 1401 1402 if isinstance(node, compiler.ast.Getattr): 1403 1404 # Obtain the identity of the access target. 1405 1406 ref = self._get_class(node.expr) 1407 1408 # Where the target is a class or module, obtain the identity of the 1409 # attribute. 1410 1411 if ref.has_kind(["<function>", "<var>"]): 1412 return None 1413 else: 1414 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1415 return self.get_object(attrname) 1416 1417 # Names can be module-level or built-in. 1418 1419 elif isinstance(node, compiler.ast.Name): 1420 1421 # Record usage of the name and attempt to identify it. 1422 1423 self.record_name(node.name) 1424 return self.find_name(node.name) 1425 else: 1426 return None 1427 1428 def get_constant(self, name, value): 1429 1430 "Return a constant reference for the given type 'name' and 'value'." 1431 1432 ref = self.get_builtin_class(name) 1433 return self.get_constant_reference(ref, value) 1434 1435 def get_literal_instance(self, n, name=None): 1436 1437 """ 1438 For node 'n', return a reference to an instance of 'name', or if 'name' 1439 is not specified, deduce the type from the value. 1440 """ 1441 1442 # Handle stray None constants (Sliceobj seems to produce them). 1443 1444 if name == "NoneType": 1445 return self.process_name_node(compiler.ast.Name("None")) 1446 1447 # Obtain the details of the literal itself. 1448 # An alias to the type is generated for sequences. 1449 1450 if name in ("dict", "list", "tuple"): 1451 ref = self.get_builtin_class(name) 1452 self.set_special_literal(name, ref) 1453 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1454 1455 # Constant values are independently recorded. 1456 1457 else: 1458 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1459 ref = self.get_builtin_class(typename) 1460 return self.get_constant_reference(ref, value, encoding) 1461 1462 # Special names. 1463 1464 def get_special(self, name): 1465 1466 "Return any stored value for the given special 'name'." 1467 1468 value = self.special.get(name) 1469 if value: 1470 ref, paths = value 1471 else: 1472 ref = None 1473 return ref 1474 1475 def set_special(self, name, value): 1476 1477 """ 1478 Set a special 'name' that merely tracks the use of an implicit object 1479 'value'. 1480 """ 1481 1482 if not self.special.has_key(name): 1483 paths = set() 1484 self.special[name] = value, paths 1485 else: 1486 _ref, paths = self.special[name] 1487 1488 paths.add(self.get_namespace_path()) 1489 1490 def set_special_literal(self, name, ref): 1491 1492 """ 1493 Set a special name for the literal type 'name' having type 'ref'. Such 1494 special names provide a way of referring to literal object types. 1495 """ 1496 1497 literal_name = "$L%s" % name 1498 value = ResolvedNameRef(literal_name, ref) 1499 self.set_special(literal_name, value) 1500 1501 # Exceptions. 1502 1503 def record_exception_handler(self): 1504 1505 "Record the current namespace as employing an exception handler." 1506 1507 self.exception_namespaces.add(self.get_namespace_path()) 1508 1509 # Return values. 1510 1511 def record_return_value(self, expr): 1512 1513 "Record the given return 'expr'." 1514 1515 path = self.get_namespace_path() 1516 l = init_item(self.return_values, path, list) 1517 l.append(expr) 1518 if not self.importer.all_return_values.has_key(path): 1519 self.importer.all_return_values[path] = l 1520 1521 # vim: tabstop=4 expandtab shiftwidth=4