1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007-2019, 2021 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 """ 21 22 from branching import BranchTracker 23 from common import CommonModule, get_argnames, init_item, \ 24 predefined_constants, privileged_attributes 25 from modules import BasicModule, CacheWritingModule, InspectionNaming 26 from errors import InspectError 27 from referencing import Reference 28 from resolving import NameResolving 29 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 30 LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \ 31 Result, VariableRef 32 import compiler 33 import sys 34 35 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 36 37 "A module inspector." 38 39 def __init__(self, name, importer): 40 41 "Initialise the module with basic details." 42 43 BasicModule.__init__(self, name, importer) 44 45 self.in_class = False 46 self.in_conditional = False 47 48 # Accesses to global attributes. 49 50 self.global_attr_accesses = {} 51 52 # Usage tracking. 53 54 self.trackers = [] 55 self.attr_accessor_branches = {} 56 57 def __repr__(self): 58 return "InspectedModule(%r, %r)" % (self.name, self.importer) 59 60 # Principal methods. 61 62 def parse(self, filename): 63 64 "Parse the file having the given 'filename'." 65 66 self.parse_file(filename) 67 68 # Inspect the module. 69 70 self.start_tracking_in_module() 71 72 # Detect and record imports and globals declared in the module. 73 74 self.process_structure(self.astnode) 75 76 # Set the class of the module after the definition has occurred. 77 78 ref = self.get_builtin("module") 79 self.set_name("__class__", ref) 80 self.set_name("__name__", self.get_constant("str", self.name).reference()) 81 self.set_name("__file__", self.get_constant("str", filename).reference()) 82 83 # Reserve a constant for the encoding. 84 85 if self.encoding: 86 self.get_constant("str", self.encoding) 87 88 # Get module-level attribute usage details. 89 90 self.stop_tracking_in_module() 91 92 # Collect external name references. 93 94 self.collect_names() 95 96 def complete(self): 97 98 "Complete the module inspection." 99 100 # Resolve names not definitively mapped to objects. 101 102 self.resolve() 103 104 # Propagate to the importer information needed in subsequent activities. 105 106 self.propagate() 107 108 # Accessory methods. 109 110 def collect_names(self): 111 112 "Collect the names used by each scope." 113 114 for path in self.names_used.keys(): 115 self.collect_names_for_path(path) 116 117 def collect_names_for_path(self, path): 118 119 """ 120 Collect the names used by the given 'path'. These are propagated to the 121 importer in advance of any dependency resolution. 122 """ 123 124 names = self.names_used.get(path) 125 if not names: 126 return 127 128 in_function = self.function_locals.has_key(path) 129 130 for name in names: 131 if in_function and name in self.function_locals[path]: 132 continue 133 134 key = "%s.%s" % (path, name) 135 136 # Find local definitions. 137 138 ref = self.get_resolved_object(key, True) 139 if ref: 140 self.set_name_reference(key, ref) 141 continue 142 143 # Find global. 144 145 ref = self.get_global(name) 146 if ref: 147 self.set_name_reference(key, ref) 148 continue 149 150 # Find presumed built-in definitions. 151 152 ref = self.get_builtin(name) 153 self.set_name_reference(key, ref) 154 155 def set_name_reference(self, path, ref): 156 157 "Map the given name 'path' to 'ref'." 158 159 self.importer.all_name_references[path] = self.name_references[path] = ref 160 161 # Module structure traversal. 162 163 def process_structure_node(self, n): 164 165 "Process the individual node 'n'." 166 167 path = self.get_namespace_path() 168 169 # Module global detection. 170 171 if isinstance(n, compiler.ast.Global): 172 self.process_global_node(n) 173 174 # Module import declarations. 175 176 elif isinstance(n, compiler.ast.From): 177 self.process_from_node(n) 178 179 elif isinstance(n, compiler.ast.Import): 180 self.process_import_node(n) 181 182 # Nodes using operator module functions. 183 184 elif isinstance(n, compiler.ast.Operator): 185 return self.process_operator_node(n) 186 187 elif isinstance(n, compiler.ast.AugAssign): 188 self.process_augassign_node(n) 189 190 elif isinstance(n, compiler.ast.Compare): 191 return self.process_compare_node(n) 192 193 elif isinstance(n, compiler.ast.Slice): 194 return self.process_slice_node(n) 195 196 elif isinstance(n, compiler.ast.Sliceobj): 197 return self.process_sliceobj_node(n) 198 199 elif isinstance(n, compiler.ast.Subscript): 200 return self.process_subscript_node(n) 201 202 # Namespaces within modules. 203 204 elif isinstance(n, compiler.ast.Class): 205 self.process_class_node(n) 206 207 elif isinstance(n, compiler.ast.Function): 208 self.process_function_node(n, n.name) 209 210 elif isinstance(n, compiler.ast.Lambda): 211 return self.process_lambda_node(n) 212 213 # Assignments. 214 215 elif isinstance(n, compiler.ast.Assign): 216 217 # Handle each assignment node. 218 219 for node in n.nodes: 220 self.process_assignment_node(node, n.expr) 221 222 # Assignments within non-Assign nodes. 223 224 elif isinstance(n, compiler.ast.AssName): 225 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 226 227 elif isinstance(n, compiler.ast.AssAttr): 228 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 229 230 # Accesses. 231 232 elif isinstance(n, compiler.ast.Getattr): 233 return self.process_attribute_access(n) 234 235 # Name recording for later testing. 236 237 elif isinstance(n, compiler.ast.Name): 238 return self.process_name_node(n) 239 240 # Conditional statement tracking. 241 242 elif isinstance(n, compiler.ast.For): 243 self.process_for_node(n) 244 245 elif isinstance(n, compiler.ast.While): 246 self.process_while_node(n) 247 248 elif isinstance(n, compiler.ast.If): 249 self.process_if_node(n) 250 251 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 252 return self.process_logical_node(n) 253 254 # Exception control-flow tracking. 255 256 elif isinstance(n, compiler.ast.TryExcept): 257 self.process_try_node(n) 258 259 elif isinstance(n, compiler.ast.TryFinally): 260 self.process_try_finally_node(n) 261 262 # Control-flow modification statements. 263 264 elif isinstance(n, compiler.ast.Break): 265 self.trackers[-1].suspend_broken_branch() 266 267 elif isinstance(n, compiler.ast.Continue): 268 self.trackers[-1].suspend_continuing_branch() 269 270 elif isinstance(n, compiler.ast.Raise): 271 self.process_structure(n) 272 self.trackers[-1].abandon_branch() 273 274 elif isinstance(n, compiler.ast.Return): 275 self.record_return_value(self.process_structure_node(n.value)) 276 self.trackers[-1].abandon_returning_branch() 277 278 # Print statements. 279 280 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 281 self.process_print_node(n) 282 283 # Invocations. 284 285 elif isinstance(n, compiler.ast.CallFunc): 286 return self.process_invocation_node(n) 287 288 # Constant usage. 289 290 elif isinstance(n, compiler.ast.Const): 291 return self.get_literal_instance(n) 292 293 elif isinstance(n, compiler.ast.Dict): 294 return self.get_literal_instance(n, "dict") 295 296 elif isinstance(n, compiler.ast.List): 297 return self.get_literal_instance(n, "list") 298 299 elif isinstance(n, compiler.ast.Tuple): 300 return self.get_literal_instance(n, "tuple") 301 302 # All other nodes are processed depth-first. 303 304 else: 305 self.process_structure(n) 306 307 # By default, no expression details are returned. 308 309 return None 310 311 # Specific node handling. 312 313 def process_assignment_node(self, n, expr): 314 315 "Process the individual node 'n' to be assigned the contents of 'expr'." 316 317 # Names and attributes are assigned the entire expression. 318 319 if isinstance(n, compiler.ast.AssName): 320 if n.name == "self": 321 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 322 323 name_ref = expr and self.process_structure_node(expr) 324 325 # Name assignments populate either function namespaces or the 326 # general namespace hierarchy. 327 328 self.assign_general_local(n.name, name_ref) 329 330 # Record usage of the name. 331 332 self.record_name(n.name) 333 334 elif isinstance(n, compiler.ast.AssAttr): 335 if expr: 336 expr = self.process_structure_node(expr) 337 338 in_assignment = self.in_assignment 339 self.in_assignment = True 340 self.process_attribute_access(n) 341 self.in_assignment = in_assignment 342 343 # Lists and tuples are matched against the expression and their 344 # items assigned to expression items. 345 346 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 347 self.process_assignment_node_items(n, expr) 348 349 # Slices and subscripts are permitted within assignment nodes. 350 351 elif isinstance(n, compiler.ast.Slice): 352 self.process_slice_node(n, expr) 353 354 elif isinstance(n, compiler.ast.Subscript): 355 self.process_subscript_node(n, expr) 356 357 def process_attribute_access(self, n): 358 359 "Process the given attribute access node 'n'." 360 361 path = self.get_namespace_path() 362 363 # Test for access to special privileged attributes. 364 365 if isinstance(n, compiler.ast.Getattr) and \ 366 n.attrname in privileged_attributes and not n.privileged: 367 368 raise InspectError("Attribute %s is accessed by an unprivileged operation." % 369 n.attrname, path, n) 370 371 # Obtain any completed chain and return the reference to it. 372 373 name_ref = self.process_attribute_chain(n) 374 375 if self.have_access_expression(n): 376 return name_ref 377 378 # Where the start of the chain of attributes has been reached, determine 379 # the complete access. 380 381 # Given a non-access node, this chain can be handled in its entirety, 382 # either being name-based and thus an access rooted on a name, or being 383 # based on some other node and thus an anonymous access of some kind. 384 385 # Start with the the full attribute chain. 386 387 remaining = self.attrs 388 attrnames = ".".join(remaining) 389 390 # If the accessor cannot be identified, or where attributes 391 # remain in an attribute chain, record the anonymous accesses. 392 393 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 394 395 init_item(self.attr_accesses, path, set) 396 self.attr_accesses[path].add(attrnames) 397 398 self.record_access_details(None, attrnames, self.in_assignment, 399 self.in_invocation) 400 del self.attrs[0] 401 return 402 403 # Name-based accesses will handle the first attribute in a 404 # chain. 405 406 else: 407 attrname = remaining[0] 408 409 # Attribute assignments are used to identify instance attributes. 410 411 if isinstance(n, compiler.ast.AssAttr) and \ 412 self.in_class and self.in_function and n.expr.name == "self": 413 414 self.set_instance_attr(attrname) 415 416 # Record attribute usage using any name local to this namespace, 417 # if assigned in the namespace, or using an external name 418 # (presently just globals within classes). 419 420 name = self.get_name_for_tracking(name_ref.name, name_ref) 421 tracker = self.trackers[-1] 422 423 immediate_access = len(self.attrs) == 1 424 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 425 426 # Record global-based chains for subsequent resolution. 427 428 if name_ref.is_global_name(): 429 self.record_global_access_details(name, attrnames) 430 431 # Make sure the name is being tracked: global names will not 432 # already be initialised in a branch and must be added 433 # explicitly. 434 435 if not tracker.have_name(name): 436 tracker.assign_names([name]) 437 if self.in_function: 438 self.scope_globals[path].add(name) 439 440 # Record attribute usage in the tracker, and record the branch 441 # information for the access. 442 443 branches = tracker.use_attribute(name, attrname, 444 self.in_invocation is not None, assignment) 445 446 if not branches: 447 raise InspectError("Name %s is accessed using %s before an assignment." % ( 448 name, attrname), path, n) 449 450 self.record_branches_for_access(branches, name, attrnames) 451 access_number = self.record_access_details(name, attrnames, 452 self.in_assignment, self.in_invocation) 453 454 del self.attrs[0] 455 return AccessRef(name, attrnames, access_number) 456 457 def process_class_node(self, n): 458 459 "Process the given class node 'n'." 460 461 path = self.get_namespace_path() 462 463 # To avoid notions of class "versions" where the same definition 464 # might be parameterised with different state and be referenced 465 # elsewhere (as base classes, for example), classes in functions or 466 # conditions are forbidden. 467 468 if self.in_function or self.in_conditional: 469 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 470 path, n.name) 471 return 472 473 # Resolve base classes. 474 475 bases = [] 476 477 for base in n.bases: 478 base_class = self.get_class(base) 479 480 if not base_class: 481 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 482 path, n.name, base) 483 return 484 else: 485 bases.append(base_class) 486 487 # Detect conflicting definitions. Such definitions cause conflicts in 488 # the storage of namespace-related information. 489 490 class_name = self.get_object_path(n.name) 491 ref = self.get_object(class_name, defer=False) 492 493 if ref and ref.static(): 494 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 495 496 # Record bases for the class and retain the class name. 497 # Note that the function class does not inherit from the object class. 498 499 if not bases and class_name != "__builtins__.core.object" and \ 500 class_name != "__builtins__.core.function": 501 502 ref = self.get_object("__builtins__.object") 503 bases.append(ref) 504 505 self.importer.classes[class_name] = self.classes[class_name] = bases 506 self.importer.subclasses[class_name] = set() 507 self.scope_globals[class_name] = set() 508 509 # Set the definition before entering the namespace rather than 510 # afterwards because methods may reference it. In normal Python, 511 # a class is not accessible until the definition is complete, but 512 # methods can generally reference it since upon being called the 513 # class will already exist. 514 515 self.set_definition(n.name, "<class>") 516 517 in_class = self.in_class 518 self.in_class = class_name 519 self.set_instance_attr("__class__", Reference("<class>", class_name)) 520 self.enter_namespace(n.name) 521 522 # Do not provide the special instantiator attributes on the function 523 # class. Function instances provide these attributes. 524 525 if class_name != "__builtins__.core.function": 526 527 self.set_name("__fn__") # special instantiator attribute 528 self.set_name("__args__") # special instantiator attribute 529 530 # Provide leafname, parent and context attributes. 531 532 parent, leafname = class_name.rsplit(".", 1) 533 self.set_name("__name__", self.get_constant("str", leafname).reference()) 534 535 if class_name != "__builtins__.core.function": 536 self.set_name("__parent__") 537 538 self.process_structure_node(n.code) 539 self.exit_namespace() 540 self.in_class = in_class 541 542 def process_from_node(self, n): 543 544 "Process the given node 'n', importing from another module." 545 546 path = self.get_namespace_path() 547 548 module_name, names = self.get_module_name(n) 549 if module_name == self.name: 550 raise InspectError("Cannot import from the current module.", path, n) 551 552 self.queue_module(module_name) 553 554 # Attempt to obtain the referenced objects. 555 556 for name, alias in n.names: 557 if name == "*": 558 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 559 560 # Explicit names. 561 562 ref = self.import_name_from_module(name, module_name) 563 value = ResolvedNameRef(alias or name, ref) 564 self.set_general_local(alias or name, value) 565 566 def process_function_node(self, n, name): 567 568 """ 569 Process the given function or lambda node 'n' with the given 'name'. 570 """ 571 572 is_lambda = isinstance(n, compiler.ast.Lambda) 573 574 # Where a function is declared conditionally, use a separate name for 575 # the definition, and assign the definition to the stated name. 576 577 if (self.in_conditional or self.in_function) and not is_lambda: 578 original_name = name 579 name = self.get_lambda_name() 580 else: 581 original_name = None 582 583 # Detect conflicting definitions. Such definitions cause conflicts in 584 # the storage of namespace-related information. 585 586 function_name = self.get_object_path(name) 587 ref = self.get_object(function_name, defer=False) 588 589 if ref and ref.static(): 590 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 591 592 # Initialise argument and local records. 593 594 argnames = get_argnames(n.argnames) 595 is_method = self.in_class and not self.in_function 596 597 # Remove explicit "self" from method parameters. 598 599 if is_method and argnames and argnames[0] == "self": 600 del argnames[0] 601 602 # Convert .name entries in the parameters, provided this is a method. 603 604 l = [] 605 attr_initialisers = [] 606 607 for argname in argnames: 608 if argname[0] == ".": 609 if not is_method: 610 raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n) 611 612 argname = argname[1:] 613 attr_initialisers.append(argname) 614 615 if argname in l: 616 raise InspectError("Argument name %s is repeated." % argname, function_name, n) 617 618 l.append(argname) 619 620 argnames = l 621 622 # Copy and propagate the parameters. 623 624 self.importer.function_parameters[function_name] = \ 625 self.function_parameters[function_name] = argnames[:] 626 627 self.importer.function_attr_initialisers[function_name] = \ 628 self.function_attr_initialisers[function_name] = attr_initialisers 629 630 # Define all arguments/parameters in the local namespace. 631 632 locals = \ 633 self.importer.function_locals[function_name] = \ 634 self.function_locals[function_name] = {} 635 636 # Insert "self" into method locals. 637 638 if is_method: 639 argnames.insert(0, "self") 640 641 # Define "self" in terms of the class if in a method. 642 # This does not diminish the need for type-narrowing in the deducer. 643 644 if argnames: 645 if self.in_class and not self.in_function and argnames[0] == "self": 646 locals[argnames[0]] = Reference("<instance>", self.in_class) 647 else: 648 locals[argnames[0]] = Reference("<var>") 649 650 for argname in argnames[1:]: 651 locals[argname] = Reference("<var>") 652 653 globals = self.scope_globals[function_name] = set() 654 655 # Process the defaults. 656 657 defaults = self.importer.function_defaults[function_name] = \ 658 self.function_defaults[function_name] = [] 659 660 for argname, default in compiler.ast.get_defaults(n): 661 if argname[0] == ".": 662 argname = argname[1:] 663 664 if default: 665 666 # Obtain any reference for the default. 667 668 name_ref = self.process_structure_node(default) 669 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 670 671 # Reset conditional tracking to focus on the function contents. 672 673 in_conditional = self.in_conditional 674 self.in_conditional = False 675 676 in_function = self.in_function 677 self.in_function = function_name 678 679 self.enter_namespace(name) 680 681 # Define a leafname attribute value for the function instance. 682 683 ref = self.get_builtin_class("str") 684 self.reserve_constant(function_name, name, ref.get_origin()) 685 686 # Track attribute usage within the namespace. 687 688 path = self.get_namespace_path() 689 self.start_tracking(locals) 690 691 # Establish attributes for .name entries, provided this is a method. 692 693 for argname in attr_initialisers: 694 self.process_assignment_node( 695 compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"), 696 compiler.ast.Name(argname)) 697 698 self.process_structure_node(n.code) 699 returns_value = self.stop_tracking() 700 701 # Record any null result. 702 703 is_initialiser = is_method and name == "__init__" 704 705 if not returns_value and not is_initialiser: 706 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 707 708 # Exit to the parent. 709 710 self.exit_namespace() 711 712 # Update flags. 713 714 self.in_function = in_function 715 self.in_conditional = in_conditional 716 717 # Define the function using the appropriate name. 718 719 self.set_definition(name, "<function>") 720 721 # Where a function is set conditionally, assign the name. 722 723 if original_name: 724 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 725 726 def process_global_node(self, n): 727 728 """ 729 Process the given "global" node 'n'. 730 """ 731 732 path = self.get_namespace_path() 733 734 if path != self.name: 735 self.scope_globals[path].update(n.names) 736 737 def process_if_node(self, n): 738 739 """ 740 Process the given "if" node 'n'. 741 """ 742 743 tracker = self.trackers[-1] 744 tracker.new_branchpoint() 745 746 for test, body in n.tests: 747 self.process_structure_node(test) 748 749 tracker.new_branch() 750 751 in_conditional = self.in_conditional 752 self.in_conditional = True 753 self.process_structure_node(body) 754 self.in_conditional = in_conditional 755 756 tracker.shelve_branch() 757 758 # Maintain a branch for the else clause. 759 760 tracker.new_branch() 761 if n.else_: 762 self.process_structure_node(n.else_) 763 tracker.shelve_branch() 764 765 tracker.merge_branches() 766 767 def process_import_node(self, n): 768 769 "Process the given import node 'n'." 770 771 path = self.get_namespace_path() 772 773 # Load the mentioned module. 774 775 for name, alias in n.names: 776 if name == self.name: 777 raise InspectError("Cannot import the current module.", path, n) 778 779 self.set_module(alias or name.split(".")[-1], name) 780 self.queue_module(name, True) 781 782 def process_invocation_node(self, n): 783 784 "Process the given invocation node 'n'." 785 786 path = self.get_namespace_path() 787 788 in_invocation = self.in_invocation 789 self.in_invocation = None 790 791 # Process the arguments. 792 793 keywords = set() 794 795 for arg in n.args: 796 self.process_structure_node(arg) 797 if isinstance(arg, compiler.ast.Keyword): 798 keywords.add(arg.name) 799 800 keywords = list(keywords) 801 keywords.sort() 802 803 # Communicate to the invocation target expression that it forms the 804 # target of an invocation, potentially affecting attribute accesses. 805 806 self.in_invocation = len(n.args), keywords 807 808 # Process the expression, obtaining any identified reference. 809 810 name_ref = self.process_structure_node(n.node) 811 self.in_invocation = in_invocation 812 813 # Detect class invocations. 814 815 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 816 return InstanceRef(name_ref.reference().instance_of()) 817 818 elif isinstance(name_ref, (NameRef, AccessRef)): 819 return InvocationRef(name_ref) 820 821 # Provide a general reference to indicate that something is produced 822 # by the invocation, useful for retaining assignment expression 823 # details. 824 825 return VariableRef() 826 827 def process_lambda_node(self, n): 828 829 "Process the given lambda node 'n'." 830 831 name = self.get_lambda_name() 832 self.process_function_node(n, name) 833 834 origin = self.get_object_path(name) 835 836 if self.function_defaults.get(origin): 837 return None 838 else: 839 return ResolvedNameRef(name, Reference("<function>", origin)) 840 841 def process_logical_node(self, n): 842 843 "Process the given operator node 'n'." 844 845 return self.process_operator_chain(n.nodes) 846 847 def process_name_node(self, n): 848 849 "Process the given name node 'n'." 850 851 path = self.get_namespace_path() 852 853 # Find predefined constant names before anything else. 854 855 if n.name in predefined_constants: 856 ref = self.get_builtin(n.name) 857 value = ResolvedNameRef(n.name, ref) 858 return value 859 860 # Special names that have already been identified. 861 862 if n.name.startswith("$"): 863 value = self.get_special(n.name) 864 if value: 865 return value 866 867 # Special case for operator functions introduced through code 868 # transformations. 869 870 if n.name.startswith("$op"): 871 872 # Obtain the location of the actual function defined in the operator 873 # package. 874 875 op = n.name[len("$op"):] 876 877 # Attempt to get a reference. 878 879 ref = self.import_name_from_module(op, "operator") 880 881 # Record the imported name and provide the resolved name reference. 882 883 value = ResolvedNameRef(n.name, ref) 884 self.set_special(n.name, value) 885 return value 886 887 # Special case for sequence length testing. 888 889 elif n.name.startswith("$seq"): 890 op = n.name[len("$seq"):] 891 ref = self.import_name_from_module(op, "__builtins__.sequence") 892 value = ResolvedNameRef(n.name, ref) 893 self.set_special(n.name, value) 894 return value 895 896 # Special case for print operations. 897 898 elif n.name.startswith("$print"): 899 900 # Attempt to get a reference. 901 902 ref = self.get_builtin("print_") 903 904 # Record the imported name and provide the resolved name reference. 905 906 value = ResolvedNameRef(n.name, ref) 907 self.set_special(n.name, value) 908 return value 909 910 # Special case for loops. 911 912 elif n.name == "$loop_exit": 913 914 # Attempt to get a reference. 915 916 ref = self.get_builtin("__loop_exit") 917 918 # Record the imported name and provide the resolved name reference. 919 920 value = ResolvedNameRef(n.name, ref) 921 self.set_special(n.name, value) 922 return value 923 924 # Test for self usage, which is only allowed in methods. 925 926 if n.name == "self" and not (self.in_function and self.in_class): 927 raise InspectError("Use of self is only allowed in methods.", path, n) 928 929 # Record usage of the name. 930 931 self.record_name(n.name) 932 933 # Search for unknown names in non-function scopes immediately. 934 # Temporary names should not be re-used between scopes. 935 # External names in functions are resolved later. 936 937 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 938 939 if ref: 940 self.record_name_access(n.name, True) 941 return ResolvedNameRef(n.name, ref, is_global=True) 942 943 # Explicitly-declared global names. 944 945 elif self.in_function and n.name in self.scope_globals[path]: 946 self.record_name_access(n.name, True) 947 return NameRef(n.name, is_global=True) 948 949 # Examine other names. 950 951 else: 952 953 # Check local names. 954 955 access_number = self.record_name_access(n.name) 956 957 # Local name. 958 959 if access_number is not None: 960 return LocalNameRef(n.name, access_number) 961 962 # Possible global or built-in name. 963 964 else: 965 self.record_name_access(n.name, True) 966 return NameRef(n.name, is_global=True) 967 968 def record_name_access(self, name, is_global=False): 969 970 """ 971 Record an access involving 'name' if the name is being tracked, using 972 'is_global' to indicate whether the name is global. 973 """ 974 975 name = self.get_name_for_tracking(name, is_global=is_global) 976 branches = self.trackers[-1].tracking_name(name) 977 if branches: 978 self.record_branches_for_access(branches, name, None) 979 return self.record_access_details(name, None, self.in_assignment, 980 self.in_invocation) 981 return None 982 983 def process_operator_chain(self, nodes): 984 985 """ 986 Process the given chain of 'nodes', processing each node or item. 987 Each node starts a new conditional region, effectively making a deeply- 988 nested collection of if-like statements. 989 """ 990 991 results = [] 992 993 tracker = self.trackers[-1] 994 995 for item in nodes: 996 tracker.new_branchpoint() 997 tracker.new_branch() 998 result = self.process_structure_node(item) 999 if result: 1000 results.append(result) 1001 1002 for item in nodes[:-1]: 1003 tracker.shelve_branch() 1004 tracker.new_branch() 1005 tracker.shelve_branch() 1006 tracker.merge_branches() 1007 1008 tracker.shelve_branch() 1009 tracker.merge_branches() 1010 1011 return MultipleRef(results) 1012 1013 def process_try_node(self, n): 1014 1015 """ 1016 Process the given "try...except" node 'n'. 1017 """ 1018 1019 self.record_exception_handler() 1020 1021 tracker = self.trackers[-1] 1022 tracker.new_branchpoint() 1023 1024 self.process_structure_node(n.body) 1025 1026 for name, var, handler in n.handlers: 1027 if name is not None: 1028 self.process_structure_node(name) 1029 1030 # Any abandoned branches from the body can now be resumed in a new 1031 # branch. 1032 1033 tracker.resume_abandoned_branches() 1034 1035 # Establish the local for the handler. 1036 1037 if var is not None: 1038 self.process_assignment_node(var, None) 1039 if handler is not None: 1040 self.process_structure_node(handler) 1041 1042 tracker.shelve_branch() 1043 1044 # The else clause maintains the usage from the body but without the 1045 # abandoned branches since they would never lead to the else clause 1046 # being executed. 1047 1048 if n.else_: 1049 tracker.new_branch() 1050 self.process_structure_node(n.else_) 1051 tracker.shelve_branch() 1052 1053 # Without an else clause, a null branch propagates the successful 1054 # outcome. 1055 1056 else: 1057 tracker.new_branch() 1058 tracker.shelve_branch() 1059 1060 tracker.merge_branches() 1061 1062 def process_try_finally_node(self, n): 1063 1064 """ 1065 Process the given "try...finally" node 'n'. 1066 """ 1067 1068 self.record_exception_handler() 1069 1070 tracker = self.trackers[-1] 1071 self.process_structure_node(n.body) 1072 1073 # Any abandoned branches from the body can now be resumed. 1074 1075 branches = tracker.resume_all_abandoned_branches() 1076 self.process_structure_node(n.final) 1077 1078 # At the end of the finally clause, abandoned branches are discarded. 1079 1080 tracker.restore_active_branches(branches) 1081 1082 def process_while_node(self, n): 1083 1084 "Process the given while node 'n'." 1085 1086 tracker = self.trackers[-1] 1087 tracker.new_branchpoint(loop_node=True) 1088 1089 # Evaluate any test or iterator outside the loop. 1090 1091 self.process_structure_node(n.test) 1092 1093 # Propagate attribute usage to branches. 1094 1095 tracker.new_branch(loop_node=True) 1096 1097 # Enter the loop. 1098 1099 in_conditional = self.in_conditional 1100 self.in_conditional = True 1101 self.process_structure_node(n.body) 1102 self.in_conditional = in_conditional 1103 1104 # Continuing branches are resumed before any test. 1105 1106 tracker.resume_continuing_branches() 1107 1108 # Evaluate any continuation test within the body. 1109 1110 self.process_structure_node(n.test) 1111 1112 tracker.shelve_branch(loop_node=True) 1113 1114 # Support the non-looping condition. 1115 1116 tracker.new_branch() 1117 tracker.shelve_branch() 1118 1119 tracker.merge_branches() 1120 1121 # Evaluate any else clause outside branches. 1122 1123 if n.else_: 1124 self.process_structure_node(n.else_) 1125 1126 # Connect broken branches to the code after any loop. 1127 1128 tracker.resume_broken_branches() 1129 1130 # Branch tracking methods. 1131 1132 def start_tracking(self, names): 1133 1134 """ 1135 Start tracking attribute usage for names in the current namespace, 1136 immediately registering the given 'names'. 1137 """ 1138 1139 path = self.get_namespace_path() 1140 parent = self.trackers[-1] 1141 tracker = BranchTracker() 1142 self.trackers.append(tracker) 1143 1144 # Record the given names established as new branches. 1145 1146 tracker.assign_names(names) 1147 1148 def assign_name(self, name, name_ref): 1149 1150 "Assign to 'name' the given 'name_ref' in the current namespace." 1151 1152 name = self.get_name_for_tracking(name) 1153 self.trackers[-1].assign_names([name], [name_ref]) 1154 1155 def stop_tracking(self): 1156 1157 """ 1158 Stop tracking attribute usage, recording computed usage for the current 1159 namespace. Indicate whether a value is always returned from the 1160 namespace. 1161 """ 1162 1163 path = self.get_namespace_path() 1164 tracker = self.trackers.pop() 1165 self.record_assignments_for_access(tracker) 1166 1167 self.attr_usage[path] = tracker.get_all_usage() 1168 self.name_initialisers[path] = tracker.get_all_values() 1169 1170 return tracker.returns_value() 1171 1172 def start_tracking_in_module(self): 1173 1174 "Start tracking attribute usage in the module." 1175 1176 tracker = BranchTracker() 1177 self.trackers.append(tracker) 1178 1179 def stop_tracking_in_module(self): 1180 1181 "Stop tracking attribute usage in the module." 1182 1183 tracker = self.trackers[0] 1184 self.record_assignments_for_access(tracker) 1185 self.attr_usage[self.name] = tracker.get_all_usage() 1186 self.name_initialisers[self.name] = tracker.get_all_values() 1187 1188 def record_assignments_for_access(self, tracker): 1189 1190 """ 1191 For the current path, use the given 'tracker' to record assignment 1192 version information for attribute accesses. 1193 """ 1194 1195 path = self.get_path_for_access() 1196 1197 if not self.attr_accessor_branches.has_key(path): 1198 return 1199 1200 init_item(self.attr_accessors, path, dict) 1201 attr_accessors = self.attr_accessors[path] 1202 1203 # Obtain the branches applying during each access. 1204 1205 for access, all_branches in self.attr_accessor_branches[path].items(): 1206 name, attrnames = access 1207 init_item(attr_accessors, access, list) 1208 1209 # Obtain the assignments applying to each branch. 1210 1211 for branches in all_branches: 1212 positions = tracker.get_assignment_positions_for_branches(name, branches) 1213 1214 # Detect missing name information. 1215 1216 if None in positions: 1217 globals = self.global_attr_accesses.get(path) 1218 accesses = globals and globals.get(name) 1219 if not accesses: 1220 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1221 self.get_namespace_path(), name) 1222 positions.remove(None) 1223 1224 attr_accessors[access].append(positions) 1225 1226 def record_branches_for_access(self, branches, name, attrnames): 1227 1228 """ 1229 Record the given 'branches' for an access involving the given 'name' and 1230 'attrnames'. 1231 """ 1232 1233 access = name, attrnames 1234 path = self.get_path_for_access() 1235 1236 init_item(self.attr_accessor_branches, path, dict) 1237 attr_accessor_branches = self.attr_accessor_branches[path] 1238 1239 init_item(attr_accessor_branches, access, list) 1240 attr_accessor_branches[access].append(branches) 1241 1242 def record_access_details(self, name, attrnames, assignment, invocation): 1243 1244 """ 1245 For the given 'name' and 'attrnames', record an access indicating 1246 whether an 'assignment' or an 'invocation' is occurring. 1247 1248 These details correspond to accesses otherwise recorded by the attribute 1249 accessor and attribute access dictionaries. 1250 """ 1251 1252 access = name, attrnames 1253 path = self.get_path_for_access() 1254 1255 init_item(self.attr_access_modifiers, path, dict) 1256 init_item(self.attr_access_modifiers[path], access, list) 1257 1258 access_number = len(self.attr_access_modifiers[path][access]) 1259 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1260 return access_number 1261 1262 def record_global_access_details(self, name, attrnames): 1263 1264 """ 1265 Record details of a global access via the given 'name' involving the 1266 indicated 'attrnames'. 1267 """ 1268 1269 path = self.get_namespace_path() 1270 1271 init_item(self.global_attr_accesses, path, dict) 1272 init_item(self.global_attr_accesses[path], name, set) 1273 self.global_attr_accesses[path][name].add(attrnames) 1274 1275 # Namespace modification. 1276 1277 def record_name(self, name): 1278 1279 "Record the use of 'name' in a namespace." 1280 1281 path = self.get_namespace_path() 1282 init_item(self.names_used, path, set) 1283 self.names_used[path].add(name) 1284 1285 def set_module(self, name, module_name): 1286 1287 """ 1288 Set a module in the current namespace using the given 'name' associated 1289 with the corresponding 'module_name'. 1290 """ 1291 1292 if name: 1293 self.set_general_local(name, Reference("<module>", module_name)) 1294 1295 def set_definition(self, name, kind): 1296 1297 """ 1298 Set the definition having the given 'name' and 'kind'. 1299 1300 Definitions are set in the static namespace hierarchy, but they can also 1301 be recorded for function locals. 1302 """ 1303 1304 if self.is_global(name): 1305 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1306 self.get_namespace_path(), name) 1307 1308 path = self.get_object_path(name) 1309 self.set_object(path, kind) 1310 1311 ref = self.get_object(path) 1312 if ref.get_kind() == "<var>": 1313 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1314 self.get_namespace_path(), name) 1315 1316 if not self.is_global(name) and self.in_function: 1317 self.set_function_local(name, ref) 1318 1319 def set_function_local(self, name, ref=None): 1320 1321 "Set the local with the given 'name' and optional 'ref'." 1322 1323 path = self.get_namespace_path() 1324 locals = self.function_locals[path] 1325 used = self.names_used.get(path) 1326 1327 if not locals.has_key(name) and used and name in used: 1328 raise InspectError("Name %s assigned locally but used previously." % name, path) 1329 1330 multiple = not ref or locals.has_key(name) and locals[name] != ref 1331 locals[name] = multiple and Reference("<var>") or ref 1332 1333 def assign_general_local(self, name, name_ref): 1334 1335 """ 1336 Set for 'name' the given 'name_ref', recording the name for attribute 1337 usage tracking. 1338 """ 1339 1340 self.set_general_local(name, name_ref) 1341 self.assign_name(name, name_ref) 1342 1343 def set_general_local(self, name, value=None): 1344 1345 """ 1346 Set the 'name' with optional 'value' in any kind of local namespace, 1347 where the 'value' should be a reference if specified. 1348 """ 1349 1350 init_value = self.get_initialising_value(value) 1351 1352 # Module global names. 1353 1354 if self.is_global(name): 1355 path = self.get_global_path(name) 1356 self.set_object(path, init_value) 1357 1358 # Function local names. 1359 1360 elif self.in_function: 1361 self.set_function_local(name, init_value) 1362 1363 # Other namespaces (classes). 1364 1365 else: 1366 self.set_name(name, init_value) 1367 1368 def set_name(self, name, ref=None): 1369 1370 "Attach the 'name' with optional 'ref' to the current namespace." 1371 1372 self.set_object(self.get_object_path(name), ref) 1373 1374 def set_instance_attr(self, name, ref=None): 1375 1376 """ 1377 Add an instance attribute of the given 'name' to the current class, 1378 using the optional 'ref'. 1379 """ 1380 1381 self._set_instance_attr(self.in_class, name, ref) 1382 1383 def _set_instance_attr(self, path, name, ref=None): 1384 1385 init_item(self.instance_attrs, path, set) 1386 self.instance_attrs[path].add(name) 1387 1388 if ref: 1389 init_item(self.instance_attr_constants, path, dict) 1390 self.instance_attr_constants[path][name] = ref 1391 1392 def get_initialising_value(self, value): 1393 1394 "Return a suitable initialiser reference for 'value'." 1395 1396 if isinstance(value, Result): 1397 return value.reference() 1398 else: 1399 return value 1400 1401 # Static, program-relative naming. 1402 1403 def find_name(self, name): 1404 1405 """ 1406 Return the qualified name for the given 'name' used in the current 1407 non-function namespace. 1408 """ 1409 1410 path = self.get_namespace_path() 1411 ref = None 1412 1413 if not self.in_function and name not in predefined_constants: 1414 if self.in_class: 1415 ref = self.get_object(self.get_object_path(name), False) 1416 if not ref: 1417 ref = self.get_global_or_builtin(name) 1418 1419 return ref 1420 1421 def get_class(self, node): 1422 1423 """ 1424 Use the given 'node' to obtain the identity of a class. Return a 1425 reference for the class. Unresolved dependencies are permitted and must 1426 be resolved later. 1427 """ 1428 1429 ref = self._get_class(node) 1430 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1431 1432 def _get_class(self, node): 1433 1434 """ 1435 Use the given 'node' to find a class definition. Return a reference to 1436 the class. 1437 """ 1438 1439 if isinstance(node, compiler.ast.Getattr): 1440 1441 # Obtain the identity of the access target. 1442 1443 ref = self._get_class(node.expr) 1444 1445 # Where the target is a class or module, obtain the identity of the 1446 # attribute. 1447 1448 if ref.has_kind(["<function>", "<var>"]): 1449 return None 1450 else: 1451 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1452 return self.get_object(attrname) 1453 1454 # Names can be module-level or built-in. 1455 1456 elif isinstance(node, compiler.ast.Name): 1457 1458 # Record usage of the name and attempt to identify it. 1459 1460 self.record_name(node.name) 1461 return self.find_name(node.name) 1462 else: 1463 return None 1464 1465 def get_constant(self, name, value): 1466 1467 "Return a constant reference for the given type 'name' and 'value'." 1468 1469 ref = self.get_builtin_class(name) 1470 return self.get_constant_reference(ref, value) 1471 1472 def get_literal_instance(self, n, name=None): 1473 1474 """ 1475 For node 'n', return a reference to an instance of 'name', or if 'name' 1476 is not specified, deduce the type from the value. 1477 """ 1478 1479 # Handle stray None constants (Sliceobj seems to produce them). 1480 1481 if name == "NoneType": 1482 return self.process_name_node(compiler.ast.Name("None")) 1483 1484 # Obtain the details of the literal itself. 1485 # An alias to the type is generated for sequences. 1486 1487 if name in ("dict", "list", "tuple"): 1488 ref = self.get_builtin_class(name) 1489 self.set_special_literal(name, ref) 1490 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1491 1492 # Constant values are independently recorded. 1493 1494 else: 1495 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1496 ref = self.get_builtin_class(typename) 1497 return self.get_constant_reference(ref, value, encoding) 1498 1499 # Special names. 1500 1501 def get_special(self, name): 1502 1503 "Return any stored value for the given special 'name'." 1504 1505 value = self.special.get(name) 1506 if value: 1507 ref, paths = value 1508 else: 1509 ref = None 1510 return ref 1511 1512 def set_special(self, name, value): 1513 1514 """ 1515 Set a special 'name' that merely tracks the use of an implicit object 1516 'value'. 1517 """ 1518 1519 if not self.special.has_key(name): 1520 paths = set() 1521 self.special[name] = value, paths 1522 else: 1523 _ref, paths = self.special[name] 1524 1525 paths.add(self.get_namespace_path()) 1526 1527 def set_special_literal(self, name, ref): 1528 1529 """ 1530 Set a special name for the literal type 'name' having type 'ref'. Such 1531 special names provide a way of referring to literal object types. 1532 """ 1533 1534 literal_name = "$L%s" % name 1535 value = ResolvedNameRef(literal_name, ref) 1536 self.set_special(literal_name, value) 1537 1538 # Exceptions. 1539 1540 def record_exception_handler(self): 1541 1542 "Record the current namespace as employing an exception handler." 1543 1544 self.exception_namespaces.add(self.get_namespace_path()) 1545 1546 # Return values. 1547 1548 def record_return_value(self, expr): 1549 1550 "Record the given return 'expr'." 1551 1552 path = self.get_namespace_path() 1553 l = init_item(self.return_values, path, list) 1554 l.append(expr) 1555 if not self.importer.all_return_values.has_key(path): 1556 self.importer.all_return_values[path] = l 1557 1558 # vim: tabstop=4 expandtab shiftwidth=4