1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 7 2018, 2019 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import CommonModule, get_argnames, init_item, \ 25 predefined_constants, privileged_attributes 26 from modules import BasicModule, CacheWritingModule, InspectionNaming 27 from errors import InspectError 28 from referencing import Reference 29 from resolving import NameResolving 30 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 31 LocalNameRef, MultipleRef, NameRef, ResolvedNameRef, \ 32 Result, VariableRef 33 import compiler 34 import sys 35 36 class InspectedModule(BasicModule, CacheWritingModule, NameResolving, InspectionNaming): 37 38 "A module inspector." 39 40 def __init__(self, name, importer): 41 42 "Initialise the module with basic details." 43 44 BasicModule.__init__(self, name, importer) 45 46 self.in_class = False 47 self.in_conditional = False 48 49 # Accesses to global attributes. 50 51 self.global_attr_accesses = {} 52 53 # Usage tracking. 54 55 self.trackers = [] 56 self.attr_accessor_branches = {} 57 58 def __repr__(self): 59 return "InspectedModule(%r, %r)" % (self.name, self.importer) 60 61 # Principal methods. 62 63 def parse(self, filename): 64 65 "Parse the file having the given 'filename'." 66 67 self.parse_file(filename) 68 69 # Inspect the module. 70 71 self.start_tracking_in_module() 72 73 # Detect and record imports and globals declared in the module. 74 75 self.process_structure(self.astnode) 76 77 # Set the class of the module after the definition has occurred. 78 79 ref = self.get_builtin("module") 80 self.set_name("__class__", ref) 81 self.set_name("__name__", self.get_constant("string", self.name).reference()) 82 self.set_name("__file__", self.get_constant("string", filename).reference()) 83 84 # Reserve a constant for the encoding. 85 86 if self.encoding: 87 self.get_constant("string", self.encoding) 88 89 # Get module-level attribute usage details. 90 91 self.stop_tracking_in_module() 92 93 # Collect external name references. 94 95 self.collect_names() 96 97 def complete(self): 98 99 "Complete the module inspection." 100 101 # Resolve names not definitively mapped to objects. 102 103 self.resolve() 104 105 # Propagate to the importer information needed in subsequent activities. 106 107 self.propagate() 108 109 # Accessory methods. 110 111 def collect_names(self): 112 113 "Collect the names used by each scope." 114 115 for path in self.names_used.keys(): 116 self.collect_names_for_path(path) 117 118 def collect_names_for_path(self, path): 119 120 """ 121 Collect the names used by the given 'path'. These are propagated to the 122 importer in advance of any dependency resolution. 123 """ 124 125 names = self.names_used.get(path) 126 if not names: 127 return 128 129 in_function = self.function_locals.has_key(path) 130 131 for name in names: 132 if in_function and name in self.function_locals[path]: 133 continue 134 135 key = "%s.%s" % (path, name) 136 137 # Find local definitions. 138 139 ref = self.get_resolved_object(key, True) 140 if ref: 141 self.set_name_reference(key, ref) 142 continue 143 144 # Find global. 145 146 ref = self.get_global(name) 147 if ref: 148 self.set_name_reference(key, ref) 149 continue 150 151 # Find presumed built-in definitions. 152 153 ref = self.get_builtin(name) 154 self.set_name_reference(key, ref) 155 156 def set_name_reference(self, path, ref): 157 158 "Map the given name 'path' to 'ref'." 159 160 self.importer.all_name_references[path] = self.name_references[path] = ref 161 162 # Module structure traversal. 163 164 def process_structure_node(self, n): 165 166 "Process the individual node 'n'." 167 168 path = self.get_namespace_path() 169 170 # Module global detection. 171 172 if isinstance(n, compiler.ast.Global): 173 self.process_global_node(n) 174 175 # Module import declarations. 176 177 elif isinstance(n, compiler.ast.From): 178 self.process_from_node(n) 179 180 elif isinstance(n, compiler.ast.Import): 181 self.process_import_node(n) 182 183 # Nodes using operator module functions. 184 185 elif isinstance(n, compiler.ast.Operator): 186 return self.process_operator_node(n) 187 188 elif isinstance(n, compiler.ast.AugAssign): 189 self.process_augassign_node(n) 190 191 elif isinstance(n, compiler.ast.Compare): 192 return self.process_compare_node(n) 193 194 elif isinstance(n, compiler.ast.Slice): 195 return self.process_slice_node(n) 196 197 elif isinstance(n, compiler.ast.Sliceobj): 198 return self.process_sliceobj_node(n) 199 200 elif isinstance(n, compiler.ast.Subscript): 201 return self.process_subscript_node(n) 202 203 # Namespaces within modules. 204 205 elif isinstance(n, compiler.ast.Class): 206 self.process_class_node(n) 207 208 elif isinstance(n, compiler.ast.Function): 209 self.process_function_node(n, n.name) 210 211 elif isinstance(n, compiler.ast.Lambda): 212 return self.process_lambda_node(n) 213 214 # Assignments. 215 216 elif isinstance(n, compiler.ast.Assign): 217 218 # Handle each assignment node. 219 220 for node in n.nodes: 221 self.process_assignment_node(node, n.expr) 222 223 # Assignments within non-Assign nodes. 224 225 elif isinstance(n, compiler.ast.AssName): 226 raise InspectError("Name assignment appearing outside assignment statement.", path, n) 227 228 elif isinstance(n, compiler.ast.AssAttr): 229 raise InspectError("Attribute assignment appearing outside assignment statement.", path, n) 230 231 # Accesses. 232 233 elif isinstance(n, compiler.ast.Getattr): 234 return self.process_attribute_access(n) 235 236 # Name recording for later testing. 237 238 elif isinstance(n, compiler.ast.Name): 239 return self.process_name_node(n) 240 241 # Conditional statement tracking. 242 243 elif isinstance(n, compiler.ast.For): 244 self.process_for_node(n) 245 246 elif isinstance(n, compiler.ast.While): 247 self.process_while_node(n) 248 249 elif isinstance(n, compiler.ast.If): 250 self.process_if_node(n) 251 252 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 253 return self.process_logical_node(n) 254 255 # Exception control-flow tracking. 256 257 elif isinstance(n, compiler.ast.TryExcept): 258 self.process_try_node(n) 259 260 elif isinstance(n, compiler.ast.TryFinally): 261 self.process_try_finally_node(n) 262 263 # Control-flow modification statements. 264 265 elif isinstance(n, compiler.ast.Break): 266 self.trackers[-1].suspend_broken_branch() 267 268 elif isinstance(n, compiler.ast.Continue): 269 self.trackers[-1].suspend_continuing_branch() 270 271 elif isinstance(n, compiler.ast.Raise): 272 self.process_structure(n) 273 self.trackers[-1].abandon_branch() 274 275 elif isinstance(n, compiler.ast.Return): 276 self.record_return_value(self.process_structure_node(n.value)) 277 self.trackers[-1].abandon_returning_branch() 278 279 # Print statements. 280 281 elif isinstance(n, (compiler.ast.Print, compiler.ast.Printnl)): 282 self.process_print_node(n) 283 284 # Invocations. 285 286 elif isinstance(n, compiler.ast.CallFunc): 287 return self.process_invocation_node(n) 288 289 # Constant usage. 290 291 elif isinstance(n, compiler.ast.Const): 292 return self.get_literal_instance(n) 293 294 elif isinstance(n, compiler.ast.Dict): 295 return self.get_literal_instance(n, "dict") 296 297 elif isinstance(n, compiler.ast.List): 298 return self.get_literal_instance(n, "list") 299 300 elif isinstance(n, compiler.ast.Tuple): 301 return self.get_literal_instance(n, "tuple") 302 303 # All other nodes are processed depth-first. 304 305 else: 306 self.process_structure(n) 307 308 # By default, no expression details are returned. 309 310 return None 311 312 # Specific node handling. 313 314 def process_assignment_node(self, n, expr): 315 316 "Process the individual node 'n' to be assigned the contents of 'expr'." 317 318 # Names and attributes are assigned the entire expression. 319 320 if isinstance(n, compiler.ast.AssName): 321 if n.name == "self": 322 raise InspectError("Redefinition of self is not allowed.", self.get_namespace_path(), n) 323 324 name_ref = expr and self.process_structure_node(expr) 325 326 # Name assignments populate either function namespaces or the 327 # general namespace hierarchy. 328 329 self.assign_general_local(n.name, name_ref) 330 331 # Record usage of the name. 332 333 self.record_name(n.name) 334 335 elif isinstance(n, compiler.ast.AssAttr): 336 if expr: 337 expr = self.process_structure_node(expr) 338 339 in_assignment = self.in_assignment 340 self.in_assignment = True 341 self.process_attribute_access(n) 342 self.in_assignment = in_assignment 343 344 # Lists and tuples are matched against the expression and their 345 # items assigned to expression items. 346 347 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 348 self.process_assignment_node_items(n, expr) 349 350 # Slices and subscripts are permitted within assignment nodes. 351 352 elif isinstance(n, compiler.ast.Slice): 353 self.process_slice_node(n, expr) 354 355 elif isinstance(n, compiler.ast.Subscript): 356 self.process_subscript_node(n, expr) 357 358 def process_attribute_access(self, n): 359 360 "Process the given attribute access node 'n'." 361 362 path = self.get_namespace_path() 363 364 # Test for access to special privileged attributes. 365 366 if isinstance(n, compiler.ast.Getattr) and \ 367 n.attrname in privileged_attributes and not n.privileged: 368 369 raise InspectError("Attribute %s is accessed by an unprivileged operation." % 370 n.attrname, path, n) 371 372 # Obtain any completed chain and return the reference to it. 373 374 name_ref = self.process_attribute_chain(n) 375 376 if self.have_access_expression(n): 377 return name_ref 378 379 # Where the start of the chain of attributes has been reached, determine 380 # the complete access. 381 382 # Given a non-access node, this chain can be handled in its entirety, 383 # either being name-based and thus an access rooted on a name, or being 384 # based on some other node and thus an anonymous access of some kind. 385 386 # Start with the the full attribute chain. 387 388 remaining = self.attrs 389 attrnames = ".".join(remaining) 390 391 # If the accessor cannot be identified, or where attributes 392 # remain in an attribute chain, record the anonymous accesses. 393 394 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 395 396 init_item(self.attr_accesses, path, set) 397 self.attr_accesses[path].add(attrnames) 398 399 self.record_access_details(None, attrnames, self.in_assignment, 400 self.in_invocation) 401 del self.attrs[0] 402 return 403 404 # Name-based accesses will handle the first attribute in a 405 # chain. 406 407 else: 408 attrname = remaining[0] 409 410 # Attribute assignments are used to identify instance attributes. 411 412 if isinstance(n, compiler.ast.AssAttr) and \ 413 self.in_class and self.in_function and n.expr.name == "self": 414 415 self.set_instance_attr(attrname) 416 417 # Record attribute usage using any name local to this namespace, 418 # if assigned in the namespace, or using an external name 419 # (presently just globals within classes). 420 421 name = self.get_name_for_tracking(name_ref.name, name_ref) 422 tracker = self.trackers[-1] 423 424 immediate_access = len(self.attrs) == 1 425 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 426 427 # Record global-based chains for subsequent resolution. 428 429 if name_ref.is_global_name(): 430 self.record_global_access_details(name, attrnames) 431 432 # Make sure the name is being tracked: global names will not 433 # already be initialised in a branch and must be added 434 # explicitly. 435 436 if not tracker.have_name(name): 437 tracker.assign_names([name]) 438 if self.in_function: 439 self.scope_globals[path].add(name) 440 441 # Record attribute usage in the tracker, and record the branch 442 # information for the access. 443 444 branches = tracker.use_attribute(name, attrname, 445 self.in_invocation is not None, assignment) 446 447 if not branches: 448 raise InspectError("Name %s is accessed using %s before an assignment." % ( 449 name, attrname), path, n) 450 451 self.record_branches_for_access(branches, name, attrnames) 452 access_number = self.record_access_details(name, attrnames, 453 self.in_assignment, self.in_invocation) 454 455 del self.attrs[0] 456 return AccessRef(name, attrnames, access_number) 457 458 def process_class_node(self, n): 459 460 "Process the given class node 'n'." 461 462 path = self.get_namespace_path() 463 464 # To avoid notions of class "versions" where the same definition 465 # might be parameterised with different state and be referenced 466 # elsewhere (as base classes, for example), classes in functions or 467 # conditions are forbidden. 468 469 if self.in_function or self.in_conditional: 470 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 471 path, n.name) 472 return 473 474 # Resolve base classes. 475 476 bases = [] 477 478 for base in n.bases: 479 base_class = self.get_class(base) 480 481 if not base_class: 482 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 483 path, n.name, base) 484 return 485 else: 486 bases.append(base_class) 487 488 # Detect conflicting definitions. Such definitions cause conflicts in 489 # the storage of namespace-related information. 490 491 class_name = self.get_object_path(n.name) 492 ref = self.get_object(class_name, defer=False) 493 494 if ref and ref.static(): 495 raise InspectError("Multiple definitions for the same name are not permitted.", class_name, n) 496 497 # Record bases for the class and retain the class name. 498 # Note that the function class does not inherit from the object class. 499 500 if not bases and class_name != "__builtins__.core.object" and \ 501 class_name != "__builtins__.core.function": 502 503 ref = self.get_object("__builtins__.object") 504 bases.append(ref) 505 506 self.importer.classes[class_name] = self.classes[class_name] = bases 507 self.importer.subclasses[class_name] = set() 508 self.scope_globals[class_name] = set() 509 510 # Set the definition before entering the namespace rather than 511 # afterwards because methods may reference it. In normal Python, 512 # a class is not accessible until the definition is complete, but 513 # methods can generally reference it since upon being called the 514 # class will already exist. 515 516 self.set_definition(n.name, "<class>") 517 518 in_class = self.in_class 519 self.in_class = class_name 520 self.set_instance_attr("__class__", Reference("<class>", class_name)) 521 self.enter_namespace(n.name) 522 523 # Do not provide the special instantiator attributes on the function 524 # class. Function instances provide these attributes. 525 526 if class_name != "__builtins__.core.function": 527 528 self.set_name("__fn__") # special instantiator attribute 529 self.set_name("__args__") # special instantiator attribute 530 531 # Provide leafname, parent and context attributes. 532 533 parent, leafname = class_name.rsplit(".", 1) 534 self.set_name("__name__", self.get_constant("string", leafname).reference()) 535 536 if class_name != "__builtins__.core.function": 537 self.set_name("__parent__") 538 539 self.process_structure_node(n.code) 540 self.exit_namespace() 541 self.in_class = in_class 542 543 def process_from_node(self, n): 544 545 "Process the given node 'n', importing from another module." 546 547 path = self.get_namespace_path() 548 549 module_name, names = self.get_module_name(n) 550 if module_name == self.name: 551 raise InspectError("Cannot import from the current module.", path, n) 552 553 self.queue_module(module_name) 554 555 # Attempt to obtain the referenced objects. 556 557 for name, alias in n.names: 558 if name == "*": 559 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 560 561 # Explicit names. 562 563 ref = self.import_name_from_module(name, module_name) 564 value = ResolvedNameRef(alias or name, ref) 565 self.set_general_local(alias or name, value) 566 567 def process_function_node(self, n, name): 568 569 """ 570 Process the given function or lambda node 'n' with the given 'name'. 571 """ 572 573 is_lambda = isinstance(n, compiler.ast.Lambda) 574 575 # Where a function is declared conditionally, use a separate name for 576 # the definition, and assign the definition to the stated name. 577 578 if (self.in_conditional or self.in_function) and not is_lambda: 579 original_name = name 580 name = self.get_lambda_name() 581 else: 582 original_name = None 583 584 # Detect conflicting definitions. Such definitions cause conflicts in 585 # the storage of namespace-related information. 586 587 function_name = self.get_object_path(name) 588 ref = self.get_object(function_name, defer=False) 589 590 if ref and ref.static(): 591 raise InspectError("Multiple definitions for the same name are not permitted.", function_name, n) 592 593 # Initialise argument and local records. 594 595 argnames = get_argnames(n.argnames) 596 is_method = self.in_class and not self.in_function 597 598 # Remove explicit "self" from method parameters. 599 600 if is_method and argnames and argnames[0] == "self": 601 del argnames[0] 602 603 # Convert .name entries in the parameters, provided this is a method. 604 605 l = [] 606 attr_initialisers = [] 607 608 for argname in argnames: 609 if argname[0] == ".": 610 if not is_method: 611 raise InspectError("Attribute initialisers are only allowed amongst method parameters.", function_name, n) 612 613 argname = argname[1:] 614 attr_initialisers.append(argname) 615 616 if argname in l: 617 raise InspectError("Argument name %s is repeated." % argname, function_name, n) 618 619 l.append(argname) 620 621 argnames = l 622 623 # Copy and propagate the parameters. 624 625 self.importer.function_parameters[function_name] = \ 626 self.function_parameters[function_name] = argnames[:] 627 628 self.importer.function_attr_initialisers[function_name] = \ 629 self.function_attr_initialisers[function_name] = attr_initialisers 630 631 # Define all arguments/parameters in the local namespace. 632 633 locals = \ 634 self.importer.function_locals[function_name] = \ 635 self.function_locals[function_name] = {} 636 637 # Insert "self" into method locals. 638 639 if is_method: 640 argnames.insert(0, "self") 641 642 # Define "self" in terms of the class if in a method. 643 # This does not diminish the need for type-narrowing in the deducer. 644 645 if argnames: 646 if self.in_class and not self.in_function and argnames[0] == "self": 647 locals[argnames[0]] = Reference("<instance>", self.in_class) 648 else: 649 locals[argnames[0]] = Reference("<var>") 650 651 for argname in argnames[1:]: 652 locals[argname] = Reference("<var>") 653 654 globals = self.scope_globals[function_name] = set() 655 656 # Process the defaults. 657 658 defaults = self.importer.function_defaults[function_name] = \ 659 self.function_defaults[function_name] = [] 660 661 for argname, default in compiler.ast.get_defaults(n): 662 if argname[0] == ".": 663 argname = argname[1:] 664 665 if default: 666 667 # Obtain any reference for the default. 668 669 name_ref = self.process_structure_node(default) 670 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 671 672 # Reset conditional tracking to focus on the function contents. 673 674 in_conditional = self.in_conditional 675 self.in_conditional = False 676 677 in_function = self.in_function 678 self.in_function = function_name 679 680 self.enter_namespace(name) 681 682 # Define a leafname attribute value for the function instance. 683 684 ref = self.get_builtin_class("string") 685 self.reserve_constant(function_name, name, ref.get_origin()) 686 687 # Track attribute usage within the namespace. 688 689 path = self.get_namespace_path() 690 self.start_tracking(locals) 691 692 # Establish attributes for .name entries, provided this is a method. 693 694 for argname in attr_initialisers: 695 self.process_assignment_node( 696 compiler.ast.AssAttr(compiler.ast.Name("self"), argname, "OP_ASSIGN"), 697 compiler.ast.Name(argname)) 698 699 self.process_structure_node(n.code) 700 returns_value = self.stop_tracking() 701 702 # Record any null result. 703 704 is_initialiser = is_method and name == "__init__" 705 706 if not returns_value and not is_initialiser: 707 self.record_return_value(ResolvedNameRef("None", self.get_builtin("None"))) 708 709 # Exit to the parent. 710 711 self.exit_namespace() 712 713 # Update flags. 714 715 self.in_function = in_function 716 self.in_conditional = in_conditional 717 718 # Define the function using the appropriate name. 719 720 self.set_definition(name, "<function>") 721 722 # Where a function is set conditionally, assign the name. 723 724 if original_name: 725 self.process_assignment_for_object(original_name, compiler.ast.Name(name)) 726 727 def process_global_node(self, n): 728 729 """ 730 Process the given "global" node 'n'. 731 """ 732 733 path = self.get_namespace_path() 734 735 if path != self.name: 736 self.scope_globals[path].update(n.names) 737 738 def process_if_node(self, n): 739 740 """ 741 Process the given "if" node 'n'. 742 """ 743 744 tracker = self.trackers[-1] 745 tracker.new_branchpoint() 746 747 for test, body in n.tests: 748 self.process_structure_node(test) 749 750 tracker.new_branch() 751 752 in_conditional = self.in_conditional 753 self.in_conditional = True 754 self.process_structure_node(body) 755 self.in_conditional = in_conditional 756 757 tracker.shelve_branch() 758 759 # Maintain a branch for the else clause. 760 761 tracker.new_branch() 762 if n.else_: 763 self.process_structure_node(n.else_) 764 tracker.shelve_branch() 765 766 tracker.merge_branches() 767 768 def process_import_node(self, n): 769 770 "Process the given import node 'n'." 771 772 path = self.get_namespace_path() 773 774 # Load the mentioned module. 775 776 for name, alias in n.names: 777 if name == self.name: 778 raise InspectError("Cannot import the current module.", path, n) 779 780 self.set_module(alias or name.split(".")[-1], name) 781 self.queue_module(name, True) 782 783 def process_invocation_node(self, n): 784 785 "Process the given invocation node 'n'." 786 787 path = self.get_namespace_path() 788 789 in_invocation = self.in_invocation 790 self.in_invocation = None 791 792 # Process the arguments. 793 794 keywords = set() 795 796 for arg in n.args: 797 self.process_structure_node(arg) 798 if isinstance(arg, compiler.ast.Keyword): 799 keywords.add(arg.name) 800 801 keywords = list(keywords) 802 keywords.sort() 803 804 # Communicate to the invocation target expression that it forms the 805 # target of an invocation, potentially affecting attribute accesses. 806 807 self.in_invocation = len(n.args), keywords 808 809 # Process the expression, obtaining any identified reference. 810 811 name_ref = self.process_structure_node(n.node) 812 self.in_invocation = in_invocation 813 814 # Detect class invocations. 815 816 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 817 return InstanceRef(name_ref.reference().instance_of()) 818 819 elif isinstance(name_ref, (NameRef, AccessRef)): 820 return InvocationRef(name_ref) 821 822 # Provide a general reference to indicate that something is produced 823 # by the invocation, useful for retaining assignment expression 824 # details. 825 826 return VariableRef() 827 828 def process_lambda_node(self, n): 829 830 "Process the given lambda node 'n'." 831 832 name = self.get_lambda_name() 833 self.process_function_node(n, name) 834 835 origin = self.get_object_path(name) 836 837 if self.function_defaults.get(origin): 838 return None 839 else: 840 return ResolvedNameRef(name, Reference("<function>", origin)) 841 842 def process_logical_node(self, n): 843 844 "Process the given operator node 'n'." 845 846 return self.process_operator_chain(n.nodes) 847 848 def process_name_node(self, n): 849 850 "Process the given name node 'n'." 851 852 path = self.get_namespace_path() 853 854 # Find predefined constant names before anything else. 855 856 if n.name in predefined_constants: 857 ref = self.get_builtin(n.name) 858 value = ResolvedNameRef(n.name, ref) 859 return value 860 861 # Special names that have already been identified. 862 863 if n.name.startswith("$"): 864 value = self.get_special(n.name) 865 if value: 866 return value 867 868 # Special case for operator functions introduced through code 869 # transformations. 870 871 if n.name.startswith("$op"): 872 873 # Obtain the location of the actual function defined in the operator 874 # package. 875 876 op = n.name[len("$op"):] 877 878 # Attempt to get a reference. 879 880 ref = self.import_name_from_module(op, "operator") 881 882 # Record the imported name and provide the resolved name reference. 883 884 value = ResolvedNameRef(n.name, ref) 885 self.set_special(n.name, value) 886 return value 887 888 # Special case for sequence length testing. 889 890 elif n.name.startswith("$seq"): 891 op = n.name[len("$seq"):] 892 ref = self.import_name_from_module(op, "__builtins__.sequence") 893 value = ResolvedNameRef(n.name, ref) 894 self.set_special(n.name, value) 895 return value 896 897 # Special case for print operations. 898 899 elif n.name.startswith("$print"): 900 901 # Attempt to get a reference. 902 903 ref = self.get_builtin("print_") 904 905 # Record the imported name and provide the resolved name reference. 906 907 value = ResolvedNameRef(n.name, ref) 908 self.set_special(n.name, value) 909 return value 910 911 # Test for self usage, which is only allowed in methods. 912 913 if n.name == "self" and not (self.in_function and self.in_class): 914 raise InspectError("Use of self is only allowed in methods.", path, n) 915 916 # Record usage of the name. 917 918 self.record_name(n.name) 919 920 # Search for unknown names in non-function scopes immediately. 921 # Temporary names should not be re-used between scopes. 922 # External names in functions are resolved later. 923 924 ref = not n.name.startswith("$t") and self.find_name(n.name) or None 925 926 if ref: 927 self.record_name_access(n.name, True) 928 return ResolvedNameRef(n.name, ref, is_global=True) 929 930 # Explicitly-declared global names. 931 932 elif self.in_function and n.name in self.scope_globals[path]: 933 self.record_name_access(n.name, True) 934 return NameRef(n.name, is_global=True) 935 936 # Examine other names. 937 938 else: 939 940 # Check local names. 941 942 access_number = self.record_name_access(n.name) 943 944 # Local name. 945 946 if access_number is not None: 947 return LocalNameRef(n.name, access_number) 948 949 # Possible global or built-in name. 950 951 else: 952 self.record_name_access(n.name, True) 953 return NameRef(n.name, is_global=True) 954 955 def record_name_access(self, name, is_global=False): 956 957 """ 958 Record an access involving 'name' if the name is being tracked, using 959 'is_global' to indicate whether the name is global. 960 """ 961 962 name = self.get_name_for_tracking(name, is_global=is_global) 963 branches = self.trackers[-1].tracking_name(name) 964 if branches: 965 self.record_branches_for_access(branches, name, None) 966 return self.record_access_details(name, None, self.in_assignment, 967 self.in_invocation) 968 return None 969 970 def process_operator_chain(self, nodes): 971 972 """ 973 Process the given chain of 'nodes', processing each node or item. 974 Each node starts a new conditional region, effectively making a deeply- 975 nested collection of if-like statements. 976 """ 977 978 results = [] 979 980 tracker = self.trackers[-1] 981 982 for item in nodes: 983 tracker.new_branchpoint() 984 tracker.new_branch() 985 result = self.process_structure_node(item) 986 if result: 987 results.append(result) 988 989 for item in nodes[:-1]: 990 tracker.shelve_branch() 991 tracker.new_branch() 992 tracker.shelve_branch() 993 tracker.merge_branches() 994 995 tracker.shelve_branch() 996 tracker.merge_branches() 997 998 return MultipleRef(results) 999 1000 def process_try_node(self, n): 1001 1002 """ 1003 Process the given "try...except" node 'n'. 1004 """ 1005 1006 self.record_exception_handler() 1007 1008 tracker = self.trackers[-1] 1009 tracker.new_branchpoint() 1010 1011 self.process_structure_node(n.body) 1012 1013 for name, var, handler in n.handlers: 1014 if name is not None: 1015 self.process_structure_node(name) 1016 1017 # Any abandoned branches from the body can now be resumed in a new 1018 # branch. 1019 1020 tracker.resume_abandoned_branches() 1021 1022 # Establish the local for the handler. 1023 1024 if var is not None: 1025 self.process_assignment_node(var, None) 1026 if handler is not None: 1027 self.process_structure_node(handler) 1028 1029 tracker.shelve_branch() 1030 1031 # The else clause maintains the usage from the body but without the 1032 # abandoned branches since they would never lead to the else clause 1033 # being executed. 1034 1035 if n.else_: 1036 tracker.new_branch() 1037 self.process_structure_node(n.else_) 1038 tracker.shelve_branch() 1039 1040 # Without an else clause, a null branch propagates the successful 1041 # outcome. 1042 1043 else: 1044 tracker.new_branch() 1045 tracker.shelve_branch() 1046 1047 tracker.merge_branches() 1048 1049 def process_try_finally_node(self, n): 1050 1051 """ 1052 Process the given "try...finally" node 'n'. 1053 """ 1054 1055 self.record_exception_handler() 1056 1057 tracker = self.trackers[-1] 1058 self.process_structure_node(n.body) 1059 1060 # Any abandoned branches from the body can now be resumed. 1061 1062 branches = tracker.resume_all_abandoned_branches() 1063 self.process_structure_node(n.final) 1064 1065 # At the end of the finally clause, abandoned branches are discarded. 1066 1067 tracker.restore_active_branches(branches) 1068 1069 def process_while_node(self, n): 1070 1071 "Process the given while node 'n'." 1072 1073 tracker = self.trackers[-1] 1074 tracker.new_branchpoint(loop_node=True) 1075 1076 # Evaluate any test or iterator outside the loop. 1077 1078 self.process_structure_node(n.test) 1079 1080 # Propagate attribute usage to branches. 1081 1082 tracker.new_branch(loop_node=True) 1083 1084 # Enter the loop. 1085 1086 in_conditional = self.in_conditional 1087 self.in_conditional = True 1088 self.process_structure_node(n.body) 1089 self.in_conditional = in_conditional 1090 1091 # Continuing branches are resumed before any test. 1092 1093 tracker.resume_continuing_branches() 1094 1095 # Evaluate any continuation test within the body. 1096 1097 self.process_structure_node(n.test) 1098 1099 tracker.shelve_branch(loop_node=True) 1100 1101 # Support the non-looping condition. 1102 1103 tracker.new_branch() 1104 tracker.shelve_branch() 1105 1106 tracker.merge_branches() 1107 1108 # Evaluate any else clause outside branches. 1109 1110 if n.else_: 1111 self.process_structure_node(n.else_) 1112 1113 # Connect broken branches to the code after any loop. 1114 1115 tracker.resume_broken_branches() 1116 1117 # Branch tracking methods. 1118 1119 def start_tracking(self, names): 1120 1121 """ 1122 Start tracking attribute usage for names in the current namespace, 1123 immediately registering the given 'names'. 1124 """ 1125 1126 path = self.get_namespace_path() 1127 parent = self.trackers[-1] 1128 tracker = BranchTracker() 1129 self.trackers.append(tracker) 1130 1131 # Record the given names established as new branches. 1132 1133 tracker.assign_names(names) 1134 1135 def assign_name(self, name, name_ref): 1136 1137 "Assign to 'name' the given 'name_ref' in the current namespace." 1138 1139 name = self.get_name_for_tracking(name) 1140 self.trackers[-1].assign_names([name], [name_ref]) 1141 1142 def stop_tracking(self): 1143 1144 """ 1145 Stop tracking attribute usage, recording computed usage for the current 1146 namespace. Indicate whether a value is always returned from the 1147 namespace. 1148 """ 1149 1150 path = self.get_namespace_path() 1151 tracker = self.trackers.pop() 1152 self.record_assignments_for_access(tracker) 1153 1154 self.attr_usage[path] = tracker.get_all_usage() 1155 self.name_initialisers[path] = tracker.get_all_values() 1156 1157 return tracker.returns_value() 1158 1159 def start_tracking_in_module(self): 1160 1161 "Start tracking attribute usage in the module." 1162 1163 tracker = BranchTracker() 1164 self.trackers.append(tracker) 1165 1166 def stop_tracking_in_module(self): 1167 1168 "Stop tracking attribute usage in the module." 1169 1170 tracker = self.trackers[0] 1171 self.record_assignments_for_access(tracker) 1172 self.attr_usage[self.name] = tracker.get_all_usage() 1173 self.name_initialisers[self.name] = tracker.get_all_values() 1174 1175 def record_assignments_for_access(self, tracker): 1176 1177 """ 1178 For the current path, use the given 'tracker' to record assignment 1179 version information for attribute accesses. 1180 """ 1181 1182 path = self.get_path_for_access() 1183 1184 if not self.attr_accessor_branches.has_key(path): 1185 return 1186 1187 init_item(self.attr_accessors, path, dict) 1188 attr_accessors = self.attr_accessors[path] 1189 1190 # Obtain the branches applying during each access. 1191 1192 for access, all_branches in self.attr_accessor_branches[path].items(): 1193 name, attrnames = access 1194 init_item(attr_accessors, access, list) 1195 1196 # Obtain the assignments applying to each branch. 1197 1198 for branches in all_branches: 1199 positions = tracker.get_assignment_positions_for_branches(name, branches) 1200 1201 # Detect missing name information. 1202 1203 if None in positions: 1204 globals = self.global_attr_accesses.get(path) 1205 accesses = globals and globals.get(name) 1206 if not accesses: 1207 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1208 self.get_namespace_path(), name) 1209 positions.remove(None) 1210 1211 attr_accessors[access].append(positions) 1212 1213 def record_branches_for_access(self, branches, name, attrnames): 1214 1215 """ 1216 Record the given 'branches' for an access involving the given 'name' and 1217 'attrnames'. 1218 """ 1219 1220 access = name, attrnames 1221 path = self.get_path_for_access() 1222 1223 init_item(self.attr_accessor_branches, path, dict) 1224 attr_accessor_branches = self.attr_accessor_branches[path] 1225 1226 init_item(attr_accessor_branches, access, list) 1227 attr_accessor_branches[access].append(branches) 1228 1229 def record_access_details(self, name, attrnames, assignment, invocation): 1230 1231 """ 1232 For the given 'name' and 'attrnames', record an access indicating 1233 whether an 'assignment' or an 'invocation' is occurring. 1234 1235 These details correspond to accesses otherwise recorded by the attribute 1236 accessor and attribute access dictionaries. 1237 """ 1238 1239 access = name, attrnames 1240 path = self.get_path_for_access() 1241 1242 init_item(self.attr_access_modifiers, path, dict) 1243 init_item(self.attr_access_modifiers[path], access, list) 1244 1245 access_number = len(self.attr_access_modifiers[path][access]) 1246 self.attr_access_modifiers[path][access].append((assignment, invocation)) 1247 return access_number 1248 1249 def record_global_access_details(self, name, attrnames): 1250 1251 """ 1252 Record details of a global access via the given 'name' involving the 1253 indicated 'attrnames'. 1254 """ 1255 1256 path = self.get_namespace_path() 1257 1258 init_item(self.global_attr_accesses, path, dict) 1259 init_item(self.global_attr_accesses[path], name, set) 1260 self.global_attr_accesses[path][name].add(attrnames) 1261 1262 # Namespace modification. 1263 1264 def record_name(self, name): 1265 1266 "Record the use of 'name' in a namespace." 1267 1268 path = self.get_namespace_path() 1269 init_item(self.names_used, path, set) 1270 self.names_used[path].add(name) 1271 1272 def set_module(self, name, module_name): 1273 1274 """ 1275 Set a module in the current namespace using the given 'name' associated 1276 with the corresponding 'module_name'. 1277 """ 1278 1279 if name: 1280 self.set_general_local(name, Reference("<module>", module_name)) 1281 1282 def set_definition(self, name, kind): 1283 1284 """ 1285 Set the definition having the given 'name' and 'kind'. 1286 1287 Definitions are set in the static namespace hierarchy, but they can also 1288 be recorded for function locals. 1289 """ 1290 1291 if self.is_global(name): 1292 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1293 self.get_namespace_path(), name) 1294 1295 path = self.get_object_path(name) 1296 self.set_object(path, kind) 1297 1298 ref = self.get_object(path) 1299 if ref.get_kind() == "<var>": 1300 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1301 self.get_namespace_path(), name) 1302 1303 if not self.is_global(name) and self.in_function: 1304 self.set_function_local(name, ref) 1305 1306 def set_function_local(self, name, ref=None): 1307 1308 "Set the local with the given 'name' and optional 'ref'." 1309 1310 path = self.get_namespace_path() 1311 locals = self.function_locals[path] 1312 used = self.names_used.get(path) 1313 1314 if not locals.has_key(name) and used and name in used: 1315 raise InspectError("Name %s assigned locally but used previously." % name, path) 1316 1317 multiple = not ref or locals.has_key(name) and locals[name] != ref 1318 locals[name] = multiple and Reference("<var>") or ref 1319 1320 def assign_general_local(self, name, name_ref): 1321 1322 """ 1323 Set for 'name' the given 'name_ref', recording the name for attribute 1324 usage tracking. 1325 """ 1326 1327 self.set_general_local(name, name_ref) 1328 self.assign_name(name, name_ref) 1329 1330 def set_general_local(self, name, value=None): 1331 1332 """ 1333 Set the 'name' with optional 'value' in any kind of local namespace, 1334 where the 'value' should be a reference if specified. 1335 """ 1336 1337 init_value = self.get_initialising_value(value) 1338 1339 # Module global names. 1340 1341 if self.is_global(name): 1342 path = self.get_global_path(name) 1343 self.set_object(path, init_value) 1344 1345 # Function local names. 1346 1347 elif self.in_function: 1348 self.set_function_local(name, init_value) 1349 1350 # Other namespaces (classes). 1351 1352 else: 1353 self.set_name(name, init_value) 1354 1355 def set_name(self, name, ref=None): 1356 1357 "Attach the 'name' with optional 'ref' to the current namespace." 1358 1359 self.set_object(self.get_object_path(name), ref) 1360 1361 def set_instance_attr(self, name, ref=None): 1362 1363 """ 1364 Add an instance attribute of the given 'name' to the current class, 1365 using the optional 'ref'. 1366 """ 1367 1368 self._set_instance_attr(self.in_class, name, ref) 1369 1370 def _set_instance_attr(self, path, name, ref=None): 1371 1372 init_item(self.instance_attrs, path, set) 1373 self.instance_attrs[path].add(name) 1374 1375 if ref: 1376 init_item(self.instance_attr_constants, path, dict) 1377 self.instance_attr_constants[path][name] = ref 1378 1379 def get_initialising_value(self, value): 1380 1381 "Return a suitable initialiser reference for 'value'." 1382 1383 if isinstance(value, Result): 1384 return value.reference() 1385 else: 1386 return value 1387 1388 # Static, program-relative naming. 1389 1390 def find_name(self, name): 1391 1392 """ 1393 Return the qualified name for the given 'name' used in the current 1394 non-function namespace. 1395 """ 1396 1397 path = self.get_namespace_path() 1398 ref = None 1399 1400 if not self.in_function and name not in predefined_constants: 1401 if self.in_class: 1402 ref = self.get_object(self.get_object_path(name), False) 1403 if not ref: 1404 ref = self.get_global_or_builtin(name) 1405 1406 return ref 1407 1408 def get_class(self, node): 1409 1410 """ 1411 Use the given 'node' to obtain the identity of a class. Return a 1412 reference for the class. Unresolved dependencies are permitted and must 1413 be resolved later. 1414 """ 1415 1416 ref = self._get_class(node) 1417 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1418 1419 def _get_class(self, node): 1420 1421 """ 1422 Use the given 'node' to find a class definition. Return a reference to 1423 the class. 1424 """ 1425 1426 if isinstance(node, compiler.ast.Getattr): 1427 1428 # Obtain the identity of the access target. 1429 1430 ref = self._get_class(node.expr) 1431 1432 # Where the target is a class or module, obtain the identity of the 1433 # attribute. 1434 1435 if ref.has_kind(["<function>", "<var>"]): 1436 return None 1437 else: 1438 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1439 return self.get_object(attrname) 1440 1441 # Names can be module-level or built-in. 1442 1443 elif isinstance(node, compiler.ast.Name): 1444 1445 # Record usage of the name and attempt to identify it. 1446 1447 self.record_name(node.name) 1448 return self.find_name(node.name) 1449 else: 1450 return None 1451 1452 def get_constant(self, name, value): 1453 1454 "Return a constant reference for the given type 'name' and 'value'." 1455 1456 ref = self.get_builtin_class(name) 1457 return self.get_constant_reference(ref, value) 1458 1459 def get_literal_instance(self, n, name=None): 1460 1461 """ 1462 For node 'n', return a reference to an instance of 'name', or if 'name' 1463 is not specified, deduce the type from the value. 1464 """ 1465 1466 # Handle stray None constants (Sliceobj seems to produce them). 1467 1468 if name == "NoneType": 1469 return self.process_name_node(compiler.ast.Name("None")) 1470 1471 # Obtain the details of the literal itself. 1472 # An alias to the type is generated for sequences. 1473 1474 if name in ("dict", "list", "tuple"): 1475 ref = self.get_builtin_class(name) 1476 self.set_special_literal(name, ref) 1477 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1478 1479 # Constant values are independently recorded. 1480 1481 else: 1482 value, typename, encoding = self.get_constant_value(n.value, n.literals) 1483 ref = self.get_builtin_class(typename) 1484 return self.get_constant_reference(ref, value, encoding) 1485 1486 # Special names. 1487 1488 def get_special(self, name): 1489 1490 "Return any stored value for the given special 'name'." 1491 1492 value = self.special.get(name) 1493 if value: 1494 ref, paths = value 1495 else: 1496 ref = None 1497 return ref 1498 1499 def set_special(self, name, value): 1500 1501 """ 1502 Set a special 'name' that merely tracks the use of an implicit object 1503 'value'. 1504 """ 1505 1506 if not self.special.has_key(name): 1507 paths = set() 1508 self.special[name] = value, paths 1509 else: 1510 _ref, paths = self.special[name] 1511 1512 paths.add(self.get_namespace_path()) 1513 1514 def set_special_literal(self, name, ref): 1515 1516 """ 1517 Set a special name for the literal type 'name' having type 'ref'. Such 1518 special names provide a way of referring to literal object types. 1519 """ 1520 1521 literal_name = "$L%s" % name 1522 value = ResolvedNameRef(literal_name, ref) 1523 self.set_special(literal_name, value) 1524 1525 # Exceptions. 1526 1527 def record_exception_handler(self): 1528 1529 "Record the current namespace as employing an exception handler." 1530 1531 self.exception_namespaces.add(self.get_namespace_path()) 1532 1533 # Return values. 1534 1535 def record_return_value(self, expr): 1536 1537 "Record the given return 'expr'." 1538 1539 path = self.get_namespace_path() 1540 l = init_item(self.return_values, path, list) 1541 l.append(expr) 1542 if not self.importer.all_return_values.has_key(path): 1543 self.importer.all_return_values[path] = l 1544 1545 # vim: tabstop=4 expandtab shiftwidth=4