1 #!/usr/bin/env python 2 3 """ 4 Inspect and obtain module structure. 5 6 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from branching import BranchTracker 24 from common import get_argnames, init_item, predefined_constants 25 from modules import BasicModule, CacheWritingModule 26 from referencing import Reference 27 from resolving import NameResolving 28 from results import AccessRef, InstanceRef, InvocationRef, LiteralSequenceRef, \ 29 LocalNameRef, NameRef, ResolvedNameRef 30 import compiler 31 import sys 32 33 class InspectedModule(BasicModule, CacheWritingModule, NameResolving): 34 35 "A module inspector." 36 37 def __init__(self, name, importer): 38 BasicModule.__init__(self, name, importer) 39 NameResolving.__init__(self) 40 41 self.in_class = False 42 self.in_conditional = False 43 self.global_attr_accesses = {} 44 45 # Nested scope handling. 46 47 self.parent_function = None 48 self.propagated_names = {} 49 50 # Usage tracking. 51 52 self.trackers = [] 53 self.attr_accessor_branches = {} 54 55 def __repr__(self): 56 return "InspectedModule(%r, %r)" % (self.name, self.importer) 57 58 def parse(self, filename): 59 60 "Parse the file having the given 'filename'." 61 62 self.parse_file(filename) 63 64 # Inspect the module. 65 66 self.start_tracking_in_module() 67 68 # Detect and record imports and globals declared in the module. 69 70 self.assign_general_local("__name__", self.get_constant("str", self.name)) 71 self.assign_general_local("__file__", self.get_constant("str", filename)) 72 self.process_structure(self.astnode) 73 74 # Set the class of the module after the definition has occurred. 75 76 ref = self.get_builtin("object") 77 self.set_name("__class__", ref) 78 79 # Get module-level attribute usage details. 80 81 self.stop_tracking_in_module() 82 83 def complete(self): 84 85 "Complete the module inspection." 86 87 # Resolve names not definitively mapped to objects. 88 89 self.resolve() 90 91 # Define the invocation requirements in each namespace. 92 93 self.set_invocation_usage() 94 95 # Propagate to the importer information needed in subsequent activities. 96 97 self.propagate() 98 99 def set_invocation_usage(self): 100 101 """ 102 Discard the current invocation storage figures, retaining the maximum 103 values. 104 """ 105 106 for path, (current, maximum) in self.function_targets.items(): 107 self.importer.function_targets[path] = self.function_targets[path] = maximum 108 109 for path, (current, maximum) in self.function_arguments.items(): 110 self.importer.function_arguments[path] = self.function_arguments[path] = maximum 111 112 # Module structure traversal. 113 114 def process_structure_node(self, n): 115 116 "Process the individual node 'n'." 117 118 # Module global detection. 119 120 if isinstance(n, compiler.ast.Global): 121 self.process_global_node(n) 122 123 # Module import declarations. 124 125 elif isinstance(n, compiler.ast.From): 126 self.process_from_node(n) 127 128 elif isinstance(n, compiler.ast.Import): 129 self.process_import_node(n) 130 131 # Nodes using operator module functions. 132 133 elif isinstance(n, compiler.ast.Operator): 134 return self.process_operator_node(n) 135 136 elif isinstance(n, compiler.ast.AugAssign): 137 self.process_augassign_node(n) 138 139 elif isinstance(n, compiler.ast.Compare): 140 return self.process_compare_node(n) 141 142 elif isinstance(n, compiler.ast.Slice): 143 return self.process_slice_node(n) 144 145 elif isinstance(n, compiler.ast.Sliceobj): 146 return self.process_sliceobj_node(n) 147 148 elif isinstance(n, compiler.ast.Subscript): 149 return self.process_subscript_node(n) 150 151 # Namespaces within modules. 152 153 elif isinstance(n, compiler.ast.Class): 154 self.process_class_node(n) 155 156 elif isinstance(n, compiler.ast.Function): 157 self.process_function_node(n, n.name) 158 159 elif isinstance(n, compiler.ast.Lambda): 160 return self.process_lambda_node(n) 161 162 # Assignments. 163 164 elif isinstance(n, compiler.ast.Assign): 165 166 # Handle each assignment node. 167 168 for node in n.nodes: 169 self.process_assignment_node(node, n.expr) 170 171 # Assignments within non-Assign nodes. 172 173 elif isinstance(n, compiler.ast.AssName): 174 self.process_assignment_node(n, None) 175 176 elif isinstance(n, compiler.ast.AssAttr): 177 self.process_attribute_access(n) 178 179 # Accesses. 180 181 elif isinstance(n, compiler.ast.Getattr): 182 return self.process_attribute_access(n) 183 184 # Name recording for later testing. 185 186 elif isinstance(n, compiler.ast.Name): 187 return self.process_name_node(n) 188 189 # Conditional statement tracking. 190 191 elif isinstance(n, compiler.ast.For): 192 self.process_for_node(n) 193 194 elif isinstance(n, compiler.ast.While): 195 self.process_while_node(n) 196 197 elif isinstance(n, compiler.ast.If): 198 self.process_if_node(n) 199 200 elif isinstance(n, (compiler.ast.And, compiler.ast.Or)): 201 return self.process_logical_node(n) 202 203 # Exception control-flow tracking. 204 205 elif isinstance(n, compiler.ast.TryExcept): 206 self.process_try_node(n) 207 208 elif isinstance(n, compiler.ast.TryFinally): 209 self.process_try_finally_node(n) 210 211 # Control-flow modification statements. 212 213 elif isinstance(n, compiler.ast.Break): 214 self.trackers[-1].suspend_broken_branch() 215 216 elif isinstance(n, compiler.ast.Continue): 217 self.trackers[-1].suspend_continuing_branch() 218 219 elif isinstance(n, compiler.ast.Raise): 220 self.process_structure(n) 221 self.trackers[-1].abandon_branch() 222 223 elif isinstance(n, compiler.ast.Return): 224 self.process_structure(n) 225 self.trackers[-1].abandon_returning_branch() 226 227 # Invocations. 228 229 elif isinstance(n, compiler.ast.CallFunc): 230 return self.process_invocation_node(n) 231 232 # Constant usage. 233 234 elif isinstance(n, compiler.ast.Const): 235 return self.get_literal_instance(n, n.value.__class__.__name__) 236 237 elif isinstance(n, compiler.ast.Dict): 238 return self.get_literal_instance(n, "dict") 239 240 elif isinstance(n, compiler.ast.List): 241 return self.get_literal_instance(n, "list") 242 243 elif isinstance(n, compiler.ast.Tuple): 244 return self.get_literal_instance(n, "tuple") 245 246 # List comprehensions and if expressions. 247 248 elif isinstance(n, compiler.ast.ListComp): 249 self.process_listcomp_node(n) 250 251 elif isinstance(n, compiler.ast.IfExp): 252 self.process_ifexp_node(n) 253 254 # All other nodes are processed depth-first. 255 256 else: 257 self.process_structure(n) 258 259 # By default, no expression details are returned. 260 261 return None 262 263 # Specific node handling. 264 265 def process_assignment_node(self, n, expr): 266 267 "Process the individual node 'n' to be assigned the contents of 'expr'." 268 269 # Names and attributes are assigned the entire expression. 270 271 if isinstance(n, compiler.ast.AssName): 272 273 name_ref = expr and self.process_structure_node(expr) 274 275 # Name assignments populate either function namespaces or the 276 # general namespace hierarchy. 277 278 self.assign_general_local(n.name, name_ref) 279 280 # Record usage of the name. 281 282 self.record_name(n.name) 283 284 elif isinstance(n, compiler.ast.AssAttr): 285 if expr: self.process_structure_node(expr) 286 self.process_attribute_access(n) 287 288 # Lists and tuples are matched against the expression and their 289 # items assigned to expression items. 290 291 elif isinstance(n, (compiler.ast.AssList, compiler.ast.AssTuple)): 292 self.process_assignment_node_items(n, expr) 293 294 # Slices and subscripts are permitted within assignment nodes. 295 296 elif isinstance(n, compiler.ast.Slice): 297 self.process_slice_node(n, expr) 298 299 elif isinstance(n, compiler.ast.Subscript): 300 self.process_subscript_node(n, expr) 301 302 def process_attribute_access(self, n): 303 304 "Process the given attribute access node 'n'." 305 306 # Obtain any completed chain and return the reference to it. 307 308 name_ref = self.process_attribute_chain(n) 309 if self.have_access_expression(n): 310 return name_ref 311 312 # Where the start of the chain of attributes has been reached, determine 313 # the complete access. 314 315 # Given a non-access node, this chain can be handled in its entirety, 316 # either being name-based and thus an access rooted on a name, or being 317 # based on some other node and thus an anonymous access of some kind. 318 319 path = self.get_namespace_path() 320 321 # Start with the the full attribute chain. 322 323 remaining = self.attrs 324 attrnames = ".".join(remaining) 325 326 # If the accessor cannot be identified, or where attributes 327 # remain in an attribute chain, record the anonymous accesses. 328 329 if not isinstance(name_ref, NameRef): # includes ResolvedNameRef 330 331 assignment = isinstance(n, compiler.ast.AssAttr) 332 333 init_item(self.attr_accesses, path, set) 334 self.attr_accesses[path].add(attrnames) 335 336 self.record_access_details(None, attrnames, assignment) 337 del self.attrs[0] 338 return 339 340 # Name-based accesses will handle the first attribute in a 341 # chain. 342 343 else: 344 attrname = remaining[0] 345 346 # Attribute assignments are used to identify instance attributes. 347 348 if isinstance(n, compiler.ast.AssAttr) and \ 349 self.in_class and self.in_function and n.expr.name == "self": 350 351 self.set_instance_attr(attrname) 352 353 # Record attribute usage using any name local to this namespace, 354 # if assigned in the namespace, or using an external name 355 # (presently just globals within classes). 356 357 name = self.get_name_for_tracking(name_ref.name, name_ref.final()) 358 tracker = self.trackers[-1] 359 360 immediate_access = len(self.attrs) == 1 361 assignment = immediate_access and isinstance(n, compiler.ast.AssAttr) 362 363 del self.attrs[0] 364 365 # Record global-based chains for subsequent resolution. 366 367 is_global = self.in_function and not self.function_locals[path].has_key(name) or \ 368 not self.in_function 369 370 if is_global: 371 self.record_global_access_details(name, attrnames) 372 373 # Make sure the name is being tracked: global names will not 374 # already be initialised in a branch and must be added 375 # explicitly. 376 377 if not tracker.have_name(name): 378 tracker.assign_names([name]) 379 if self.in_function: 380 self.scope_globals[path].add(name) 381 382 # Record attribute usage in the tracker, and record the branch 383 # information for the access. 384 385 branches = tracker.use_attribute(name, attrname) 386 387 if not branches: 388 print >>sys.stderr, "In %s, name %s is accessed using %s before an assignment." % ( 389 path, name, attrname) 390 branches = tracker.use_attribute(name, attrname) 391 392 self.record_branches_for_access(branches, name, attrnames) 393 access_number = self.record_access_details(name, attrnames, assignment) 394 return AccessRef(name, attrnames, access_number) 395 396 def process_class_node(self, n): 397 398 "Process the given class node 'n'." 399 400 path = self.get_namespace_path() 401 402 # To avoid notions of class "versions" where the same definition 403 # might be parameterised with different state and be referenced 404 # elsewhere (as base classes, for example), classes in functions or 405 # conditions are forbidden. 406 407 if self.in_function or self.in_conditional: 408 print >>sys.stderr, "In %s, class %s in function or conditional statement ignored." % ( 409 path, n.name) 410 return 411 412 # Resolve base classes. 413 414 bases = [] 415 416 for base in n.bases: 417 base_class = self.get_class(base) 418 419 if not base_class: 420 print >>sys.stderr, "In %s, class %s has unidentifiable base class: %s" % ( 421 path, n.name, base) 422 return 423 else: 424 bases.append(base_class) 425 426 # Record bases for the class and retain the class name. 427 428 class_name = self.get_object_path(n.name) 429 430 if not bases and class_name != "__builtins__.core.object": 431 ref = self.get_object("__builtins__.object") 432 bases.append(ref) 433 434 self.importer.classes[class_name] = self.classes[class_name] = bases 435 self.importer.subclasses[class_name] = set() 436 self.scope_globals[class_name] = set() 437 438 # Set the definition before entering the namespace rather than 439 # afterwards because methods may reference it. In normal Python, 440 # a class is not accessible until the definition is complete, but 441 # methods can generally reference it since upon being called the 442 # class will already exist. 443 444 self.set_definition(n.name, "<class>") 445 446 in_class = self.in_class 447 self.in_class = class_name 448 self.set_instance_attr("__class__", Reference("<class>", class_name)) 449 self.enter_namespace(n.name) 450 self.set_name("__fn__") # special instantiator attribute 451 self.set_name("__args__") # special instantiator attribute 452 self.assign_general_local("__name__", self.get_constant("str", class_name)) 453 self.process_structure_node(n.code) 454 self.exit_namespace() 455 self.in_class = in_class 456 457 def process_from_node(self, n): 458 459 "Process the given node 'n', importing from another module." 460 461 path = self.get_namespace_path() 462 463 module_name, names = self.get_module_name(n) 464 if module_name == self.name: 465 raise InspectError("Cannot import from the current module.", path, n) 466 467 self.importer.queue_module(module_name, self) 468 469 # Attempt to obtain the referenced objects. 470 471 for name, alias in n.names: 472 if name == "*": 473 raise InspectError("Only explicitly specified names can be imported from modules.", path, n) 474 475 # Explicit names. 476 477 ref = self.import_name_from_module(name, module_name) 478 value = ResolvedNameRef(alias or name, ref) 479 self.set_general_local(alias or name, value) 480 481 def process_function_node(self, n, name): 482 483 """ 484 Process the given function or lambda node 'n' with the given 'name'. 485 """ 486 487 is_lambda = isinstance(n, compiler.ast.Lambda) 488 489 # Where a function is declared conditionally, use a separate name for 490 # the definition, and assign the definition to the stated name. 491 492 if (self.in_conditional or self.in_function) and not is_lambda: 493 original_name = name 494 name = self.get_lambda_name() 495 else: 496 original_name = None 497 498 # Initialise argument and local records. 499 500 function_name = self.get_object_path(name) 501 502 argnames = self.importer.function_parameters[function_name] = \ 503 self.function_parameters[function_name] = get_argnames(n.argnames) 504 locals = self.function_locals[function_name] = {} 505 506 for argname in argnames: 507 locals[argname] = Reference("<var>") 508 509 globals = self.scope_globals[function_name] = set() 510 511 # Process the defaults. 512 513 defaults = self.importer.function_defaults[function_name] = \ 514 self.function_defaults[function_name] = [] 515 516 for argname, default in compiler.ast.get_defaults(n): 517 if default: 518 519 # Obtain any reference for the default. 520 521 name_ref = self.process_structure_node(default) 522 defaults.append((argname, name_ref.is_name() and name_ref.reference() or Reference("<var>"))) 523 524 # Reset conditional tracking to focus on the function contents. 525 526 parent_function = self.parent_function 527 self.parent_function = self.in_function and self.get_namespace_path() or None 528 529 in_conditional = self.in_conditional 530 self.in_conditional = False 531 532 in_function = self.in_function 533 self.in_function = function_name 534 535 self.enter_namespace(name) 536 537 # Track attribute usage within the namespace. 538 539 path = self.get_namespace_path() 540 init_item(self.propagated_names, path, set) 541 542 self.start_tracking(locals) 543 self.process_structure_node(n.code) 544 self.stop_tracking() 545 546 # Propagate names from parent scopes. 547 548 for local in self.propagated_names[path]: 549 if not local in argnames and self.trackers[-1].have_name(local): 550 argnames.append(local) 551 defaults.append((local, Reference("<var>"))) 552 self.set_function_local(local) 553 554 # Exit to the parent and note propagated names. 555 556 self.exit_namespace() 557 558 parent = self.get_namespace_path() 559 if self.propagated_names.has_key(parent): 560 for local in self.propagated_names[path]: 561 self.propagated_names[parent].add(local) 562 563 # Update flags. 564 565 self.in_function = in_function 566 self.in_conditional = in_conditional 567 self.parent_function = parent_function 568 569 # Define the function using the appropriate name. 570 571 self.set_definition(name, "<function>") 572 573 # Where a function is set conditionally, assign the name. 574 575 if original_name: 576 self.process_assignment_for_function(original_name, name) 577 578 def process_global_node(self, n): 579 580 """ 581 Process the given "global" node 'n'. 582 """ 583 584 path = self.get_namespace_path() 585 586 if path != self.name: 587 self.scope_globals[path].update(n.names) 588 589 def process_if_node(self, n): 590 591 """ 592 Process the given "if" node 'n'. 593 """ 594 595 tracker = self.trackers[-1] 596 tracker.new_branchpoint() 597 598 for test, body in n.tests: 599 self.process_structure_node(test) 600 601 tracker.new_branch() 602 603 in_conditional = self.in_conditional 604 self.in_conditional = True 605 self.process_structure_node(body) 606 self.in_conditional = in_conditional 607 608 tracker.shelve_branch() 609 610 # Maintain a branch for the else clause. 611 612 tracker.new_branch() 613 if n.else_: 614 self.process_structure_node(n.else_) 615 tracker.shelve_branch() 616 617 tracker.merge_branches() 618 619 def process_ifexp_node(self, n): 620 621 "Process the given if expression node 'n'." 622 623 name_ref = self.process_structure_node(self.convert_ifexp_node(n)) 624 625 path = self.get_namespace_path() 626 self.allocate_arguments(path, self.function_defaults[name_ref.get_origin()]) 627 self.deallocate_arguments(path, self.function_defaults[name_ref.get_origin()]) 628 629 return InvocationRef(name_ref) 630 631 def process_import_node(self, n): 632 633 "Process the given import node 'n'." 634 635 path = self.get_namespace_path() 636 637 # Load the mentioned module. 638 639 for name, alias in n.names: 640 if name == self.name: 641 raise InspectError("Cannot import the current module.", path, n) 642 if not alias and len(n.names) > 1: 643 raise InspectError("Imported modules must be aliased unless a simple module is imported.", path, n) 644 645 self.set_module(alias or name.split(".")[0], name) 646 self.importer.queue_module(name, self) 647 648 def process_invocation_node(self, n): 649 650 "Process the given invocation node 'n'." 651 652 path = self.get_namespace_path() 653 654 self.allocate_arguments(path, n.args) 655 656 try: 657 # Process the expression, obtaining any identified reference. 658 659 name_ref = self.process_structure_node(n.node) 660 661 # Process the arguments. 662 663 for arg in n.args: 664 self.process_structure_node(arg) 665 666 # Detect class invocations. 667 668 if isinstance(name_ref, ResolvedNameRef) and name_ref.has_kind("<class>"): 669 return InstanceRef(name_ref.reference().instance_of()) 670 671 elif isinstance(name_ref, NameRef): 672 return InvocationRef(name_ref) 673 674 return None 675 676 finally: 677 self.deallocate_arguments(path, n.args) 678 679 def process_lambda_node(self, n): 680 681 "Process the given lambda node 'n'." 682 683 name = self.get_lambda_name() 684 self.process_function_node(n, name) 685 686 origin = self.get_object_path(name) 687 return ResolvedNameRef(name, Reference("<function>", origin)) 688 689 def process_listcomp_node(self, n): 690 691 "Process the given list comprehension node 'n'." 692 693 name_ref = self.process_structure_node(self.convert_listcomp_node(n)) 694 695 path = self.get_namespace_path() 696 self.allocate_arguments(path, self.function_defaults[name_ref.get_origin()]) 697 self.deallocate_arguments(path, self.function_defaults[name_ref.get_origin()]) 698 699 return InvocationRef(name_ref) 700 701 def process_logical_node(self, n): 702 703 "Process the given operator node 'n'." 704 705 self.process_operator_chain(n.nodes, self.process_structure_node) 706 707 def process_name_node(self, n): 708 709 "Process the given name node 'n'." 710 711 path = self.get_namespace_path() 712 713 # Special names. 714 715 if n.name.startswith("$"): 716 value = self.get_special(n.name) 717 if value: 718 return value 719 720 # Special case for operator functions introduced through code 721 # transformations. 722 723 if n.name.startswith("$op"): 724 725 # Obtain the location of the actual function defined in the operator 726 # package. 727 728 op = n.name[len("$op"):] 729 730 # Attempt to get a reference. 731 732 ref = self.import_name_from_module(op, "operator") 733 734 # Record the imported name and provide the resolved name reference. 735 # NOTE: Maybe use a different class. 736 737 value = ResolvedNameRef(n.name, ref) 738 self.set_special(n.name, value) 739 return value 740 741 # Record usage of the name. 742 743 self.record_name(n.name) 744 745 # Search for unknown names in non-function scopes immediately. 746 # External names in functions are resolved later. 747 748 ref = self.find_name(n.name) 749 if ref: 750 return ResolvedNameRef(n.name, ref) 751 752 # Global name. 753 754 elif self.in_function and n.name in self.scope_globals[path]: 755 return NameRef(n.name) 756 757 # Examine other names. 758 759 else: 760 tracker = self.trackers[-1] 761 762 # Check local names. 763 764 branches = tracker.tracking_name(n.name) 765 766 # Find names inherited from a parent scope. 767 768 if not branches and self.parent_function: 769 branches = tracker.have_name(n.name) 770 if branches: 771 self.propagate_name(n.name) 772 773 # Local or inherited name. 774 775 if branches: 776 self.record_branches_for_access(branches, n.name, None) 777 access_number = self.record_access_details(n.name, None, False) 778 return LocalNameRef(n.name, access_number) 779 780 # Possible global name. 781 782 else: 783 return NameRef(n.name) 784 785 def process_operator_chain(self, nodes, fn): 786 787 """ 788 Process the given chain of 'nodes', applying 'fn' to each node or item. 789 Each node starts a new conditional region, effectively making a deeply- 790 nested collection of if-like statements. 791 """ 792 793 tracker = self.trackers[-1] 794 795 for item in nodes: 796 tracker.new_branchpoint() 797 tracker.new_branch() 798 fn(item) 799 800 for item in nodes[:-1]: 801 tracker.shelve_branch() 802 tracker.new_branch() 803 tracker.shelve_branch() 804 tracker.merge_branches() 805 806 tracker.shelve_branch() 807 tracker.merge_branches() 808 809 def process_try_node(self, n): 810 811 """ 812 Process the given "try...except" node 'n'. 813 """ 814 815 tracker = self.trackers[-1] 816 tracker.new_branchpoint() 817 818 self.process_structure_node(n.body) 819 820 for name, var, handler in n.handlers: 821 if name is not None: 822 self.process_structure_node(name) 823 824 # Any abandoned branches from the body can now be resumed in a new 825 # branch. 826 827 tracker.resume_abandoned_branches() 828 829 # Establish the local for the handler. 830 831 if var is not None: 832 self.process_structure_node(var) 833 if handler is not None: 834 self.process_structure_node(handler) 835 836 tracker.shelve_branch() 837 838 # The else clause maintains the usage from the body but without the 839 # abandoned branches since they would never lead to the else clause 840 # being executed. 841 842 if n.else_: 843 tracker.new_branch() 844 self.process_structure_node(n.else_) 845 tracker.shelve_branch() 846 847 # Without an else clause, a null branch propagates the successful 848 # outcome. 849 850 else: 851 tracker.new_branch() 852 tracker.shelve_branch() 853 854 tracker.merge_branches() 855 856 def process_try_finally_node(self, n): 857 858 """ 859 Process the given "try...finally" node 'n'. 860 """ 861 862 tracker = self.trackers[-1] 863 self.process_structure_node(n.body) 864 865 # Any abandoned branches from the body can now be resumed. 866 867 branches = tracker.resume_all_abandoned_branches() 868 self.process_structure_node(n.final) 869 870 # At the end of the finally clause, abandoned branches are discarded. 871 872 tracker.restore_active_branches(branches) 873 874 def process_while_node(self, n): 875 876 "Process the given while node 'n'." 877 878 tracker = self.trackers[-1] 879 tracker.new_branchpoint(loop_node=True) 880 881 # Evaluate any test or iterator outside the loop. 882 883 self.process_structure_node(n.test) 884 885 # Propagate attribute usage to branches. 886 887 tracker.new_branch(loop_node=True) 888 889 # Enter the loop. 890 891 in_conditional = self.in_conditional 892 self.in_conditional = True 893 self.process_structure_node(n.body) 894 self.in_conditional = in_conditional 895 896 # Continuing branches are resumed before any test. 897 898 tracker.resume_continuing_branches() 899 900 # Evaluate any continuation test within the body. 901 902 self.process_structure_node(n.test) 903 904 tracker.shelve_branch(loop_node=True) 905 906 # Support the non-looping condition. 907 908 tracker.new_branch() 909 tracker.shelve_branch() 910 911 tracker.merge_branches() 912 913 # Evaluate any else clause outside branches. 914 915 if n.else_: 916 self.process_structure_node(n.else_) 917 918 # Connect broken branches to the code after any loop. 919 920 tracker.resume_broken_branches() 921 922 # Branch tracking methods. 923 924 def start_tracking(self, names): 925 926 """ 927 Start tracking attribute usage for names in the current namespace, 928 immediately registering the given 'names'. 929 """ 930 931 path = self.get_namespace_path() 932 parent = self.trackers[-1] 933 tracker = BranchTracker() 934 self.trackers.append(tracker) 935 936 # For functions created from expressions or for functions within 937 # functions, propagate usage to the new namespace. 938 939 if self.parent_function: 940 tracker.inherit_branches(parent, names) 941 942 # Record the given names established as new branches. 943 944 tracker.assign_names(names) 945 946 def assign_name(self, name, name_ref): 947 948 "Assign to 'name' the given 'name_ref' in the current namespace." 949 950 name = self.get_name_for_tracking(name) 951 self.trackers[-1].assign_names([name], [name_ref]) 952 953 def stop_tracking(self): 954 955 """ 956 Stop tracking attribute usage, recording computed usage for the current 957 namespace. 958 """ 959 960 path = self.get_namespace_path() 961 tracker = self.trackers.pop() 962 self.record_assignments_for_access(tracker) 963 964 self.attr_usage[path] = tracker.get_all_usage() 965 self.name_initialisers[path] = tracker.get_all_values() 966 967 def start_tracking_in_module(self): 968 969 "Start tracking attribute usage in the module." 970 971 tracker = BranchTracker() 972 self.trackers.append(tracker) 973 974 def stop_tracking_in_module(self): 975 976 "Stop tracking attribute usage in the module." 977 978 tracker = self.trackers[0] 979 self.record_assignments_for_access(tracker) 980 self.attr_usage[self.name] = tracker.get_all_usage() 981 self.name_initialisers[self.name] = tracker.get_all_values() 982 983 def propagate_name(self, name): 984 985 "Propagate the given 'name' into the current namespace." 986 987 path = self.get_namespace_path() 988 self.propagated_names[path].add(name) 989 990 def record_assignments_for_access(self, tracker): 991 992 """ 993 For the current path, use the given 'tracker' to record assignment 994 version information for attribute accesses. 995 """ 996 997 path = self.get_path_for_access() 998 999 if not self.attr_accessor_branches.has_key(path): 1000 return 1001 1002 init_item(self.attr_accessors, path, dict) 1003 attr_accessors = self.attr_accessors[path] 1004 1005 # Obtain the branches applying during each access. 1006 1007 for access, all_branches in self.attr_accessor_branches[path].items(): 1008 name, attrnames = access 1009 init_item(attr_accessors, access, list) 1010 1011 # Obtain the assignments applying to each branch. 1012 1013 for branches in all_branches: 1014 positions = tracker.get_assignment_positions_for_branches(name, branches) 1015 1016 # Detect missing name information. 1017 1018 if None in positions: 1019 globals = self.global_attr_accesses.get(path) 1020 accesses = globals and globals.get(name) 1021 if not accesses: 1022 print >>sys.stderr, "In %s, %s may not be defined when used." % ( 1023 self.get_namespace_path(), name) 1024 positions.remove(None) 1025 1026 attr_accessors[access].append(positions) 1027 1028 def record_branches_for_access(self, branches, name, attrnames): 1029 1030 """ 1031 Record the given 'branches' for an access involving the given 'name' and 1032 'attrnames'. 1033 """ 1034 1035 access = name, attrnames 1036 path = self.get_path_for_access() 1037 1038 init_item(self.attr_accessor_branches, path, dict) 1039 attr_accessor_branches = self.attr_accessor_branches[path] 1040 1041 init_item(attr_accessor_branches, access, list) 1042 attr_accessor_branches[access].append(branches) 1043 1044 def record_access_details(self, name, attrnames, assignment): 1045 1046 """ 1047 For the given 'name' and 'attrnames', record an access indicating 1048 whether 'assignment' is occurring. 1049 1050 These details correspond to accesses otherwise recorded by the attribute 1051 accessor and attribute access dictionaries. 1052 """ 1053 1054 access = name, attrnames 1055 path = self.get_path_for_access() 1056 1057 init_item(self.attr_access_modifiers, path, dict) 1058 init_item(self.attr_access_modifiers[path], access, list) 1059 1060 access_number = len(self.attr_access_modifiers[path][access]) 1061 self.attr_access_modifiers[path][access].append(assignment) 1062 return access_number 1063 1064 def record_global_access_details(self, name, attrnames): 1065 1066 """ 1067 Record details of a global access via the given 'name' involving the 1068 indicated 'attrnames'. 1069 """ 1070 1071 path = self.get_namespace_path() 1072 1073 init_item(self.global_attr_accesses, path, dict) 1074 init_item(self.global_attr_accesses[path], name, set) 1075 self.global_attr_accesses[path][name].add(attrnames) 1076 1077 # Namespace modification. 1078 1079 def record_name(self, name): 1080 1081 "Record the use of 'name' in a namespace." 1082 1083 path = self.get_namespace_path() 1084 init_item(self.names_used, path, set) 1085 self.names_used[path].add(name) 1086 1087 def set_module(self, name, module_name): 1088 1089 """ 1090 Set a module in the current namespace using the given 'name' associated 1091 with the corresponding 'module_name'. 1092 """ 1093 1094 if name: 1095 self.set_general_local(name, Reference("<module>", module_name)) 1096 1097 def set_definition(self, name, kind): 1098 1099 """ 1100 Set the definition having the given 'name' and 'kind'. 1101 1102 Definitions are set in the static namespace hierarchy, but they can also 1103 be recorded for function locals. 1104 """ 1105 1106 if self.is_global(name): 1107 print >>sys.stderr, "In %s, %s is defined as being global." % ( 1108 self.get_namespace_path(), name) 1109 1110 path = self.get_object_path(name) 1111 self.set_object(path, kind) 1112 1113 ref = self.get_object(path) 1114 if ref.get_kind() == "<var>": 1115 print >>sys.stderr, "In %s, %s is defined more than once." % ( 1116 self.get_namespace_path(), name) 1117 1118 if not self.is_global(name) and self.in_function: 1119 self.set_function_local(name, ref) 1120 1121 def set_function_local(self, name, ref=None): 1122 1123 "Set the local with the given 'name' and optional 'ref'." 1124 1125 locals = self.function_locals[self.get_namespace_path()] 1126 multiple = not ref or locals.has_key(name) and locals[name] != ref 1127 locals[name] = multiple and Reference("<var>") or ref 1128 1129 def assign_general_local(self, name, name_ref): 1130 1131 """ 1132 Set for 'name' the given 'name_ref', recording the name for attribute 1133 usage tracking. 1134 """ 1135 1136 self.set_general_local(name, name_ref) 1137 self.assign_name(name, name_ref) 1138 1139 def set_general_local(self, name, value=None): 1140 1141 """ 1142 Set the 'name' with optional 'value' in any kind of local namespace, 1143 where the 'value' should be a reference if specified. 1144 """ 1145 1146 init_value = self.get_initialising_value(value) 1147 1148 # Module global names. 1149 1150 if self.is_global(name): 1151 path = self.get_global_path(name) 1152 self.set_object(path, init_value) 1153 1154 # Function local names. 1155 1156 elif self.in_function: 1157 path = self.get_object_path(name) 1158 self.set_function_local(name, init_value) 1159 1160 # Other namespaces (classes). 1161 1162 else: 1163 path = self.get_object_path(name) 1164 self.set_name(name, init_value) 1165 1166 def set_name(self, name, ref=None): 1167 1168 "Attach the 'name' with optional 'ref' to the current namespace." 1169 1170 self.set_object(self.get_object_path(name), ref) 1171 1172 def set_instance_attr(self, name, ref=None): 1173 1174 """ 1175 Add an instance attribute of the given 'name' to the current class, 1176 using the optional 'ref'. 1177 """ 1178 1179 init_item(self.instance_attrs, self.in_class, set) 1180 self.instance_attrs[self.in_class].add(name) 1181 1182 if ref: 1183 init_item(self.instance_attr_constants, self.in_class, dict) 1184 self.instance_attr_constants[self.in_class][name] = ref 1185 1186 def get_initialising_value(self, value): 1187 1188 "Return a suitable initialiser reference for 'value'." 1189 1190 if isinstance(value, (NameRef, AccessRef, InstanceRef)): # includes LiteralSequenceRef, ResolvedNameRef 1191 return value.reference() 1192 1193 # In general, invocations do not produce known results. However, the 1194 # name initialisers are resolved once a module has been inspected. 1195 1196 elif isinstance(value, InvocationRef): 1197 return None 1198 1199 else: 1200 return value 1201 1202 # Static, program-relative naming. 1203 1204 def find_name(self, name): 1205 1206 """ 1207 Return the qualified name for the given 'name' used in the current 1208 non-function namespace. 1209 """ 1210 1211 path = self.get_namespace_path() 1212 ref = None 1213 1214 if not self.in_function and name not in predefined_constants: 1215 if self.in_class: 1216 ref = self.get_object(self.get_object_path(name)) 1217 if not ref: 1218 ref = self.get_global_or_builtin(name) 1219 1220 return ref 1221 1222 def get_class(self, node): 1223 1224 """ 1225 Use the given 'node' to obtain the identity of a class. Return a 1226 reference for the class. Unresolved dependencies are permitted and must 1227 be resolved later. 1228 """ 1229 1230 ref = self._get_class(node) 1231 return ref.has_kind(["<class>", "<depends>"]) and ref or None 1232 1233 def _get_class(self, node): 1234 1235 """ 1236 Use the given 'node' to find a class definition. Return a reference to 1237 the class. 1238 """ 1239 1240 if isinstance(node, compiler.ast.Getattr): 1241 1242 # Obtain the identity of the access target. 1243 1244 ref = self._get_class(node.expr) 1245 1246 # Where the target is a class or module, obtain the identity of the 1247 # attribute. 1248 1249 if ref.has_kind(["<function>", "<var>"]): 1250 return None 1251 else: 1252 attrname = "%s.%s" % (ref.get_origin(), node.attrname) 1253 return self.get_object(attrname) 1254 1255 # Names can be module-level or built-in. 1256 1257 elif isinstance(node, compiler.ast.Name): 1258 1259 # Record usage of the name and attempt to identify it. 1260 1261 self.record_name(node.name) 1262 return self.get_global_or_builtin(node.name) 1263 else: 1264 return None 1265 1266 def get_constant(self, name, value): 1267 1268 "Return a constant reference for the given type 'name' and 'value'." 1269 1270 ref = self.get_builtin_class(name) 1271 return self.get_constant_reference(ref, value) 1272 1273 def get_literal_instance(self, n, name): 1274 1275 "For node 'n', return a reference to an instance of 'name'." 1276 1277 # Get a reference to the built-in class. 1278 1279 ref = self.get_builtin_class(name) 1280 1281 # Obtain the details of the literal itself. 1282 # An alias to the type is generated for sequences. 1283 1284 if name in ("dict", "list", "tuple"): 1285 self.set_special_literal(name, ref) 1286 return self.process_literal_sequence_node(n, name, ref, LiteralSequenceRef) 1287 1288 # Constant values are independently recorded. 1289 1290 else: 1291 return self.get_constant_reference(ref, n.value) 1292 1293 # Functions and invocations. 1294 1295 def allocate_arguments(self, path, args): 1296 1297 """ 1298 Allocate temporary argument storage using current and maximum 1299 requirements for the given 'path' and 'args'. 1300 """ 1301 1302 init_item(self.function_targets, path, lambda: [0, 0]) 1303 t = self.function_targets[path] 1304 t[0] += 1 1305 t[1] = max(t[0], t[1]) 1306 1307 init_item(self.function_arguments, path, lambda: [0, 0]) 1308 t = self.function_arguments[path] 1309 t[0] += len(args) + 1 1310 t[1] = max(t[0], t[1]) 1311 1312 def deallocate_arguments(self, path, args): 1313 1314 "Deallocate temporary argument storage for the given 'path' and 'args'." 1315 1316 self.function_targets[path][0] -= 1 1317 self.function_arguments[path][0] -= len(args) + 1 1318 1319 # vim: tabstop=4 expandtab shiftwidth=4