1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 202 # Only consider deleting entire unused objects or things accessible 203 # via names which are never used. 204 205 if delete_all or not self.importer.uses_name(name): 206 del obj[name] 207 208 # Delete any unambiguous attribute value. Such values can only 209 # have been defined within the object and therefore are not 210 # redefined by other code regions. 211 212 if attr.assignments == 1: 213 value = attr.get_value() 214 215 if value is not obj and value in self.all_objects: 216 self.all_objects.remove(value) 217 218 # Delete class contents. 219 220 if isinstance(value, Class): 221 self.vacuum_object(value, 1) 222 223 def finalise(self): 224 225 "Finalise the module." 226 227 for obj in self.all_objects: 228 if isinstance(obj, (Class, Function)): 229 obj.finalise_attributes() 230 231 def add_object(self, obj, any_scope=0): 232 233 """ 234 Record 'obj' if non-local or if the optional 'any_scope' is set to a 235 true value. 236 """ 237 238 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 239 self.all_objects.add(obj) 240 241 # Optimisation tests. 242 243 def should_optimise_unused_objects(self): 244 return "unused_objects" in self.optimisations 245 246 # Namespace methods. 247 248 def store(self, name, obj): 249 250 "Record attribute or local 'name', storing 'obj'." 251 252 if not self.namespaces: 253 self.set(name, obj, not self.in_loop) 254 else: 255 self.namespaces[-1].set(name, obj, not self.in_loop) 256 257 def store_lambda(self, obj): 258 259 "Store a lambda function 'obj'." 260 261 self.add_object(obj) 262 263 def store_module_attr(self, name, module): 264 265 """ 266 Record module attribute 'name' in the given 'module' using the current 267 expression. 268 """ 269 270 module.set(name, self.expr, 0) 271 272 def store_class_attr(self, name): 273 274 """ 275 Record class attribute 'name' in the current class using the current 276 expression. 277 """ 278 279 if self.in_method and self.namespaces[-2].has_key(name): 280 self.namespaces[-2].set(name, self.expr, 0) 281 return 1 282 283 return 0 284 285 def store_instance_attr(self, name): 286 287 "Record instance attribute 'name' in the current class." 288 289 if self.in_method: 290 291 # Current namespace is the function. 292 # Previous namespace is the class. 293 294 self.namespaces[-2].add_instance_attribute(name) 295 296 def get_namespace(self): 297 298 "Return the parent (or most recent) namespace currently exposed." 299 300 return (self.namespaces[-1:] or [self])[0] 301 302 def use_name(self, name): 303 304 "Use the given 'name' within the current namespace/unit." 305 306 unit = self.get_namespace() 307 self.importer.use_name(name, unit.name) 308 309 # Attribute usage methods. 310 # These are convenience methods which refer to the specific namespace's 311 # implementation of these operations. 312 313 def new_branchpoint(self): 314 self.get_namespace()._new_branchpoint() 315 316 def new_branch(self): 317 self.get_namespace()._new_branch() 318 319 def shelve_branch(self): 320 self.get_namespace()._shelve_branch() 321 322 def merge_branches(self): 323 self.get_namespace()._merge_branches() 324 325 def define_attribute_user(self, node): 326 327 """ 328 Define 'node' as the user of attributes, indicating the point where the 329 user is defined. 330 """ 331 332 self.get_namespace()._define_attribute_user(node) 333 334 def reset_all_attributes(self): 335 self.get_namespace()._reset_all_attributes() 336 337 def use_attribute(self, attr, attrname): 338 return self.get_namespace()._use_attribute(attr, attrname) 339 340 # Visitor methods. 341 342 def default(self, node, *args): 343 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 344 345 def dispatch(self, node, *args): 346 return ASTVisitor.dispatch(self, node, *args) 347 348 def NOP(self, node): 349 for n in node.getChildNodes(): 350 self.dispatch(n) 351 return None 352 353 def OP(self, node): 354 for n in node.getChildNodes(): 355 self.dispatch(n) 356 return Instance() 357 358 def _visitUnary(self, node): 359 360 "Accounting method for the unary operator 'node'." 361 362 method = unary_methods[node.__class__.__name__] 363 self.use_name(method) 364 return self.OP(node) 365 366 def _visitBinary(self, node): 367 368 "Accounting method for the binary operator 'node'." 369 370 left_method, right_method = binary_methods[node.__class__.__name__] 371 self.use_name(left_method) 372 self.use_name(right_method) 373 return self.OP(node) 374 375 def _visitFunction(self, node, name): 376 377 """ 378 Return a function object for the function defined by 'node' with the 379 given 'name'. If a lambda expression is being visited, 'name' should be 380 None. 381 """ 382 383 # Define the function object. 384 385 function = Function( 386 name, 387 self.get_namespace(), 388 node.argnames, 389 node.defaults, 390 (node.flags & 4 != 0), 391 (node.flags & 8 != 0), 392 self, 393 node 394 ) 395 396 self.add_object(function, any_scope=1) 397 398 # Make a back reference from the node for code generation. 399 400 node.unit = function 401 402 # Process the defaults. 403 404 for n in node.defaults: 405 self.expr = self.dispatch(n) 406 function.store_default(self.expr) 407 408 self.functions.append((node, self.namespaces + [function])) 409 410 if name is not None: 411 self.store(name, function) 412 else: 413 self.store_lambda(function) 414 415 # Where defaults exist, an instance needs creating. Thus, it makes 416 # no sense to return a reference to the function here, since the 417 # recipient will not be referencing the function itself. 418 419 if node.defaults: 420 return Instance() # indicates no known target 421 422 return function 423 424 def _visitFunctionBody(self, node, namespaces): 425 426 "Enter the function." 427 428 # Current namespace is the function. 429 # Previous namespace is the class. 430 431 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 432 if namespaces[-1].name == "__init__": 433 self.in_init = 1 434 self.in_method = 1 435 436 self.namespaces = namespaces 437 self.dispatch(node.code) 438 439 self.in_init = 0 440 self.in_method = 0 441 442 # Specific handler methods. 443 444 visitAdd = _visitBinary 445 446 visitAnd = OP 447 448 visitAssert = NOP 449 450 def visitAssign(self, node): 451 self.expr = self.dispatch(node.expr) 452 for n in node.nodes: 453 self.dispatch(n) 454 return None 455 456 def visitAssAttr(self, node): 457 expr = self.dispatch(node.expr) 458 459 # Record the attribute on the presumed target. 460 461 if isinstance(expr, Attr): 462 if expr.name == "self": 463 if not self.store_class_attr(node.attrname): 464 self.store_instance_attr(node.attrname) 465 elif isinstance(expr.get_value(), Module): 466 self.store_module_attr(node.attrname, expr.get_value()) 467 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 468 469 # Note usage of the attribute. 470 471 if expr.parent is self.get_namespace(): 472 node._attrnames = self.use_attribute(expr, node.attrname) 473 474 return None 475 476 def visitAssList(self, node): 477 478 # Declare names which will be used by generated code. 479 480 self.use_name("__getitem__") 481 482 # Process the assignment. 483 484 for i, n in enumerate(node.nodes): 485 self.dispatch(n) 486 self.importer.make_constant(i) # for __getitem__(i) at run-time 487 return None 488 489 def visitAssName(self, node): 490 if node.flags == "OP_DELETE": 491 raise InspectError(self.full_name(), node, "Deletion of attribute %r is not supported." % node.name) 492 493 self.store(node.name, self.expr) 494 self.define_attribute_user(node) 495 self.use_name(node.name) 496 return None 497 498 visitAssTuple = visitAssList 499 500 def visitAugAssign(self, node): 501 502 # Accounting. 503 504 aug_method, (left_method, right_method) = augassign_methods[node.op] 505 self.use_name(aug_method) 506 self.use_name(left_method) 507 self.use_name(right_method) 508 509 # Process the assignment. 510 511 self.expr = self.dispatch(node.expr) 512 513 # NOTE: Similar to micropython.ast handler code. 514 # NOTE: Slices and subscripts not supported. 515 516 if isinstance(node.node, compiler.ast.Name): 517 self.visitAssName(node.node) 518 elif isinstance(node.node, compiler.ast.Getattr): 519 self.visitAssAttr(node.node) 520 else: 521 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 522 523 return None 524 525 visitBackquote = OP 526 527 visitBitand = _visitBinary 528 529 visitBitor = _visitBinary 530 531 visitBitxor = _visitBinary 532 533 def visitBreak(self, node): 534 self.reset_all_attributes() 535 536 visitCallFunc = OP 537 538 def visitClass(self, node): 539 540 """ 541 Register the class at the given 'node' subject to the restrictions 542 mentioned in the module docstring. 543 """ 544 545 if self.namespaces: 546 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 547 return None 548 else: 549 cls = Class(node.name, self.get_namespace(), self, node) 550 551 # Visit the base class expressions, attempting to find concrete 552 # definitions of classes. 553 554 for base in node.bases: 555 expr = self.dispatch(base) 556 if isinstance(expr, Attr): 557 if expr.assignments != 1: 558 raise InspectError(self.full_name(), node, 559 "Base class %r for %r is not constant." % (base, cls.full_name())) 560 else: 561 cls.add_base(expr.get_value()) 562 else: # if expr is None: 563 raise InspectError(self.full_name(), node, 564 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 565 566 # NOTE: Potentially dubious measure to permit __init__ availability. 567 # If no bases exist, adopt the 'object' class. 568 569 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 570 expr = self.dispatch(compiler.ast.Name("object")) 571 cls.add_base(expr.get_value()) 572 573 # Make a back reference from the node for code generation. 574 575 node.unit = cls 576 577 # Make an entry for the class. 578 579 self.store(node.name, cls) 580 self.add_object(cls) 581 582 # Process the class body. 583 584 self.namespaces.append(cls) 585 self.dispatch(node.code) 586 self.namespaces.pop() 587 588 return cls 589 590 def visitCompare(self, node): 591 592 # Accounting. 593 # NOTE: Replicates some code in micropython.ast.visitCompare. 594 595 for op in node.ops: 596 op_name, next_node = op 597 methods = comparison_methods[op_name] 598 if methods is not None: 599 self.use_name(methods[0]) 600 self.use_name(methods[1]) 601 elif op_name.endswith("in"): 602 self.use_name("__contains__") 603 604 return self.OP(node) 605 606 def visitConst(self, node): 607 608 # Register the constant, if necessary, returning the resulting object. 609 610 self.use_name(self.importer.get_constant_type_name(node.value)) 611 return self.importer.make_constant(node.value) 612 613 def visitContinue(self, node): 614 self.reset_all_attributes() 615 616 visitDecorators = NOP 617 618 visitDict = OP 619 620 visitDiscard = NOP 621 622 visitDiv = _visitBinary 623 624 visitEllipsis = NOP 625 626 visitExec = NOP 627 628 visitExpression = OP 629 630 visitFloorDiv = _visitBinary 631 632 def visitFor(self, node): 633 self.new_branchpoint() 634 635 # Declare names which will be used by generated code. 636 637 self.use_name("__iter__") 638 self.use_name("next") 639 640 self.dispatch(node.assign) 641 self.dispatch(node.list) 642 643 # Enter the loop. 644 # Propagate attribute usage to branches. 645 646 self.in_loop = 1 647 self.new_branch() 648 self.dispatch(node.body) 649 self.shelve_branch() 650 self.in_loop = 0 651 652 # Maintain a branch for the else clause or the current retained usage 653 # where execution avoids the conditional clauses. 654 655 self.new_branch() 656 if node.else_ is not None: 657 self.dispatch(node.else_) 658 self.shelve_branch() 659 660 self.merge_branches() 661 return None 662 663 def visitFrom(self, node): 664 module = self.importer.load(node.modname, 1) 665 666 #if module is None: 667 # print "Warning:", node.modname, "not imported." 668 669 for name, alias in node.names: 670 if name != "*": 671 if module is not None and module.namespace.has_key(name): 672 attr = module[name] 673 self.store(alias or name, attr) 674 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 675 self.importer.load(attr.get_value().name) 676 677 # Support the import of names from missing modules. 678 679 else: 680 self.store(alias or name, UnresolvedName(name, node.modname, self)) 681 else: 682 if module is not None: 683 for n in module.namespace.keys(): 684 attr = module[n] 685 self.store(n, attr) 686 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 687 self.importer.load(attr.get_value().name) 688 689 return None 690 691 def visitFunction(self, node): 692 return self._visitFunction(node, node.name) 693 694 visitGenExpr = OP 695 696 visitGenExprFor = NOP 697 698 visitGenExprIf = NOP 699 700 visitGenExprInner = NOP 701 702 def visitGetattr(self, node): 703 expr = self.dispatch(node.expr) 704 attrname = node.attrname 705 706 # Attempt to identify the nature of the attribute. 707 708 if isinstance(expr, Attr): 709 value = expr.get_value() 710 if isinstance(value, (Class, Module)): 711 attr = value.namespace.get(attrname) 712 elif isinstance(value, UnresolvedName): 713 attr = UnresolvedName(attrname, value.full_name(), self) 714 else: 715 attr = None 716 717 # Note usage of the attribute. 718 719 if expr.parent is self.get_namespace(): 720 node._attrnames = self.use_attribute(expr, attrname) 721 722 elif self.builtins is not None: 723 attr = self.builtins.get(attrname) 724 else: 725 attr = UnresolvedName(attrname, value.full_name(), self) 726 727 # Accounting. 728 729 self.use_name(attrname) 730 731 return attr 732 733 def visitGlobal(self, node): 734 if self.namespaces: 735 for name in node.names: 736 ns = self.namespaces[-1] 737 if not ns.make_global(name): 738 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 739 740 # The name is recorded in an earlier process. 741 742 def visitIf(self, node): 743 self.new_branchpoint() 744 745 # Propagate attribute usage to branches. 746 747 for test, body in node.tests: 748 self.dispatch(test) 749 750 self.new_branch() 751 self.dispatch(body) 752 self.shelve_branch() 753 754 # Maintain a branch for the else clause or the current retained usage 755 # where execution avoids the conditional clauses. 756 757 self.new_branch() 758 if node.else_ is not None: 759 self.dispatch(node.else_) 760 self.shelve_branch() 761 762 self.merge_branches() 763 return None 764 765 visitIfExp = NOP 766 767 def visitImport(self, node): 768 for name, alias in node.names: 769 if alias is not None: 770 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 771 self.store(alias, module) 772 else: 773 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 774 self.store(name.split(".")[0], module) 775 776 return None 777 778 visitInvert = _visitUnary 779 780 def visitKeyword(self, node): 781 self.dispatch(node.expr) 782 self.importer.make_constant(node.name) 783 self.keyword_names.add(node.name) 784 return None 785 786 def visitLambda(self, node): 787 self.use_name(None) # lambda functions have no names but are assumed to be invoked 788 return self._visitFunction(node, None) 789 790 visitLeftShift = _visitBinary 791 792 visitList = OP 793 794 visitListComp = OP 795 796 visitListCompFor = NOP 797 798 visitListCompIf = NOP 799 800 visitMod = _visitBinary 801 802 def visitModule(self, node): 803 804 # Make a back reference from the node for code generation. 805 806 node.unit = self 807 return self.dispatch(node.node) 808 809 visitMul = _visitBinary 810 811 def visitName(self, node): 812 name = node.name 813 814 if self.importer.predefined_constants.has_key(name): 815 attr = self.importer.get_predefined_constant(name) 816 elif self.namespaces and self.namespaces[-1].has_key(name): 817 attr = self.namespaces[-1][name] 818 elif self.has_key(name): 819 attr = self[name] 820 elif self.builtins is not None and self.builtins.has_key(name): 821 attr = self.builtins[name] 822 else: 823 attr = None 824 825 self.use_name(name) 826 return attr 827 828 visitNot = OP 829 830 visitOr = OP 831 832 visitPass = NOP 833 834 visitPower = _visitBinary 835 836 visitPrint = NOP 837 838 visitPrintnl = NOP 839 840 visitRaise = NOP 841 842 visitReturn = NOP 843 844 visitRightShift = _visitBinary 845 846 visitSlice = OP 847 848 visitSliceobj = OP 849 850 def visitStmt(self, node): 851 for n in node.nodes: 852 self.dispatch(n) 853 return None 854 855 visitSub = _visitBinary 856 857 def visitSubscript(self, node): 858 self.use_name("__getitem__") 859 self.OP(node) 860 861 def visitTryExcept(self, node): 862 self.dispatch(node.body) 863 864 self.new_branchpoint() 865 866 for name, var, n in node.handlers: 867 self.new_branch() 868 869 # Establish the local for the handler. 870 871 if var is not None: 872 self.dispatch(var) 873 if n is not None: 874 self.dispatch(n) 875 876 self.shelve_branch() 877 878 if node.else_ is not None: 879 self.new_branch() 880 self.dispatch(node.else_) 881 self.shelve_branch() 882 883 self.merge_branches() 884 return None 885 886 visitTryFinally = NOP 887 888 visitTuple = OP 889 890 visitUnaryAdd = _visitUnary 891 892 visitUnarySub = _visitUnary 893 894 def visitWhile(self, node): 895 self.new_branchpoint() 896 897 # Propagate attribute usage to branches. 898 899 self.in_loop = 1 900 self.dispatch(node.test) 901 self.new_branch() 902 self.dispatch(node.body) 903 self.shelve_branch() 904 self.in_loop = 0 905 906 # Maintain a branch for the else clause or the current retained usage 907 # where execution avoids the conditional clauses. 908 909 self.new_branch() 910 if node.else_ is not None: 911 self.dispatch(node.else_) 912 self.shelve_branch() 913 914 self.merge_branches() 915 return None 916 917 visitWith = NOP 918 919 visitYield = NOP 920 921 # vim: tabstop=4 expandtab shiftwidth=4