1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 202 # Only consider deleting entire unused objects or things accessible 203 # via names which are never used. 204 205 if delete_all or not self.importer.uses_name(name): 206 del obj[name] 207 208 # Delete any unambiguous attribute value. Such values can only 209 # have been defined within the object and therefore are not 210 # redefined by other code regions. 211 212 if attr.assignments == 1: 213 value = attr.get_value() 214 215 if value is not obj and value in self.all_objects: 216 self.all_objects.remove(value) 217 218 # Delete class contents. 219 220 if isinstance(value, Class): 221 self.vacuum_object(value, 1) 222 223 def finalise(self): 224 225 "Finalise the module." 226 227 for obj in self.all_objects: 228 if isinstance(obj, (Class, Function)): 229 obj.finalise_attributes() 230 231 def add_object(self, obj, any_scope=0): 232 233 """ 234 Record 'obj' if non-local or if the optional 'any_scope' is set to a 235 true value. 236 """ 237 238 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 239 self.all_objects.add(obj) 240 241 # Optimisation tests. 242 243 def should_optimise_unused_objects(self): 244 return "unused_objects" in self.optimisations 245 246 # Namespace methods. 247 248 def store(self, name, obj): 249 250 "Record attribute or local 'name', storing 'obj'." 251 252 if not self.namespaces: 253 self.set(name, obj, not self.in_loop) 254 else: 255 self.namespaces[-1].set(name, obj, not self.in_loop) 256 257 def store_lambda(self, obj): 258 259 "Store a lambda function 'obj'." 260 261 self.add_object(obj) 262 263 def store_module_attr(self, name, module): 264 265 """ 266 Record module attribute 'name' in the given 'module' using the current 267 expression. 268 """ 269 270 module.set(name, self.expr, 0) 271 272 def store_class_attr(self, name): 273 274 """ 275 Record class attribute 'name' in the current class using the current 276 expression. 277 """ 278 279 if self.in_method and self.namespaces[-2].has_key(name): 280 self.namespaces[-2].set(name, self.expr, 0) 281 return 1 282 283 return 0 284 285 def store_instance_attr(self, name): 286 287 "Record instance attribute 'name' in the current class." 288 289 if self.in_method: 290 291 # Current namespace is the function. 292 # Previous namespace is the class. 293 294 self.namespaces[-2].add_instance_attribute(name) 295 296 def get_namespace(self): 297 298 "Return the parent (or most recent) namespace currently exposed." 299 300 return (self.namespaces[-1:] or [self])[0] 301 302 def use_name(self, name): 303 304 "Use the given 'name' within the current namespace/unit." 305 306 unit = self.get_namespace() 307 self.importer.use_name(name, unit.name) 308 309 def new_branchpoint(self): 310 self.get_namespace()._new_branchpoint() 311 312 def new_branch(self): 313 self.get_namespace()._new_branch() 314 315 def shelve_branch(self): 316 self.get_namespace()._shelve_branch() 317 318 def merge_branches(self): 319 self.get_namespace()._merge_branches() 320 321 def reset_attributes(self, name): 322 self.get_namespace()._reset_attributes(name) 323 324 def reset_all_attributes(self): 325 self.get_namespace()._reset_all_attributes() 326 327 def use_attribute(self, attr, attrname): 328 return self.get_namespace()._use_attribute(attr, attrname) 329 330 # Visitor methods. 331 332 def default(self, node, *args): 333 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 334 335 def dispatch(self, node, *args): 336 return ASTVisitor.dispatch(self, node, *args) 337 338 def NOP(self, node): 339 for n in node.getChildNodes(): 340 self.dispatch(n) 341 return None 342 343 def OP(self, node): 344 for n in node.getChildNodes(): 345 self.dispatch(n) 346 return Instance() 347 348 def _visitUnary(self, node): 349 350 "Accounting method for the unary operator 'node'." 351 352 method = unary_methods[node.__class__.__name__] 353 self.use_name(method) 354 return self.OP(node) 355 356 def _visitBinary(self, node): 357 358 "Accounting method for the binary operator 'node'." 359 360 left_method, right_method = binary_methods[node.__class__.__name__] 361 self.use_name(left_method) 362 self.use_name(right_method) 363 return self.OP(node) 364 365 def _visitFunction(self, node, name): 366 367 """ 368 Return a function object for the function defined by 'node' with the 369 given 'name'. If a lambda expression is being visited, 'name' should be 370 None. 371 """ 372 373 # Define the function object. 374 375 function = Function( 376 name, 377 self.get_namespace(), 378 node.argnames, 379 node.defaults, 380 (node.flags & 4 != 0), 381 (node.flags & 8 != 0), 382 self, 383 node 384 ) 385 386 self.add_object(function, any_scope=1) 387 388 # Make a back reference from the node for code generation. 389 390 node.unit = function 391 392 # Process the defaults. 393 394 for n in node.defaults: 395 self.expr = self.dispatch(n) 396 function.store_default(self.expr) 397 398 self.functions.append((node, self.namespaces + [function])) 399 400 if name is not None: 401 self.store(name, function) 402 else: 403 self.store_lambda(function) 404 405 # Where defaults exist, an instance needs creating. Thus, it makes 406 # no sense to return a reference to the function here, since the 407 # recipient will not be referencing the function itself. 408 409 if node.defaults: 410 return Instance() # indicates no known target 411 412 return function 413 414 def _visitFunctionBody(self, node, namespaces): 415 416 "Enter the function." 417 418 # Current namespace is the function. 419 # Previous namespace is the class. 420 421 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 422 if namespaces[-1].name == "__init__": 423 self.in_init = 1 424 self.in_method = 1 425 426 self.namespaces = namespaces 427 self.dispatch(node.code) 428 429 self.in_init = 0 430 self.in_method = 0 431 432 # Specific handler methods. 433 434 visitAdd = _visitBinary 435 436 visitAnd = OP 437 438 visitAssert = NOP 439 440 def visitAssign(self, node): 441 self.expr = self.dispatch(node.expr) 442 for n in node.nodes: 443 self.dispatch(n) 444 return None 445 446 def visitAssAttr(self, node): 447 expr = self.dispatch(node.expr) 448 449 # Record the attribute on the presumed target. 450 451 if isinstance(expr, Attr): 452 if expr.name == "self": 453 if not self.store_class_attr(node.attrname): 454 self.store_instance_attr(node.attrname) 455 elif isinstance(expr.get_value(), Module): 456 self.store_module_attr(node.attrname, expr.get_value()) 457 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 458 459 # Note usage of the attribute. 460 461 if expr.parent is self.get_namespace(): 462 node._attrnames = self.use_attribute(expr, node.attrname) 463 464 return None 465 466 def visitAssList(self, node): 467 468 # Declare names which will be used by generated code. 469 470 self.use_name("__getitem__") 471 472 # Process the assignment. 473 474 for i, n in enumerate(node.nodes): 475 self.dispatch(n) 476 self.importer.make_constant(i) # for __getitem__(i) at run-time 477 return None 478 479 def visitAssName(self, node): 480 if node.flags == "OP_DELETE": 481 raise InspectError(self.full_name(), node, "Deletion of attribute %r is not supported." % node.name) 482 483 self.store(node.name, self.expr) 484 self.reset_attributes(node.name) 485 self.use_name(node.name) 486 return None 487 488 visitAssTuple = visitAssList 489 490 def visitAugAssign(self, node): 491 492 # Accounting. 493 494 aug_method, (left_method, right_method) = augassign_methods[node.op] 495 self.use_name(aug_method) 496 self.use_name(left_method) 497 self.use_name(right_method) 498 499 # Process the assignment. 500 501 self.expr = self.dispatch(node.expr) 502 503 # NOTE: Similar to micropython.ast handler code. 504 # NOTE: Slices and subscripts not supported. 505 506 if isinstance(node.node, compiler.ast.Name): 507 self.visitAssName(node.node) 508 elif isinstance(node.node, compiler.ast.Getattr): 509 self.visitAssAttr(node.node) 510 else: 511 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 512 513 return None 514 515 visitBackquote = OP 516 517 visitBitand = _visitBinary 518 519 visitBitor = _visitBinary 520 521 visitBitxor = _visitBinary 522 523 def visitBreak(self, node): 524 self.reset_all_attributes() 525 526 visitCallFunc = OP 527 528 def visitClass(self, node): 529 530 """ 531 Register the class at the given 'node' subject to the restrictions 532 mentioned in the module docstring. 533 """ 534 535 if self.namespaces: 536 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 537 return None 538 else: 539 cls = Class(node.name, self.get_namespace(), self, node) 540 541 # Visit the base class expressions, attempting to find concrete 542 # definitions of classes. 543 544 for base in node.bases: 545 expr = self.dispatch(base) 546 if isinstance(expr, Attr): 547 if expr.assignments != 1: 548 raise InspectError(self.full_name(), node, 549 "Base class %r for %r is not constant." % (base, cls.full_name())) 550 else: 551 cls.add_base(expr.get_value()) 552 else: # if expr is None: 553 raise InspectError(self.full_name(), node, 554 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 555 556 # NOTE: Potentially dubious measure to permit __init__ availability. 557 # If no bases exist, adopt the 'object' class. 558 559 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 560 expr = self.dispatch(compiler.ast.Name("object")) 561 cls.add_base(expr.get_value()) 562 563 # Make a back reference from the node for code generation. 564 565 node.unit = cls 566 567 # Make an entry for the class. 568 569 self.store(node.name, cls) 570 self.add_object(cls) 571 572 # Process the class body. 573 574 self.namespaces.append(cls) 575 self.dispatch(node.code) 576 self.namespaces.pop() 577 578 return cls 579 580 def visitCompare(self, node): 581 582 # Accounting. 583 # NOTE: Replicates some code in micropython.ast.visitCompare. 584 585 for op in node.ops: 586 op_name, next_node = op 587 methods = comparison_methods[op_name] 588 if methods is not None: 589 self.use_name(methods[0]) 590 self.use_name(methods[1]) 591 elif op_name.endswith("in"): 592 self.use_name("__contains__") 593 594 return self.OP(node) 595 596 def visitConst(self, node): 597 598 # Register the constant, if necessary, returning the resulting object. 599 600 self.use_name(self.importer.get_constant_type_name(node.value)) 601 return self.importer.make_constant(node.value) 602 603 def visitContinue(self, node): 604 self.reset_all_attributes() 605 606 visitDecorators = NOP 607 608 visitDict = OP 609 610 visitDiscard = NOP 611 612 visitDiv = _visitBinary 613 614 visitEllipsis = NOP 615 616 visitExec = NOP 617 618 visitExpression = OP 619 620 visitFloorDiv = _visitBinary 621 622 def visitFor(self, node): 623 self.new_branchpoint() 624 625 # Declare names which will be used by generated code. 626 627 self.use_name("__iter__") 628 self.use_name("next") 629 630 self.dispatch(node.assign) 631 self.dispatch(node.list) 632 633 # Enter the loop. 634 # Propagate attribute usage to branches. 635 636 self.in_loop = 1 637 self.new_branch() 638 self.dispatch(node.body) 639 self.shelve_branch() 640 self.in_loop = 0 641 642 # Maintain a branch for the else clause or the current retained usage 643 # where execution avoids the conditional clauses. 644 645 self.new_branch() 646 if node.else_ is not None: 647 self.dispatch(node.else_) 648 self.shelve_branch() 649 650 self.merge_branches() 651 return None 652 653 def visitFrom(self, node): 654 module = self.importer.load(node.modname, 1) 655 656 #if module is None: 657 # print "Warning:", node.modname, "not imported." 658 659 for name, alias in node.names: 660 if name != "*": 661 if module is not None and module.namespace.has_key(name): 662 attr = module[name] 663 self.store(alias or name, attr) 664 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 665 self.importer.load(attr.get_value().name) 666 667 # Support the import of names from missing modules. 668 669 else: 670 self.store(alias or name, UnresolvedName(name, node.modname, self)) 671 else: 672 if module is not None: 673 for n in module.namespace.keys(): 674 attr = module[n] 675 self.store(n, attr) 676 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 677 self.importer.load(attr.get_value().name) 678 679 return None 680 681 def visitFunction(self, node): 682 return self._visitFunction(node, node.name) 683 684 visitGenExpr = OP 685 686 visitGenExprFor = NOP 687 688 visitGenExprIf = NOP 689 690 visitGenExprInner = NOP 691 692 def visitGetattr(self, node): 693 expr = self.dispatch(node.expr) 694 attrname = node.attrname 695 696 # Attempt to identify the nature of the attribute. 697 698 if isinstance(expr, Attr): 699 value = expr.get_value() 700 if isinstance(value, (Class, Module)): 701 attr = value.namespace.get(attrname) 702 elif isinstance(value, UnresolvedName): 703 attr = UnresolvedName(attrname, value.full_name(), self) 704 else: 705 attr = None 706 707 # Note usage of the attribute. 708 709 if expr.parent is self.get_namespace(): 710 node._attrnames = self.use_attribute(expr, attrname) 711 712 elif self.builtins is not None: 713 attr = self.builtins.get(attrname) 714 else: 715 attr = UnresolvedName(attrname, value.full_name(), self) 716 717 # Accounting. 718 719 self.use_name(attrname) 720 721 return attr 722 723 def visitGlobal(self, node): 724 if self.namespaces: 725 for name in node.names: 726 ns = self.namespaces[-1] 727 if not ns.make_global(name): 728 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 729 730 # The name is recorded in an earlier process. 731 732 def visitIf(self, node): 733 self.new_branchpoint() 734 735 # Propagate attribute usage to branches. 736 737 for test, body in node.tests: 738 self.dispatch(test) 739 740 self.new_branch() 741 self.dispatch(body) 742 self.shelve_branch() 743 744 # Maintain a branch for the else clause or the current retained usage 745 # where execution avoids the conditional clauses. 746 747 self.new_branch() 748 if node.else_ is not None: 749 self.dispatch(node.else_) 750 self.shelve_branch() 751 752 self.merge_branches() 753 return None 754 755 visitIfExp = NOP 756 757 def visitImport(self, node): 758 for name, alias in node.names: 759 if alias is not None: 760 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 761 self.store(alias, module) 762 else: 763 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 764 self.store(name.split(".")[0], module) 765 766 return None 767 768 visitInvert = _visitUnary 769 770 def visitKeyword(self, node): 771 self.dispatch(node.expr) 772 self.importer.make_constant(node.name) 773 self.keyword_names.add(node.name) 774 return None 775 776 def visitLambda(self, node): 777 self.use_name(None) # lambda functions have no names but are assumed to be invoked 778 return self._visitFunction(node, None) 779 780 visitLeftShift = _visitBinary 781 782 visitList = OP 783 784 visitListComp = OP 785 786 visitListCompFor = NOP 787 788 visitListCompIf = NOP 789 790 visitMod = _visitBinary 791 792 def visitModule(self, node): 793 794 # Make a back reference from the node for code generation. 795 796 node.unit = self 797 return self.dispatch(node.node) 798 799 visitMul = _visitBinary 800 801 def visitName(self, node): 802 name = node.name 803 804 if self.importer.predefined_constants.has_key(name): 805 attr = self.importer.get_predefined_constant(name) 806 elif self.namespaces and self.namespaces[-1].has_key(name): 807 attr = self.namespaces[-1][name] 808 elif self.has_key(name): 809 attr = self[name] 810 elif self.builtins is not None and self.builtins.has_key(name): 811 attr = self.builtins[name] 812 else: 813 attr = None 814 815 self.use_name(name) 816 return attr 817 818 visitNot = OP 819 820 visitOr = OP 821 822 visitPass = NOP 823 824 visitPower = _visitBinary 825 826 visitPrint = NOP 827 828 visitPrintnl = NOP 829 830 visitRaise = NOP 831 832 visitReturn = NOP 833 834 visitRightShift = _visitBinary 835 836 visitSlice = OP 837 838 visitSliceobj = OP 839 840 def visitStmt(self, node): 841 for n in node.nodes: 842 self.dispatch(n) 843 return None 844 845 visitSub = _visitBinary 846 847 def visitSubscript(self, node): 848 self.use_name("__getitem__") 849 self.OP(node) 850 851 def visitTryExcept(self, node): 852 self.dispatch(node.body) 853 854 self.new_branchpoint() 855 856 for name, var, n in node.handlers: 857 self.new_branch() 858 859 # Establish the local for the handler. 860 861 if var is not None: 862 self.dispatch(var) 863 if n is not None: 864 self.dispatch(n) 865 866 self.shelve_branch() 867 868 if node.else_ is not None: 869 self.new_branch() 870 self.dispatch(node.else_) 871 self.shelve_branch() 872 873 self.merge_branches() 874 return None 875 876 visitTryFinally = NOP 877 878 visitTuple = OP 879 880 visitUnaryAdd = _visitUnary 881 882 visitUnarySub = _visitUnary 883 884 def visitWhile(self, node): 885 self.new_branchpoint() 886 887 # Propagate attribute usage to branches. 888 889 self.in_loop = 1 890 self.dispatch(node.test) 891 self.new_branch() 892 self.dispatch(node.body) 893 self.shelve_branch() 894 self.in_loop = 0 895 896 # Maintain a branch for the else clause or the current retained usage 897 # where execution avoids the conditional clauses. 898 899 self.new_branch() 900 if node.else_ is not None: 901 self.dispatch(node.else_) 902 self.shelve_branch() 903 904 self.merge_branches() 905 return None 906 907 visitWith = NOP 908 909 visitYield = NOP 910 911 # vim: tabstop=4 expandtab shiftwidth=4