1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 if delete_all or not self.importer.uses_name(name): 202 del obj[name] 203 if attr.assignments == 1: 204 value = attr.get_value() 205 206 # Delete any unambiguous attribute value. 207 208 if value is not obj and value in self.all_objects: 209 self.all_objects.remove(value) 210 211 # Delete class contents. 212 213 if isinstance(value, Class): 214 self.vacuum_object(value, 1) 215 216 def finalise(self): 217 218 "Finalise the module." 219 220 for obj in self.all_objects: 221 if isinstance(obj, (Class, Function)): 222 obj.finalise_attributes() 223 224 def add_object(self, obj, any_scope=0): 225 226 """ 227 Record 'obj' if non-local or if the optional 'any_scope' is set to a 228 true value. 229 """ 230 231 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 232 self.all_objects.add(obj) 233 234 # Optimisation tests. 235 236 def should_optimise_unused_objects(self): 237 return "unused_objects" in self.optimisations 238 239 # Namespace methods. 240 241 def store(self, name, obj): 242 243 "Record attribute or local 'name', storing 'obj'." 244 245 if not self.namespaces: 246 self.set(name, obj, not self.in_loop) 247 else: 248 self.namespaces[-1].set(name, obj, not self.in_loop) 249 250 def store_lambda(self, obj): 251 252 "Store a lambda function 'obj'." 253 254 self.add_object(obj) 255 256 def store_module_attr(self, name, module): 257 258 """ 259 Record module attribute 'name' in the given 'module' using the current 260 expression. 261 """ 262 263 module.set(name, self.expr, 0) 264 265 def store_class_attr(self, name): 266 267 """ 268 Record class attribute 'name' in the current class using the current 269 expression. 270 """ 271 272 if self.in_method and self.namespaces[-2].has_key(name): 273 self.namespaces[-2].set(name, self.expr, 0) 274 return 1 275 276 return 0 277 278 def store_instance_attr(self, name): 279 280 "Record instance attribute 'name' in the current class." 281 282 if self.in_method: 283 284 # Current namespace is the function. 285 # Previous namespace is the class. 286 287 self.namespaces[-2].add_instance_attribute(name) 288 289 def get_namespace(self): 290 291 "Return the parent (or most recent) namespace currently exposed." 292 293 return (self.namespaces[-1:] or [self])[0] 294 295 def use_name(self, name): 296 297 "Use the given 'name' within the current namespace/unit." 298 299 unit = self.get_namespace() 300 self.importer.use_name(name, unit.name) 301 302 def new_branchpoint(self): 303 self.get_namespace()._new_branchpoint() 304 305 def new_branch(self): 306 self.get_namespace()._new_branch() 307 308 def shelve_branch(self): 309 self.get_namespace()._shelve_branch() 310 311 def merge_branches(self): 312 self.get_namespace()._merge_branches() 313 314 def reset_attributes(self, name): 315 self.get_namespace()._reset_attributes(name) 316 317 def use_attribute(self, attr, attrname): 318 self.get_namespace()._use_attribute(attr, attrname) 319 320 # Visitor methods. 321 322 def default(self, node, *args): 323 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 324 325 def dispatch(self, node, *args): 326 return ASTVisitor.dispatch(self, node, *args) 327 328 def NOP(self, node): 329 for n in node.getChildNodes(): 330 self.dispatch(n) 331 return None 332 333 def OP(self, node): 334 for n in node.getChildNodes(): 335 self.dispatch(n) 336 return Instance() 337 338 def _visitUnary(self, node): 339 340 "Accounting method for the unary operator 'node'." 341 342 method = unary_methods[node.__class__.__name__] 343 self.use_name(method) 344 return self.OP(node) 345 346 def _visitBinary(self, node): 347 348 "Accounting method for the binary operator 'node'." 349 350 left_method, right_method = binary_methods[node.__class__.__name__] 351 self.use_name(left_method) 352 self.use_name(right_method) 353 return self.OP(node) 354 355 def _visitFunction(self, node, name): 356 357 """ 358 Return a function object for the function defined by 'node' with the 359 given 'name'. If a lambda expression is being visited, 'name' should be 360 None. 361 """ 362 363 # Define the function object. 364 365 function = Function( 366 name, 367 self.get_namespace(), 368 node.argnames, 369 node.defaults, 370 (node.flags & 4 != 0), 371 (node.flags & 8 != 0), 372 self, 373 node 374 ) 375 376 self.add_object(function, any_scope=1) 377 378 # Make a back reference from the node for code generation. 379 380 node.unit = function 381 382 # Process the defaults. 383 384 for n in node.defaults: 385 self.expr = self.dispatch(n) 386 function.store_default(self.expr) 387 388 self.functions.append((node, self.namespaces + [function])) 389 390 if name is not None: 391 self.store(name, function) 392 else: 393 self.store_lambda(function) 394 395 # Where defaults exist, an instance needs creating. Thus, it makes 396 # no sense to return a reference to the function here, since the 397 # recipient will not be referencing the function itself. 398 399 if node.defaults: 400 return Instance() # indicates no known target 401 402 return function 403 404 def _visitFunctionBody(self, node, namespaces): 405 406 "Enter the function." 407 408 # Current namespace is the function. 409 # Previous namespace is the class. 410 411 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 412 if namespaces[-1].name == "__init__": 413 self.in_init = 1 414 self.in_method = 1 415 416 self.namespaces = namespaces 417 self.dispatch(node.code) 418 419 self.in_init = 0 420 self.in_method = 0 421 422 # Specific handler methods. 423 424 visitAdd = _visitBinary 425 426 visitAnd = OP 427 428 visitAssert = NOP 429 430 def visitAssign(self, node): 431 self.expr = self.dispatch(node.expr) 432 for n in node.nodes: 433 self.dispatch(n) 434 return None 435 436 def visitAssAttr(self, node): 437 expr = self.dispatch(node.expr) 438 439 # Record the attribute on the presumed target. 440 441 if isinstance(expr, Attr): 442 if expr.name == "self": 443 if not self.store_class_attr(node.attrname): 444 self.store_instance_attr(node.attrname) 445 elif isinstance(expr.get_value(), Module): 446 self.store_module_attr(node.attrname, expr.get_value()) 447 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 448 449 # Note usage of the attribute. 450 451 if expr.parent is self.get_namespace(): 452 self.use_attribute(expr, node.attrname) 453 454 return None 455 456 def visitAssList(self, node): 457 458 # Declare names which will be used by generated code. 459 460 self.use_name("__getitem__") 461 462 # Process the assignment. 463 464 for i, n in enumerate(node.nodes): 465 self.dispatch(n) 466 self.importer.make_constant(i) # for __getitem__(i) at run-time 467 return None 468 469 def visitAssName(self, node): 470 if node.flags == "OP_DELETE": 471 raise InspectError(self.full_name(), node, "Deletion of attribute %r is not supported." % node.name) 472 473 self.store(node.name, self.expr) 474 self.reset_attributes(node.name) 475 self.use_name(node.name) 476 return None 477 478 visitAssTuple = visitAssList 479 480 def visitAugAssign(self, node): 481 482 # Accounting. 483 484 aug_method, (left_method, right_method) = augassign_methods[node.op] 485 self.use_name(aug_method) 486 self.use_name(left_method) 487 self.use_name(right_method) 488 489 # Process the assignment. 490 491 self.expr = self.dispatch(node.expr) 492 493 # NOTE: Similar to micropython.ast handler code. 494 # NOTE: Slices and subscripts not supported. 495 496 if isinstance(node.node, compiler.ast.Name): 497 self.visitAssName(node.node) 498 elif isinstance(node.node, compiler.ast.Getattr): 499 self.visitAssAttr(node.node) 500 else: 501 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 502 503 return None 504 505 visitBackquote = OP 506 507 visitBitand = _visitBinary 508 509 visitBitor = _visitBinary 510 511 visitBitxor = _visitBinary 512 513 visitBreak = NOP 514 515 visitCallFunc = OP 516 517 def visitClass(self, node): 518 519 """ 520 Register the class at the given 'node' subject to the restrictions 521 mentioned in the module docstring. 522 """ 523 524 if self.namespaces: 525 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 526 return None 527 else: 528 cls = Class(node.name, self.get_namespace(), self, node) 529 530 # Visit the base class expressions, attempting to find concrete 531 # definitions of classes. 532 533 for base in node.bases: 534 expr = self.dispatch(base) 535 if isinstance(expr, Attr): 536 if expr.assignments != 1: 537 raise InspectError(self.full_name(), node, 538 "Base class %r for %r is not constant." % (base, cls.full_name())) 539 else: 540 cls.add_base(expr.get_value()) 541 else: # if expr is None: 542 raise InspectError(self.full_name(), node, 543 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 544 545 # NOTE: Potentially dubious measure to permit __init__ availability. 546 # If no bases exist, adopt the 'object' class. 547 548 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 549 expr = self.dispatch(compiler.ast.Name("object")) 550 cls.add_base(expr.get_value()) 551 552 # Make a back reference from the node for code generation. 553 554 node.unit = cls 555 556 # Make an entry for the class. 557 558 self.store(node.name, cls) 559 self.add_object(cls) 560 561 # Process the class body. 562 563 self.namespaces.append(cls) 564 self.dispatch(node.code) 565 self.namespaces.pop() 566 567 return cls 568 569 def visitCompare(self, node): 570 571 # Accounting. 572 # NOTE: Replicates some code in micropython.ast.visitCompare. 573 574 for op in node.ops: 575 op_name, next_node = op 576 methods = comparison_methods[op_name] 577 if methods is not None: 578 self.use_name(methods[0]) 579 self.use_name(methods[1]) 580 elif op_name.endswith("in"): 581 self.use_name("__contains__") 582 583 return self.OP(node) 584 585 def visitConst(self, node): 586 587 # Register the constant, if necessary, returning the resulting object. 588 589 self.use_name(self.importer.get_constant_type_name(node.value)) 590 return self.importer.make_constant(node.value) 591 592 visitContinue = NOP 593 594 visitDecorators = NOP 595 596 visitDict = OP 597 598 visitDiscard = NOP 599 600 visitDiv = _visitBinary 601 602 visitEllipsis = NOP 603 604 visitExec = NOP 605 606 visitExpression = OP 607 608 visitFloorDiv = _visitBinary 609 610 def visitFor(self, node): 611 self.new_branchpoint() 612 613 # Declare names which will be used by generated code. 614 615 self.use_name("__iter__") 616 self.use_name("next") 617 618 self.dispatch(node.assign) 619 self.dispatch(node.list) 620 621 # Enter the loop. 622 623 self.in_loop = 1 624 self.new_branch() 625 self.dispatch(node.body) 626 self.shelve_branch() 627 self.in_loop = 0 628 629 if node.else_ is not None: 630 self.new_branch() 631 self.dispatch(node.else_) 632 self.shelve_branch() 633 634 self.merge_branches() 635 return None 636 637 def visitFrom(self, node): 638 module = self.importer.load(node.modname, 1) 639 640 #if module is None: 641 # print "Warning:", node.modname, "not imported." 642 643 for name, alias in node.names: 644 if name != "*": 645 if module is not None and module.namespace.has_key(name): 646 attr = module[name] 647 self.store(alias or name, attr) 648 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 649 self.importer.load(attr.get_value().name) 650 651 # Support the import of names from missing modules. 652 653 else: 654 self.store(alias or name, UnresolvedName(name, node.modname, self)) 655 else: 656 if module is not None: 657 for n in module.namespace.keys(): 658 attr = module[n] 659 self.store(n, attr) 660 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 661 self.importer.load(attr.get_value().name) 662 663 return None 664 665 def visitFunction(self, node): 666 return self._visitFunction(node, node.name) 667 668 visitGenExpr = OP 669 670 visitGenExprFor = NOP 671 672 visitGenExprIf = NOP 673 674 visitGenExprInner = NOP 675 676 def visitGetattr(self, node): 677 expr = self.dispatch(node.expr) 678 attrname = node.attrname 679 680 # Attempt to identify the nature of the attribute. 681 682 if isinstance(expr, Attr): 683 value = expr.get_value() 684 if isinstance(value, (Class, Module)): 685 attr = value.namespace.get(attrname) 686 elif isinstance(value, UnresolvedName): 687 attr = UnresolvedName(attrname, value.full_name(), self) 688 else: 689 attr = None 690 691 # Note usage of the attribute. 692 693 if expr.parent is self.get_namespace(): 694 self.use_attribute(expr, attrname) 695 696 elif self.builtins is not None: 697 attr = self.builtins.get(attrname) 698 else: 699 attr = UnresolvedName(attrname, value.full_name(), self) 700 701 # Accounting. 702 703 self.use_name(attrname) 704 705 return attr 706 707 def visitGlobal(self, node): 708 if self.namespaces: 709 for name in node.names: 710 ns = self.namespaces[-1] 711 if not ns.make_global(name): 712 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 713 714 # The name is recorded in an earlier process. 715 716 def visitIf(self, node): 717 self.new_branchpoint() 718 719 for test, body in node.tests: 720 self.dispatch(test) 721 722 self.new_branch() 723 self.dispatch(body) 724 self.shelve_branch() 725 726 if node.else_ is not None: 727 self.new_branch() 728 self.dispatch(node.else_) 729 self.shelve_branch() 730 731 self.merge_branches() 732 return None 733 734 visitIfExp = NOP 735 736 def visitImport(self, node): 737 for name, alias in node.names: 738 if alias is not None: 739 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 740 self.store(alias, module) 741 else: 742 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 743 self.store(name.split(".")[0], module) 744 745 return None 746 747 visitInvert = _visitUnary 748 749 def visitKeyword(self, node): 750 self.dispatch(node.expr) 751 self.importer.make_constant(node.name) 752 self.keyword_names.add(node.name) 753 return None 754 755 def visitLambda(self, node): 756 self.use_name(None) # lambda functions have no names but are assumed to be invoked 757 return self._visitFunction(node, None) 758 759 visitLeftShift = _visitBinary 760 761 visitList = OP 762 763 visitListComp = OP 764 765 visitListCompFor = NOP 766 767 visitListCompIf = NOP 768 769 visitMod = _visitBinary 770 771 def visitModule(self, node): 772 773 # Make a back reference from the node for code generation. 774 775 node.unit = self 776 return self.dispatch(node.node) 777 778 visitMul = _visitBinary 779 780 def visitName(self, node): 781 name = node.name 782 783 if self.importer.predefined_constants.has_key(name): 784 attr = self.importer.get_predefined_constant(name) 785 elif self.namespaces and self.namespaces[-1].has_key(name): 786 attr = self.namespaces[-1][name] 787 elif self.has_key(name): 788 attr = self[name] 789 elif self.builtins is not None and self.builtins.has_key(name): 790 attr = self.builtins[name] 791 else: 792 attr = None 793 794 self.use_name(name) 795 return attr 796 797 visitNot = OP 798 799 visitOr = OP 800 801 visitPass = NOP 802 803 visitPower = _visitBinary 804 805 visitPrint = NOP 806 807 visitPrintnl = NOP 808 809 visitRaise = NOP 810 811 visitReturn = NOP 812 813 visitRightShift = _visitBinary 814 815 visitSlice = OP 816 817 visitSliceobj = OP 818 819 def visitStmt(self, node): 820 for n in node.nodes: 821 self.dispatch(n) 822 return None 823 824 visitSub = _visitBinary 825 826 def visitSubscript(self, node): 827 self.use_name("__getitem__") 828 self.OP(node) 829 830 def visitTryExcept(self, node): 831 self.dispatch(node.body) 832 833 self.new_branchpoint() 834 835 for name, var, n in node.handlers: 836 self.new_branch() 837 838 # Establish the local for the handler. 839 840 if var is not None: 841 self.dispatch(var) 842 if n is not None: 843 self.dispatch(n) 844 845 self.shelve_branch() 846 847 if node.else_ is not None: 848 self.new_branch() 849 self.dispatch(node.else_) 850 self.shelve_branch() 851 852 self.merge_branches() 853 return None 854 855 visitTryFinally = NOP 856 857 visitTuple = OP 858 859 visitUnaryAdd = _visitUnary 860 861 visitUnarySub = _visitUnary 862 863 def visitWhile(self, node): 864 self.new_branchpoint() 865 866 self.in_loop = 1 867 self.dispatch(node.test) 868 self.new_branch() 869 self.dispatch(node.body) 870 self.shelve_branch() 871 self.in_loop = 0 872 873 if node.else_ is not None: 874 self.new_branch() 875 self.dispatch(node.else_) 876 self.shelve_branch() 877 878 self.merge_branches() 879 return None 880 881 visitWith = NOP 882 883 visitYield = NOP 884 885 # vim: tabstop=4 expandtab shiftwidth=4