1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # First, visit module-level code, recording global names. 133 134 processed = self.dispatch(module) 135 136 # Then, for each function, detect and record globals declared in those 137 # functions. 138 139 for node, namespaces in self.functions: 140 self.process_globals(node) 141 142 # Then, visit each function, recording other names. 143 144 for node, namespaces in self.functions: 145 self._visitFunctionBody(node, namespaces) 146 147 # Add references to other modules declared using the __all__ global. 148 149 if self.has_key("__all__"): 150 all = self["__all__"] 151 if isinstance(all, compiler.ast.List): 152 for n in all.nodes: 153 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 154 155 return processed 156 157 def process_globals(self, node): 158 159 """ 160 Within the given 'node', process global declarations, adjusting the 161 module namespace. 162 """ 163 164 for n in node.getChildNodes(): 165 if isinstance(n, compiler.ast.Global): 166 for name in n.names: 167 if not self.has_key(name): 168 self[name] = None 169 else: 170 self.process_globals(n) 171 172 def vacuum(self): 173 174 """ 175 Vacuum the module namespace, removing unreferenced objects and unused 176 names. 177 """ 178 179 # Remove unreferenced objects. 180 181 if self.should_optimise_unused_objects(): 182 183 for name, value in self.items(): 184 185 # Remove entries for unreferenced objects. 186 # This, due to the nature of the referenced attribute, assumes 187 # that only explicitly mentioned classes and functions are 188 # employed in the final program. 189 190 if isinstance(value, Attr): 191 192 # Only remove entries for classes and functions, not methods. 193 194 for attr_value in value.get_values(): 195 if (isinstance(attr_value, Function) and not attr_value.is_method() or 196 isinstance(attr_value, Class)) and not attr_value.referenced: 197 pass 198 else: 199 break 200 else: 201 del self[name] 202 203 all_objects = list(self.all_objects) 204 205 for obj in all_objects: 206 207 # Only remove entries for classes and functions, not methods. 208 209 if (isinstance(obj, Function) and not obj.is_method() or 210 isinstance(obj, Class)) and not obj.referenced: 211 212 self.all_objects.remove(obj) 213 obj_was_removed = 1 214 215 else: 216 obj_was_removed = 0 217 218 # Remove unused entries from classes plus associated methods. 219 220 if isinstance(obj, Class): 221 for name, attr in obj.class_attributes().items(): 222 223 # In removed classes, all explicitly defined methods can 224 # be removed. 225 226 # In other classes, methods can only be removed if they 227 # are the only assigned object to the class for a 228 # particular attribute and are unreferenced. 229 230 if obj_was_removed and isinstance(attr.get_value(), Function) and attr.get_value().is_method() or \ 231 not self.importer.uses_name(name) and \ 232 attr.assignments == 1 and isinstance(attr.get_value(), Function) and \ 233 attr.get_value().is_method() and not attr.get_value().referenced: 234 235 method = attr.get_value() 236 237 if method in self.all_objects: 238 self.all_objects.remove(method) 239 del obj[name] 240 241 def finalise(self): 242 243 "Finalise the module." 244 245 for obj in self.all_objects: 246 if isinstance(obj, (Class, Function)): 247 obj.finalise_attributes() 248 249 def add_object(self, obj, any_scope=0): 250 251 """ 252 Record 'obj' if non-local or if the optional 'any_scope' is set to a 253 true value. 254 """ 255 256 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 257 self.all_objects.add(obj) 258 259 # Optimisation tests. 260 261 def should_optimise_unused_objects(self): 262 return "unused_objects" in self.optimisations 263 264 # Namespace methods. 265 266 def store(self, name, obj): 267 268 "Record attribute or local 'name', storing 'obj'." 269 270 if not self.namespaces: 271 self.set(name, obj, not self.in_loop) 272 else: 273 self.namespaces[-1].set(name, obj, not self.in_loop) 274 275 def store_lambda(self, obj): 276 277 "Store a lambda function 'obj'." 278 279 self.add_object(obj) 280 281 def store_module_attr(self, name, module): 282 283 """ 284 Record module attribute 'name' in the given 'module' using the current 285 expression. 286 """ 287 288 module.set(name, self.expr, 0) 289 290 def store_class_attr(self, name): 291 292 """ 293 Record class attribute 'name' in the current class using the current 294 expression. 295 """ 296 297 if self.in_method and self.namespaces[-2].has_key(name): 298 self.namespaces[-2].set(name, self.expr, 0) 299 return 1 300 301 return 0 302 303 def store_instance_attr(self, name): 304 305 "Record instance attribute 'name' in the current class." 306 307 if self.in_method: 308 309 # Current namespace is the function. 310 # Previous namespace is the class. 311 312 self.namespaces[-2].add_instance_attribute(name) 313 314 def get_parent(self): 315 316 "Return the parent (or most recent) namespace currently exposed." 317 318 return (self.namespaces[-1:] or [self])[0] 319 320 def use_name(self, name): 321 322 "Use the given 'name' within the current namespace/unit." 323 324 unit = self.get_parent() 325 self.importer.use_name(name, unit.name) 326 327 # Visitor methods. 328 329 def default(self, node, *args): 330 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 331 332 def dispatch(self, node, *args): 333 return ASTVisitor.dispatch(self, node, *args) 334 335 def NOP(self, node): 336 for n in node.getChildNodes(): 337 self.dispatch(n) 338 return None 339 340 def OP(self, node): 341 for n in node.getChildNodes(): 342 self.dispatch(n) 343 return Instance() 344 345 def _visitUnary(self, node): 346 347 "Accounting method for the unary operator 'node'." 348 349 method = unary_methods[node.__class__.__name__] 350 self.use_name(method) 351 return self.OP(node) 352 353 def _visitBinary(self, node): 354 355 "Accounting method for the binary operator 'node'." 356 357 left_method, right_method = binary_methods[node.__class__.__name__] 358 self.use_name(left_method) 359 self.use_name(right_method) 360 return self.OP(node) 361 362 def _visitFunction(self, node, name): 363 364 """ 365 Return a function object for the function defined by 'node' with the 366 given 'name'. If a lambda expression is being visited, 'name' should be 367 None. 368 """ 369 370 # Define the function object. 371 372 function = Function( 373 name, 374 self.get_parent(), 375 node.argnames, 376 node.defaults, 377 (node.flags & 4 != 0), 378 (node.flags & 8 != 0), 379 self, 380 node 381 ) 382 383 self.add_object(function, any_scope=1) 384 385 # Make a back reference from the node for code generation. 386 387 node.unit = function 388 389 # Process the defaults. 390 391 for n in node.defaults: 392 self.expr = self.dispatch(n) 393 function.store_default(self.expr) 394 395 self.functions.append((node, self.namespaces + [function])) 396 397 if name is not None: 398 self.store(name, function) 399 else: 400 self.store_lambda(function) 401 402 # Lambda functions are always assumed to be referenced. This is 403 # because other means of discovering the referencing of objects rely 404 # on the original names inherent in the definition of those objects. 405 406 function.set_referenced() 407 408 # Where defaults exist, an instance needs creating. Thus, it makes 409 # no sense to return a reference to the function here, since the 410 # recipient will not be referencing the function itself. 411 412 if node.defaults: 413 return Instance() # indicates no known target 414 415 return function 416 417 def _visitFunctionBody(self, node, namespaces): 418 419 "Enter the function." 420 421 # Current namespace is the function. 422 # Previous namespace is the class. 423 424 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 425 if namespaces[-1].name == "__init__": 426 self.in_init = 1 427 self.in_method = 1 428 429 self.namespaces = namespaces 430 self.dispatch(node.code) 431 432 self.in_init = 0 433 self.in_method = 0 434 435 # Specific handler methods. 436 437 visitAdd = _visitBinary 438 439 visitAnd = OP 440 441 visitAssert = NOP 442 443 def visitAssign(self, node): 444 self.expr = self.dispatch(node.expr) 445 for n in node.nodes: 446 self.dispatch(n) 447 return None 448 449 def visitAssAttr(self, node): 450 expr = self.dispatch(node.expr) 451 if isinstance(expr, Attr): 452 if expr.name == "self": 453 if not self.store_class_attr(node.attrname): 454 self.store_instance_attr(node.attrname) 455 elif isinstance(expr.get_value(), Module): 456 self.store_module_attr(node.attrname, expr.get_value()) 457 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 458 return None 459 460 def visitAssList(self, node): 461 462 # Declare names which will be used by generated code. 463 464 self.use_name("__getitem__") 465 466 # Process the assignment. 467 468 for i, n in enumerate(node.nodes): 469 self.dispatch(n) 470 self.importer.make_constant(i) # for __getitem__(i) at run-time 471 return None 472 473 def visitAssName(self, node): 474 self.store(node.name, self.expr) 475 return None 476 477 visitAssTuple = visitAssList 478 479 def visitAugAssign(self, node): 480 481 # Accounting. 482 483 aug_method, (left_method, right_method) = augassign_methods[node.op] 484 self.use_name(aug_method) 485 self.use_name(left_method) 486 self.use_name(right_method) 487 488 # Process the assignment. 489 490 self.expr = self.dispatch(node.expr) 491 492 # NOTE: Similar to micropython.ast handler code. 493 # NOTE: Slices and subscripts not supported. 494 495 if isinstance(node.node, compiler.ast.Name): 496 self.visitAssName(node.node) 497 elif isinstance(node.node, compiler.ast.Getattr): 498 self.visitAssAttr(node.node) 499 else: 500 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 501 502 return None 503 504 visitBackquote = OP 505 506 visitBitand = _visitBinary 507 508 visitBitor = _visitBinary 509 510 visitBitxor = _visitBinary 511 512 visitBreak = NOP 513 514 visitCallFunc = OP 515 516 def visitClass(self, node): 517 518 """ 519 Register the class at the given 'node' subject to the restrictions 520 mentioned in the module docstring. 521 """ 522 523 if self.namespaces: 524 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 525 return None 526 else: 527 cls = Class(node.name, self.get_parent(), self, node) 528 529 # Visit the base class expressions, attempting to find concrete 530 # definitions of classes. 531 532 for base in node.bases: 533 expr = self.dispatch(base) 534 if isinstance(expr, Attr): 535 if expr.assignments != 1: 536 raise InspectError(self.full_name(), node, 537 "Base class %r for %r is not constant." % (base, cls.full_name())) 538 else: 539 cls.add_base(expr.get_value()) 540 else: # if expr is None: 541 raise InspectError(self.full_name(), node, 542 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 543 544 # NOTE: Potentially dubious measure to permit __init__ availability. 545 # If no bases exist, adopt the 'object' class. 546 547 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 548 expr = self.dispatch(compiler.ast.Name("object")) 549 cls.add_base(expr.get_value()) 550 551 # Make a back reference from the node for code generation. 552 553 node.unit = cls 554 555 # Make an entry for the class. 556 557 self.store(node.name, cls) 558 self.add_object(cls) 559 560 # Process the class body. 561 562 self.namespaces.append(cls) 563 self.dispatch(node.code) 564 self.namespaces.pop() 565 566 return cls 567 568 def visitCompare(self, node): 569 570 # Accounting. 571 # NOTE: Replicates some code in micropython.ast.visitCompare. 572 573 for op in node.ops: 574 op_name, next_node = op 575 methods = comparison_methods[op_name] 576 if methods is not None: 577 self.use_name(methods[0]) 578 self.use_name(methods[1]) 579 elif op_name.endswith("in"): 580 self.use_name("__contains__") 581 582 return self.OP(node) 583 584 def visitConst(self, node): 585 586 # Register the constant, if necessary, returning the resulting object. 587 588 return self.importer.make_constant(node.value) 589 590 visitContinue = NOP 591 592 visitDecorators = NOP 593 594 visitDict = OP 595 596 visitDiscard = NOP 597 598 visitDiv = _visitBinary 599 600 visitEllipsis = NOP 601 602 visitExec = NOP 603 604 visitExpression = OP 605 606 visitFloorDiv = _visitBinary 607 608 def visitFor(self, node): 609 610 # Declare names which will be used by generated code. 611 612 self.use_name("__iter__") 613 self.use_name("next") 614 615 # Enter the loop. 616 617 self.in_loop = 1 618 self.NOP(node) 619 self.in_loop = 0 620 return None 621 622 def visitFrom(self, node): 623 module = self.importer.load(node.modname, 1) 624 625 if module is not None: 626 module.set_referenced() 627 628 #if module is None: 629 # print "Warning:", node.modname, "not imported." 630 631 for name, alias in node.names: 632 if name != "*": 633 if module is not None and module.namespace.has_key(name): 634 attr = module[name] 635 self.store(alias or name, attr) 636 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 637 self.importer.load(attr.get_value().name) 638 639 # Support the import of names from missing modules. 640 641 else: 642 self.store(alias or name, UnresolvedName(name, node.modname, self)) 643 else: 644 if module is not None: 645 for n in module.namespace.keys(): 646 attr = module[n] 647 self.store(n, attr) 648 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 649 self.importer.load(attr.get_value().name) 650 651 return None 652 653 def visitFunction(self, node): 654 return self._visitFunction(node, node.name) 655 656 visitGenExpr = OP 657 658 visitGenExprFor = NOP 659 660 visitGenExprIf = NOP 661 662 visitGenExprInner = NOP 663 664 def visitGetattr(self, node): 665 expr = self.dispatch(node.expr) 666 attrname = node.attrname 667 668 if isinstance(expr, Attr): 669 value = expr.get_value() 670 if isinstance(value, (Class, Module)): 671 attr = value.namespace.get(attrname) 672 elif isinstance(value, UnresolvedName): 673 attr = UnresolvedName(attrname, value.full_name(), self) 674 else: 675 attr = None 676 elif self.builtins is not None: 677 attr = self.builtins.get(attrname) 678 else: 679 attr = UnresolvedName(attrname, value.full_name(), self) 680 681 # Accounting. 682 683 if attr is not None: 684 attr.set_referenced() 685 686 self.use_name(attrname) 687 688 return attr 689 690 def visitGlobal(self, node): 691 if self.namespaces: 692 for name in node.names: 693 ns = self.namespaces[-1] 694 if not ns.make_global(name): 695 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 696 697 # The name is recorded in an earlier process. 698 699 def visitIf(self, node): 700 for test, body in node.tests: 701 self.dispatch(test) 702 self.dispatch(body) 703 if node.else_ is not None: 704 self.dispatch(node.else_) 705 return None 706 707 visitIfExp = NOP 708 709 def visitImport(self, node): 710 for name, alias in node.names: 711 if alias is not None: 712 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 713 self.store(alias, module) 714 else: 715 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 716 self.store(name.split(".")[0], module) 717 module.set_referenced() 718 719 return None 720 721 visitInvert = _visitUnary 722 723 def visitKeyword(self, node): 724 self.dispatch(node.expr) 725 self.importer.make_constant(node.name) 726 self.keyword_names.add(node.name) 727 return None 728 729 def visitLambda(self, node): 730 return self._visitFunction(node, None) 731 732 visitLeftShift = _visitBinary 733 734 visitList = OP 735 736 visitListComp = OP 737 738 visitListCompFor = NOP 739 740 visitListCompIf = NOP 741 742 visitMod = _visitBinary 743 744 def visitModule(self, node): 745 746 # Make a back reference from the node for code generation. 747 748 node.unit = self 749 return self.dispatch(node.node) 750 751 visitMul = _visitBinary 752 753 def visitName(self, node): 754 name = node.name 755 756 if self.importer.predefined_constants.has_key(name): 757 attr = self.importer.get_predefined_constant(name) 758 elif self.namespaces and self.namespaces[-1].has_key(name): 759 attr = self.namespaces[-1][name] 760 elif self.has_key(name): 761 attr = self[name] 762 elif self.builtins is not None and self.builtins.has_key(name): 763 attr = self.builtins[name] 764 else: 765 attr = None 766 767 # Accounting. 768 769 if attr is not None: 770 attr.set_referenced() 771 772 self.use_name(name) 773 774 return attr 775 776 visitNot = OP 777 778 visitOr = OP 779 780 visitPass = NOP 781 782 visitPower = _visitBinary 783 784 visitPrint = NOP 785 786 visitPrintnl = NOP 787 788 visitRaise = NOP 789 790 visitReturn = NOP 791 792 visitRightShift = _visitBinary 793 794 visitSlice = OP 795 796 visitSliceobj = OP 797 798 def visitStmt(self, node): 799 for n in node.nodes: 800 self.dispatch(n) 801 return None 802 803 visitSub = _visitBinary 804 805 def visitSubscript(self, node): 806 self.use_name("__getitem__") 807 self.OP(node) 808 809 def visitTryExcept(self, node): 810 self.dispatch(node.body) 811 for name, var, n in node.handlers: 812 813 # Establish the local for the handler. 814 815 if var is not None: 816 self.dispatch(var) 817 if n is not None: 818 self.dispatch(n) 819 if node.else_ is not None: 820 self.dispatch(node.else_) 821 return None 822 823 visitTryFinally = NOP 824 825 visitTuple = OP 826 827 visitUnaryAdd = _visitUnary 828 829 visitUnarySub = _visitUnary 830 831 def visitWhile(self, node): 832 self.in_loop = 1 833 self.NOP(node) 834 self.in_loop = 0 835 return None 836 837 visitWith = NOP 838 839 visitYield = NOP 840 841 # vim: tabstop=4 expandtab shiftwidth=4