1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 202 # Only consider deleting entire unused objects or things accessible 203 # via names which are never used. 204 205 if delete_all or not self.importer.uses_name(name): 206 del obj[name] 207 208 # Delete any unambiguous attribute value. Such values can only 209 # have been defined within the object and therefore are not 210 # redefined by other code regions. 211 212 if attr.assignments == 1: 213 value = attr.get_value() 214 215 if value is not obj and value in self.all_objects: 216 self.all_objects.remove(value) 217 218 # Delete class contents. 219 220 if isinstance(value, Class): 221 self.vacuum_object(value, 1) 222 223 def finalise(self): 224 225 "Finalise the module." 226 227 for obj in self.all_objects: 228 if isinstance(obj, (Class, Function)): 229 obj.finalise_attributes() 230 231 def add_object(self, obj, any_scope=0): 232 233 """ 234 Record 'obj' if non-local or if the optional 'any_scope' is set to a 235 true value. 236 """ 237 238 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 239 self.all_objects.add(obj) 240 241 # Optimisation tests. 242 243 def should_optimise_unused_objects(self): 244 return "unused_objects" in self.optimisations 245 246 # Namespace methods. 247 248 def store(self, name, obj): 249 250 "Record attribute or local 'name', storing 'obj'." 251 252 if not self.namespaces: 253 self.set(name, obj, not self.in_loop) 254 else: 255 self.namespaces[-1].set(name, obj, not self.in_loop) 256 257 def store_lambda(self, obj): 258 259 "Store a lambda function 'obj'." 260 261 self.add_object(obj) 262 263 def store_module_attr(self, name, module): 264 265 """ 266 Record module attribute 'name' in the given 'module' using the current 267 expression. 268 """ 269 270 module.set(name, self.expr, 0) 271 272 def store_class_attr(self, name): 273 274 """ 275 Record class attribute 'name' in the current class using the current 276 expression. 277 """ 278 279 if self.in_method and self.namespaces[-2].has_key(name): 280 self.namespaces[-2].set(name, self.expr, 0) 281 return 1 282 283 return 0 284 285 def store_instance_attr(self, name): 286 287 "Record instance attribute 'name' in the current class." 288 289 if self.in_method: 290 291 # Current namespace is the function. 292 # Previous namespace is the class. 293 294 self.namespaces[-2].add_instance_attribute(name) 295 296 def get_parent(self): 297 298 "Return the parent (or most recent) namespace currently exposed." 299 300 return (self.namespaces[-1:] or [self])[0] 301 302 def use_name(self, name): 303 304 "Use the given 'name' within the current namespace/unit." 305 306 unit = self.get_parent() 307 self.importer.use_name(name, unit.name) 308 309 # Visitor methods. 310 311 def default(self, node, *args): 312 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 313 314 def dispatch(self, node, *args): 315 return ASTVisitor.dispatch(self, node, *args) 316 317 def NOP(self, node): 318 for n in node.getChildNodes(): 319 self.dispatch(n) 320 return None 321 322 def OP(self, node): 323 for n in node.getChildNodes(): 324 self.dispatch(n) 325 return Instance() 326 327 def _visitUnary(self, node): 328 329 "Accounting method for the unary operator 'node'." 330 331 method = unary_methods[node.__class__.__name__] 332 self.use_name(method) 333 return self.OP(node) 334 335 def _visitBinary(self, node): 336 337 "Accounting method for the binary operator 'node'." 338 339 left_method, right_method = binary_methods[node.__class__.__name__] 340 self.use_name(left_method) 341 self.use_name(right_method) 342 return self.OP(node) 343 344 def _visitFunction(self, node, name): 345 346 """ 347 Return a function object for the function defined by 'node' with the 348 given 'name'. If a lambda expression is being visited, 'name' should be 349 None. 350 """ 351 352 # Define the function object. 353 354 function = Function( 355 name, 356 self.get_parent(), 357 node.argnames, 358 node.defaults, 359 (node.flags & 4 != 0), 360 (node.flags & 8 != 0), 361 self, 362 node 363 ) 364 365 self.add_object(function, any_scope=1) 366 367 # Make a back reference from the node for code generation. 368 369 node.unit = function 370 371 # Process the defaults. 372 373 for n in node.defaults: 374 self.expr = self.dispatch(n) 375 function.store_default(self.expr) 376 377 self.functions.append((node, self.namespaces + [function])) 378 379 if name is not None: 380 self.store(name, function) 381 else: 382 self.store_lambda(function) 383 384 # Where defaults exist, an instance needs creating. Thus, it makes 385 # no sense to return a reference to the function here, since the 386 # recipient will not be referencing the function itself. 387 388 if node.defaults: 389 return Instance() # indicates no known target 390 391 return function 392 393 def _visitFunctionBody(self, node, namespaces): 394 395 "Enter the function." 396 397 # Current namespace is the function. 398 # Previous namespace is the class. 399 400 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 401 if namespaces[-1].name == "__init__": 402 self.in_init = 1 403 self.in_method = 1 404 405 self.namespaces = namespaces 406 self.dispatch(node.code) 407 408 self.in_init = 0 409 self.in_method = 0 410 411 # Specific handler methods. 412 413 visitAdd = _visitBinary 414 415 visitAnd = OP 416 417 visitAssert = NOP 418 419 def visitAssign(self, node): 420 self.expr = self.dispatch(node.expr) 421 for n in node.nodes: 422 self.dispatch(n) 423 return None 424 425 def visitAssAttr(self, node): 426 expr = self.dispatch(node.expr) 427 428 # Record the attribute on the presumed target. 429 430 if isinstance(expr, Attr): 431 if expr.name == "self": 432 if not self.store_class_attr(node.attrname): 433 self.store_instance_attr(node.attrname) 434 elif isinstance(expr.get_value(), Module): 435 self.store_module_attr(node.attrname, expr.get_value()) 436 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 437 438 # Note usage of the attribute. 439 440 node._attrnames = expr.use_attribute(node.attrname) 441 442 return None 443 444 def visitAssList(self, node): 445 446 # Declare names which will be used by generated code. 447 448 self.use_name("__getitem__") 449 450 # Process the assignment. 451 452 for i, n in enumerate(node.nodes): 453 self.dispatch(n) 454 self.importer.make_constant(i) # for __getitem__(i) at run-time 455 return None 456 457 def visitAssName(self, node): 458 if node.flags == "OP_DELETE": 459 raise InspectError(self.full_name(), node, "Deletion of attribute %r is not supported." % node.name) 460 461 self.store(node.name, self.expr) 462 self.use_name(node.name) 463 return None 464 465 visitAssTuple = visitAssList 466 467 def visitAugAssign(self, node): 468 469 # Accounting. 470 471 aug_method, (left_method, right_method) = augassign_methods[node.op] 472 self.use_name(aug_method) 473 self.use_name(left_method) 474 self.use_name(right_method) 475 476 # Process the assignment. 477 478 self.expr = self.dispatch(node.expr) 479 480 # NOTE: Similar to micropython.ast handler code. 481 # NOTE: Slices and subscripts not supported. 482 483 if isinstance(node.node, compiler.ast.Name): 484 self.visitAssName(node.node) 485 elif isinstance(node.node, compiler.ast.Getattr): 486 self.visitAssAttr(node.node) 487 else: 488 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 489 490 return None 491 492 visitBackquote = OP 493 494 visitBitand = _visitBinary 495 496 visitBitor = _visitBinary 497 498 visitBitxor = _visitBinary 499 500 visitBreak = NOP 501 502 visitCallFunc = OP 503 504 def visitClass(self, node): 505 506 """ 507 Register the class at the given 'node' subject to the restrictions 508 mentioned in the module docstring. 509 """ 510 511 if self.namespaces: 512 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 513 return None 514 else: 515 cls = Class(node.name, self.get_parent(), self, node) 516 517 # Visit the base class expressions, attempting to find concrete 518 # definitions of classes. 519 520 for base in node.bases: 521 expr = self.dispatch(base) 522 if isinstance(expr, Attr): 523 if expr.assignments != 1: 524 raise InspectError(self.full_name(), node, 525 "Base class %r for %r is not constant." % (base, cls.full_name())) 526 else: 527 cls.add_base(expr.get_value()) 528 else: # if expr is None: 529 raise InspectError(self.full_name(), node, 530 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 531 532 # NOTE: Potentially dubious measure to permit __init__ availability. 533 # If no bases exist, adopt the 'object' class. 534 535 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 536 expr = self.dispatch(compiler.ast.Name("object")) 537 cls.add_base(expr.get_value()) 538 539 # Make a back reference from the node for code generation. 540 541 node.unit = cls 542 543 # Make an entry for the class. 544 545 self.store(node.name, cls) 546 self.add_object(cls) 547 548 # Process the class body. 549 550 self.namespaces.append(cls) 551 self.dispatch(node.code) 552 self.namespaces.pop() 553 554 return cls 555 556 def visitCompare(self, node): 557 558 # Accounting. 559 # NOTE: Replicates some code in micropython.ast.visitCompare. 560 561 for op in node.ops: 562 op_name, next_node = op 563 methods = comparison_methods[op_name] 564 if methods is not None: 565 self.use_name(methods[0]) 566 self.use_name(methods[1]) 567 elif op_name.endswith("in"): 568 self.use_name("__contains__") 569 570 return self.OP(node) 571 572 def visitConst(self, node): 573 574 # Register the constant, if necessary, returning the resulting object. 575 576 self.use_name(self.importer.get_constant_type_name(node.value)) 577 return self.importer.make_constant(node.value) 578 579 visitContinue = NOP 580 581 visitDecorators = NOP 582 583 visitDict = OP 584 585 visitDiscard = NOP 586 587 visitDiv = _visitBinary 588 589 visitEllipsis = NOP 590 591 visitExec = NOP 592 593 visitExpression = OP 594 595 visitFloorDiv = _visitBinary 596 597 def visitFor(self, node): 598 599 # Declare names which will be used by generated code. 600 601 self.use_name("__iter__") 602 self.use_name("next") 603 604 # Enter the loop. 605 606 self.in_loop = 1 607 self.NOP(node) 608 self.in_loop = 0 609 return None 610 611 def visitFrom(self, node): 612 module = self.importer.load(node.modname, 1) 613 614 #if module is None: 615 # print "Warning:", node.modname, "not imported." 616 617 for name, alias in node.names: 618 if name != "*": 619 if module is not None and module.namespace.has_key(name): 620 attr = module[name] 621 self.store(alias or name, attr) 622 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 623 self.importer.load(attr.get_value().name) 624 625 # Support the import of names from missing modules. 626 627 else: 628 self.store(alias or name, UnresolvedName(name, node.modname, self)) 629 else: 630 if module is not None: 631 for n in module.namespace.keys(): 632 attr = module[n] 633 self.store(n, attr) 634 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 635 self.importer.load(attr.get_value().name) 636 637 return None 638 639 def visitFunction(self, node): 640 return self._visitFunction(node, node.name) 641 642 visitGenExpr = OP 643 644 visitGenExprFor = NOP 645 646 visitGenExprIf = NOP 647 648 visitGenExprInner = NOP 649 650 def visitGetattr(self, node): 651 expr = self.dispatch(node.expr) 652 attrname = node.attrname 653 654 # Attempt to identify the nature of the attribute. 655 656 if isinstance(expr, Attr): 657 value = expr.get_value() 658 if isinstance(value, (Class, Module)): 659 attr = value.namespace.get(attrname) 660 elif isinstance(value, UnresolvedName): 661 attr = UnresolvedName(attrname, value.full_name(), self) 662 else: 663 attr = None 664 665 # Note usage of the attribute. 666 667 node._attrnames = expr.use_attribute(node.attrname) 668 669 elif self.builtins is not None: 670 attr = self.builtins.get(attrname) 671 else: 672 attr = UnresolvedName(attrname, value.full_name(), self) 673 674 # Accounting. 675 676 self.use_name(attrname) 677 678 return attr 679 680 def visitGlobal(self, node): 681 if self.namespaces: 682 for name in node.names: 683 ns = self.namespaces[-1] 684 if not ns.make_global(name): 685 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 686 687 # The name is recorded in an earlier process. 688 689 def visitIf(self, node): 690 for test, body in node.tests: 691 self.dispatch(test) 692 self.dispatch(body) 693 if node.else_ is not None: 694 self.dispatch(node.else_) 695 return None 696 697 visitIfExp = NOP 698 699 def visitImport(self, node): 700 for name, alias in node.names: 701 if alias is not None: 702 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 703 self.store(alias, module) 704 else: 705 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 706 self.store(name.split(".")[0], module) 707 708 return None 709 710 visitInvert = _visitUnary 711 712 def visitKeyword(self, node): 713 self.dispatch(node.expr) 714 self.importer.make_constant(node.name) 715 self.keyword_names.add(node.name) 716 return None 717 718 def visitLambda(self, node): 719 self.use_name(None) # lambda functions have no names but are assumed to be invoked 720 return self._visitFunction(node, None) 721 722 visitLeftShift = _visitBinary 723 724 visitList = OP 725 726 visitListComp = OP 727 728 visitListCompFor = NOP 729 730 visitListCompIf = NOP 731 732 visitMod = _visitBinary 733 734 def visitModule(self, node): 735 736 # Make a back reference from the node for code generation. 737 738 node.unit = self 739 return self.dispatch(node.node) 740 741 visitMul = _visitBinary 742 743 def visitName(self, node): 744 name = node.name 745 746 if self.importer.predefined_constants.has_key(name): 747 attr = self.importer.get_predefined_constant(name) 748 elif self.namespaces and self.namespaces[-1].has_key(name): 749 attr = self.namespaces[-1][name] 750 elif self.has_key(name): 751 attr = self[name] 752 elif self.builtins is not None and self.builtins.has_key(name): 753 attr = self.builtins[name] 754 else: 755 attr = None 756 757 self.use_name(name) 758 return attr 759 760 visitNot = OP 761 762 visitOr = OP 763 764 visitPass = NOP 765 766 visitPower = _visitBinary 767 768 visitPrint = NOP 769 770 visitPrintnl = NOP 771 772 visitRaise = NOP 773 774 visitReturn = NOP 775 776 visitRightShift = _visitBinary 777 778 visitSlice = OP 779 780 visitSliceobj = OP 781 782 def visitStmt(self, node): 783 for n in node.nodes: 784 self.dispatch(n) 785 return None 786 787 visitSub = _visitBinary 788 789 def visitSubscript(self, node): 790 self.use_name("__getitem__") 791 self.OP(node) 792 793 def visitTryExcept(self, node): 794 self.dispatch(node.body) 795 for name, var, n in node.handlers: 796 797 # Establish the local for the handler. 798 799 if var is not None: 800 self.dispatch(var) 801 if n is not None: 802 self.dispatch(n) 803 if node.else_ is not None: 804 self.dispatch(node.else_) 805 return None 806 807 visitTryFinally = NOP 808 809 visitTuple = OP 810 811 visitUnaryAdd = _visitUnary 812 813 visitUnarySub = _visitUnary 814 815 def visitWhile(self, node): 816 self.in_loop = 1 817 self.NOP(node) 818 self.in_loop = 0 819 return None 820 821 visitWith = NOP 822 823 visitYield = NOP 824 825 # vim: tabstop=4 expandtab shiftwidth=4