1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 202 # Only consider deleting entire unused objects or things accessible 203 # via names which are never used. 204 205 if delete_all or not self.importer.uses_name(name): 206 del obj[name] 207 208 # Delete any unambiguous attribute value. Such values can only 209 # have been defined within the object and therefore are not 210 # redefined by other code regions. 211 212 if attr.assignments == 1: 213 value = attr.get_value() 214 215 if value is not obj and value in self.all_objects: 216 self.all_objects.remove(value) 217 218 # Delete class contents. 219 220 if isinstance(value, Class): 221 self.vacuum_object(value, 1) 222 223 def finalise(self): 224 225 "Finalise the module." 226 227 for obj in self.all_objects: 228 if isinstance(obj, (Class, Function)): 229 obj.finalise_attributes() 230 231 def add_object(self, obj, any_scope=0): 232 233 """ 234 Record 'obj' if non-local or if the optional 'any_scope' is set to a 235 true value. 236 """ 237 238 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 239 self.all_objects.add(obj) 240 241 # Optimisation tests. 242 243 def should_optimise_unused_objects(self): 244 return "unused_objects" in self.optimisations 245 246 # Namespace methods. 247 248 def store(self, name, obj): 249 250 "Record attribute or local 'name', storing 'obj'." 251 252 if not self.namespaces: 253 self.set(name, obj, not self.in_loop) 254 else: 255 self.namespaces[-1].set(name, obj, not self.in_loop) 256 257 def store_lambda(self, obj): 258 259 "Store a lambda function 'obj'." 260 261 self.add_object(obj) 262 263 def store_module_attr(self, name, module): 264 265 """ 266 Record module attribute 'name' in the given 'module' using the current 267 expression. 268 """ 269 270 module.set(name, self.expr, 0) 271 272 def store_class_attr(self, name): 273 274 """ 275 Record class attribute 'name' in the current class using the current 276 expression. 277 """ 278 279 if self.in_method and self.namespaces[-2].has_key(name): 280 self.namespaces[-2].set(name, self.expr, 0) 281 return 1 282 283 return 0 284 285 def store_instance_attr(self, name): 286 287 "Record instance attribute 'name' in the current class." 288 289 if self.in_method: 290 291 # Current namespace is the function. 292 # Previous namespace is the class. 293 294 self.namespaces[-2].add_instance_attribute(name) 295 296 def get_parent(self): 297 298 "Return the parent (or most recent) namespace currently exposed." 299 300 return (self.namespaces[-1:] or [self])[0] 301 302 def use_name(self, name): 303 304 "Use the given 'name' within the current namespace/unit." 305 306 unit = self.get_parent() 307 self.importer.use_name(name, unit.name) 308 309 # Visitor methods. 310 311 def default(self, node, *args): 312 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 313 314 def dispatch(self, node, *args): 315 return ASTVisitor.dispatch(self, node, *args) 316 317 def NOP(self, node): 318 for n in node.getChildNodes(): 319 self.dispatch(n) 320 return None 321 322 def OP(self, node): 323 for n in node.getChildNodes(): 324 self.dispatch(n) 325 return Instance() 326 327 def _visitUnary(self, node): 328 329 "Accounting method for the unary operator 'node'." 330 331 method = unary_methods[node.__class__.__name__] 332 self.use_name(method) 333 return self.OP(node) 334 335 def _visitBinary(self, node): 336 337 "Accounting method for the binary operator 'node'." 338 339 left_method, right_method = binary_methods[node.__class__.__name__] 340 self.use_name(left_method) 341 self.use_name(right_method) 342 return self.OP(node) 343 344 def _visitFunction(self, node, name): 345 346 """ 347 Return a function object for the function defined by 'node' with the 348 given 'name'. If a lambda expression is being visited, 'name' should be 349 None. 350 """ 351 352 # Define the function object. 353 354 function = Function( 355 name, 356 self.get_parent(), 357 node.argnames, 358 node.defaults, 359 (node.flags & 4 != 0), 360 (node.flags & 8 != 0), 361 self, 362 node 363 ) 364 365 self.add_object(function, any_scope=1) 366 367 # Make a back reference from the node for code generation. 368 369 node.unit = function 370 371 # Process the defaults. 372 373 for n in node.defaults: 374 self.expr = self.dispatch(n) 375 function.store_default(self.expr) 376 377 self.functions.append((node, self.namespaces + [function])) 378 379 if name is not None: 380 self.store(name, function) 381 else: 382 self.store_lambda(function) 383 384 # Where defaults exist, an instance needs creating. Thus, it makes 385 # no sense to return a reference to the function here, since the 386 # recipient will not be referencing the function itself. 387 388 if node.defaults: 389 return Instance() # indicates no known target 390 391 return function 392 393 def _visitFunctionBody(self, node, namespaces): 394 395 "Enter the function." 396 397 # Current namespace is the function. 398 # Previous namespace is the class. 399 400 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 401 if namespaces[-1].name == "__init__": 402 self.in_init = 1 403 self.in_method = 1 404 405 self.namespaces = namespaces 406 self.dispatch(node.code) 407 408 self.in_init = 0 409 self.in_method = 0 410 411 # Specific handler methods. 412 413 visitAdd = _visitBinary 414 415 visitAnd = OP 416 417 visitAssert = NOP 418 419 def visitAssign(self, node): 420 self.expr = self.dispatch(node.expr) 421 for n in node.nodes: 422 self.dispatch(n) 423 return None 424 425 def visitAssAttr(self, node): 426 expr = self.dispatch(node.expr) 427 428 # Record the attribute on the presumed target. 429 430 if isinstance(expr, Attr): 431 if expr.name == "self": 432 if not self.store_class_attr(node.attrname): 433 self.store_instance_attr(node.attrname) 434 elif isinstance(expr.get_value(), Module): 435 self.store_module_attr(node.attrname, expr.get_value()) 436 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 437 438 # Note usage of the attribute. 439 440 node._attrnames = expr.use_attribute(node.attrname) 441 442 return None 443 444 def visitAssList(self, node): 445 446 # Declare names which will be used by generated code. 447 448 self.use_name("__getitem__") 449 450 # Process the assignment. 451 452 for i, n in enumerate(node.nodes): 453 self.dispatch(n) 454 self.importer.make_constant(i) # for __getitem__(i) at run-time 455 return None 456 457 def visitAssName(self, node): 458 self.store(node.name, self.expr) 459 self.use_name(node.name) 460 return None 461 462 visitAssTuple = visitAssList 463 464 def visitAugAssign(self, node): 465 466 # Accounting. 467 468 aug_method, (left_method, right_method) = augassign_methods[node.op] 469 self.use_name(aug_method) 470 self.use_name(left_method) 471 self.use_name(right_method) 472 473 # Process the assignment. 474 475 self.expr = self.dispatch(node.expr) 476 477 # NOTE: Similar to micropython.ast handler code. 478 # NOTE: Slices and subscripts not supported. 479 480 if isinstance(node.node, compiler.ast.Name): 481 self.visitAssName(node.node) 482 elif isinstance(node.node, compiler.ast.Getattr): 483 self.visitAssAttr(node.node) 484 else: 485 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 486 487 return None 488 489 visitBackquote = OP 490 491 visitBitand = _visitBinary 492 493 visitBitor = _visitBinary 494 495 visitBitxor = _visitBinary 496 497 visitBreak = NOP 498 499 visitCallFunc = OP 500 501 def visitClass(self, node): 502 503 """ 504 Register the class at the given 'node' subject to the restrictions 505 mentioned in the module docstring. 506 """ 507 508 if self.namespaces: 509 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 510 return None 511 else: 512 cls = Class(node.name, self.get_parent(), self, node) 513 514 # Visit the base class expressions, attempting to find concrete 515 # definitions of classes. 516 517 for base in node.bases: 518 expr = self.dispatch(base) 519 if isinstance(expr, Attr): 520 if expr.assignments != 1: 521 raise InspectError(self.full_name(), node, 522 "Base class %r for %r is not constant." % (base, cls.full_name())) 523 else: 524 cls.add_base(expr.get_value()) 525 else: # if expr is None: 526 raise InspectError(self.full_name(), node, 527 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 528 529 # NOTE: Potentially dubious measure to permit __init__ availability. 530 # If no bases exist, adopt the 'object' class. 531 532 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 533 expr = self.dispatch(compiler.ast.Name("object")) 534 cls.add_base(expr.get_value()) 535 536 # Make a back reference from the node for code generation. 537 538 node.unit = cls 539 540 # Make an entry for the class. 541 542 self.store(node.name, cls) 543 self.add_object(cls) 544 545 # Process the class body. 546 547 self.namespaces.append(cls) 548 self.dispatch(node.code) 549 self.namespaces.pop() 550 551 return cls 552 553 def visitCompare(self, node): 554 555 # Accounting. 556 # NOTE: Replicates some code in micropython.ast.visitCompare. 557 558 for op in node.ops: 559 op_name, next_node = op 560 methods = comparison_methods[op_name] 561 if methods is not None: 562 self.use_name(methods[0]) 563 self.use_name(methods[1]) 564 elif op_name.endswith("in"): 565 self.use_name("__contains__") 566 567 return self.OP(node) 568 569 def visitConst(self, node): 570 571 # Register the constant, if necessary, returning the resulting object. 572 573 self.use_name(self.importer.get_constant_type_name(node.value)) 574 return self.importer.make_constant(node.value) 575 576 visitContinue = NOP 577 578 visitDecorators = NOP 579 580 visitDict = OP 581 582 visitDiscard = NOP 583 584 visitDiv = _visitBinary 585 586 visitEllipsis = NOP 587 588 visitExec = NOP 589 590 visitExpression = OP 591 592 visitFloorDiv = _visitBinary 593 594 def visitFor(self, node): 595 596 # Declare names which will be used by generated code. 597 598 self.use_name("__iter__") 599 self.use_name("next") 600 601 # Enter the loop. 602 603 self.in_loop = 1 604 self.NOP(node) 605 self.in_loop = 0 606 return None 607 608 def visitFrom(self, node): 609 module = self.importer.load(node.modname, 1) 610 611 #if module is None: 612 # print "Warning:", node.modname, "not imported." 613 614 for name, alias in node.names: 615 if name != "*": 616 if module is not None and module.namespace.has_key(name): 617 attr = module[name] 618 self.store(alias or name, attr) 619 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 620 self.importer.load(attr.get_value().name) 621 622 # Support the import of names from missing modules. 623 624 else: 625 self.store(alias or name, UnresolvedName(name, node.modname, self)) 626 else: 627 if module is not None: 628 for n in module.namespace.keys(): 629 attr = module[n] 630 self.store(n, attr) 631 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 632 self.importer.load(attr.get_value().name) 633 634 return None 635 636 def visitFunction(self, node): 637 return self._visitFunction(node, node.name) 638 639 visitGenExpr = OP 640 641 visitGenExprFor = NOP 642 643 visitGenExprIf = NOP 644 645 visitGenExprInner = NOP 646 647 def visitGetattr(self, node): 648 expr = self.dispatch(node.expr) 649 attrname = node.attrname 650 651 # Attempt to identify the nature of the attribute. 652 653 if isinstance(expr, Attr): 654 value = expr.get_value() 655 if isinstance(value, (Class, Module)): 656 attr = value.namespace.get(attrname) 657 elif isinstance(value, UnresolvedName): 658 attr = UnresolvedName(attrname, value.full_name(), self) 659 else: 660 attr = None 661 662 # Note usage of the attribute. 663 664 node._attrnames = expr.use_attribute(node.attrname) 665 666 elif self.builtins is not None: 667 attr = self.builtins.get(attrname) 668 else: 669 attr = UnresolvedName(attrname, value.full_name(), self) 670 671 # Accounting. 672 673 self.use_name(attrname) 674 675 return attr 676 677 def visitGlobal(self, node): 678 if self.namespaces: 679 for name in node.names: 680 ns = self.namespaces[-1] 681 if not ns.make_global(name): 682 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 683 684 # The name is recorded in an earlier process. 685 686 def visitIf(self, node): 687 for test, body in node.tests: 688 self.dispatch(test) 689 self.dispatch(body) 690 if node.else_ is not None: 691 self.dispatch(node.else_) 692 return None 693 694 visitIfExp = NOP 695 696 def visitImport(self, node): 697 for name, alias in node.names: 698 if alias is not None: 699 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 700 self.store(alias, module) 701 else: 702 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 703 self.store(name.split(".")[0], module) 704 705 return None 706 707 visitInvert = _visitUnary 708 709 def visitKeyword(self, node): 710 self.dispatch(node.expr) 711 self.importer.make_constant(node.name) 712 self.keyword_names.add(node.name) 713 return None 714 715 def visitLambda(self, node): 716 self.use_name(None) # lambda functions have no names but are assumed to be invoked 717 return self._visitFunction(node, None) 718 719 visitLeftShift = _visitBinary 720 721 visitList = OP 722 723 visitListComp = OP 724 725 visitListCompFor = NOP 726 727 visitListCompIf = NOP 728 729 visitMod = _visitBinary 730 731 def visitModule(self, node): 732 733 # Make a back reference from the node for code generation. 734 735 node.unit = self 736 return self.dispatch(node.node) 737 738 visitMul = _visitBinary 739 740 def visitName(self, node): 741 name = node.name 742 743 if self.importer.predefined_constants.has_key(name): 744 attr = self.importer.get_predefined_constant(name) 745 elif self.namespaces and self.namespaces[-1].has_key(name): 746 attr = self.namespaces[-1][name] 747 elif self.has_key(name): 748 attr = self[name] 749 elif self.builtins is not None and self.builtins.has_key(name): 750 attr = self.builtins[name] 751 else: 752 attr = None 753 754 self.use_name(name) 755 return attr 756 757 visitNot = OP 758 759 visitOr = OP 760 761 visitPass = NOP 762 763 visitPower = _visitBinary 764 765 visitPrint = NOP 766 767 visitPrintnl = NOP 768 769 visitRaise = NOP 770 771 visitReturn = NOP 772 773 visitRightShift = _visitBinary 774 775 visitSlice = OP 776 777 visitSliceobj = OP 778 779 def visitStmt(self, node): 780 for n in node.nodes: 781 self.dispatch(n) 782 return None 783 784 visitSub = _visitBinary 785 786 def visitSubscript(self, node): 787 self.use_name("__getitem__") 788 self.OP(node) 789 790 def visitTryExcept(self, node): 791 self.dispatch(node.body) 792 for name, var, n in node.handlers: 793 794 # Establish the local for the handler. 795 796 if var is not None: 797 self.dispatch(var) 798 if n is not None: 799 self.dispatch(n) 800 if node.else_ is not None: 801 self.dispatch(node.else_) 802 return None 803 804 visitTryFinally = NOP 805 806 visitTuple = OP 807 808 visitUnaryAdd = _visitUnary 809 810 visitUnarySub = _visitUnary 811 812 def visitWhile(self, node): 813 self.in_loop = 1 814 self.NOP(node) 815 self.in_loop = 0 816 return None 817 818 visitWith = NOP 819 820 visitYield = NOP 821 822 # vim: tabstop=4 expandtab shiftwidth=4