1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # First, visit module-level code, recording global names. 133 134 processed = self.dispatch(module) 135 136 # Then, for each function, detect and record globals declared in those 137 # functions. 138 139 for node, namespaces in self.functions: 140 self.process_globals(node) 141 142 # Then, visit each function, recording other names. 143 144 for node, namespaces in self.functions: 145 self._visitFunctionBody(node, namespaces) 146 147 # Add references to other modules declared using the __all__ global. 148 149 if self.has_key("__all__"): 150 all = self["__all__"] 151 if isinstance(all, compiler.ast.List): 152 for n in all.nodes: 153 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 154 155 return processed 156 157 def process_globals(self, node): 158 159 """ 160 Within the given 'node', process global declarations, adjusting the 161 module namespace. 162 """ 163 164 for n in node.getChildNodes(): 165 if isinstance(n, compiler.ast.Global): 166 for name in n.names: 167 if not self.has_key(name): 168 self[name] = None 169 else: 170 self.process_globals(n) 171 172 def vacuum(self): 173 174 """ 175 Vacuum the module namespace, removing unreferenced objects and unused 176 names. 177 """ 178 179 # Remove unreferenced objects. 180 181 if self.should_optimise_unused_objects(): 182 183 for name, value in self.items(): 184 185 # Remove entries for unreferenced objects. 186 # This, due to the nature of the referenced attribute, assumes 187 # that only explicitly mentioned classes and functions are 188 # employed in the final program. 189 190 if isinstance(value, Attr): 191 192 # Only remove entries for classes and functions, not methods. 193 194 for attr_value in value.get_values(): 195 if (isinstance(attr_value, Function) and not attr_value.is_method() or 196 isinstance(attr_value, Class)) and not attr_value.referenced: 197 pass 198 else: 199 break 200 else: 201 del self[name] 202 203 all_objects = list(self.all_objects) 204 205 for obj in all_objects: 206 207 # Only remove entries for classes and functions, not methods. 208 209 if (isinstance(obj, Function) and not obj.is_method() or 210 isinstance(obj, Class)) and not obj.referenced: 211 212 self.all_objects.remove(obj) 213 obj_was_removed = 1 214 215 else: 216 obj_was_removed = 0 217 218 # Remove unused entries from classes plus associated methods. 219 220 if isinstance(obj, Class): 221 for name, attr in obj.class_attributes().items(): 222 223 # In removed classes, all explicitly defined methods can 224 # be removed. 225 226 # In other classes, methods can only be removed if they 227 # are the only assigned object to the class for a 228 # particular attribute and are unreferenced. 229 230 if obj_was_removed and isinstance(attr.get_value(), Function) and attr.get_value().is_method() or \ 231 name not in self.importer.names_used and \ 232 attr.assignments == 1 and isinstance(attr.get_value(), Function) and \ 233 attr.get_value().is_method() and not attr.get_value().referenced: 234 235 method = attr.get_value() 236 237 if method in self.all_objects: 238 self.all_objects.remove(method) 239 del obj[name] 240 241 def finalise(self): 242 243 "Finalise the module." 244 245 for obj in self.all_objects: 246 if isinstance(obj, (Class, Function)): 247 obj.finalise_attributes() 248 249 def add_object(self, obj, any_scope=0): 250 251 """ 252 Record 'obj' if non-local or if the optional 'any_scope' is set to a 253 true value. 254 """ 255 256 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 257 self.all_objects.add(obj) 258 259 # Optimisation tests. 260 261 def should_optimise_unused_objects(self): 262 return "unused_objects" in self.optimisations 263 264 # Namespace methods. 265 266 def store(self, name, obj): 267 268 "Record attribute or local 'name', storing 'obj'." 269 270 if not self.namespaces: 271 self.set(name, obj, not self.in_loop) 272 else: 273 self.namespaces[-1].set(name, obj, not self.in_loop) 274 275 def store_lambda(self, obj): 276 277 "Store a lambda function 'obj'." 278 279 self.add_object(obj) 280 281 def store_module_attr(self, name, module): 282 283 """ 284 Record module attribute 'name' in the given 'module' using the current 285 expression. 286 """ 287 288 module.set(name, self.expr, 0) 289 290 def store_class_attr(self, name): 291 292 """ 293 Record class attribute 'name' in the current class using the current 294 expression. 295 """ 296 297 if self.in_method and self.namespaces[-2].has_key(name): 298 self.namespaces[-2].set(name, self.expr, 0) 299 return 1 300 301 return 0 302 303 def store_instance_attr(self, name): 304 305 "Record instance attribute 'name' in the current class." 306 307 if self.in_method: 308 309 # Current namespace is the function. 310 # Previous namespace is the class. 311 312 self.namespaces[-2].add_instance_attribute(name) 313 314 def get_parent(self): 315 316 "Return the parent (or most recent) namespace currently exposed." 317 318 return (self.namespaces[-1:] or [self])[0] 319 320 # Visitor methods. 321 322 def default(self, node, *args): 323 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 324 325 def dispatch(self, node, *args): 326 return ASTVisitor.dispatch(self, node, *args) 327 328 def NOP(self, node): 329 for n in node.getChildNodes(): 330 self.dispatch(n) 331 return None 332 333 def OP(self, node): 334 for n in node.getChildNodes(): 335 self.dispatch(n) 336 return Instance() 337 338 def _visitUnary(self, node): 339 340 "Accounting method for the unary operator 'node'." 341 342 method = unary_methods[node.__class__.__name__] 343 self.importer.use_name(method) 344 return self.OP(node) 345 346 def _visitBinary(self, node): 347 348 "Accounting method for the binary operator 'node'." 349 350 left_method, right_method = binary_methods[node.__class__.__name__] 351 self.importer.use_name(left_method) 352 self.importer.use_name(right_method) 353 return self.OP(node) 354 355 def _visitFunction(self, node, name): 356 357 """ 358 Return a function object for the function defined by 'node' with the 359 given 'name'. If a lambda expression is being visited, 'name' should be 360 None. 361 """ 362 363 # Define the function object. 364 365 function = Function( 366 name, 367 self.get_parent(), 368 node.argnames, 369 node.defaults, 370 (node.flags & 4 != 0), 371 (node.flags & 8 != 0), 372 self, 373 node 374 ) 375 376 self.add_object(function, any_scope=1) 377 378 # Make a back reference from the node for code generation. 379 380 node.unit = function 381 382 # Process the defaults. 383 384 for n in node.defaults: 385 self.expr = self.dispatch(n) 386 function.store_default(self.expr) 387 388 self.functions.append((node, self.namespaces + [function])) 389 390 if name is not None: 391 self.store(name, function) 392 else: 393 self.store_lambda(function) 394 395 # Lambda functions are always assumed to be referenced. This is 396 # because other means of discovering the referencing of objects rely 397 # on the original names inherent in the definition of those objects. 398 399 function.set_referenced() 400 401 # Where defaults exist, an instance needs creating. Thus, it makes 402 # no sense to return a reference to the function here, since the 403 # recipient will not be referencing the function itself. 404 405 if node.defaults: 406 return Instance() # indicates no known target 407 408 return function 409 410 def _visitFunctionBody(self, node, namespaces): 411 412 "Enter the function." 413 414 # Current namespace is the function. 415 # Previous namespace is the class. 416 417 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 418 if namespaces[-1].name == "__init__": 419 self.in_init = 1 420 self.in_method = 1 421 422 self.namespaces = namespaces 423 self.dispatch(node.code) 424 425 self.in_init = 0 426 self.in_method = 0 427 428 # Specific handler methods. 429 430 visitAdd = _visitBinary 431 432 visitAnd = OP 433 434 visitAssert = NOP 435 436 def visitAssign(self, node): 437 self.expr = self.dispatch(node.expr) 438 for n in node.nodes: 439 self.dispatch(n) 440 return None 441 442 def visitAssAttr(self, node): 443 expr = self.dispatch(node.expr) 444 if isinstance(expr, Attr): 445 if expr.name == "self": 446 if not self.store_class_attr(node.attrname): 447 self.store_instance_attr(node.attrname) 448 elif isinstance(expr.get_value(), Module): 449 self.store_module_attr(node.attrname, expr.get_value()) 450 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 451 return None 452 453 def visitAssList(self, node): 454 455 # Declare names which will be used by generated code. 456 457 self.importer.use_name("__getitem__") 458 459 # Process the assignment. 460 461 for i, n in enumerate(node.nodes): 462 self.dispatch(n) 463 self.importer.make_constant(i) # for __getitem__(i) at run-time 464 return None 465 466 def visitAssName(self, node): 467 self.store(node.name, self.expr) 468 return None 469 470 visitAssTuple = visitAssList 471 472 def visitAugAssign(self, node): 473 474 # Accounting. 475 476 aug_method, (left_method, right_method) = augassign_methods[node.op] 477 self.importer.use_name(aug_method) 478 self.importer.use_name(left_method) 479 self.importer.use_name(right_method) 480 481 # Process the assignment. 482 483 self.expr = self.dispatch(node.expr) 484 485 # NOTE: Similar to micropython.ast handler code. 486 # NOTE: Slices and subscripts not supported. 487 488 if isinstance(node.node, compiler.ast.Name): 489 self.visitAssName(node.node) 490 elif isinstance(node.node, compiler.ast.Getattr): 491 self.visitAssAttr(node.node) 492 else: 493 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 494 495 return None 496 497 visitBackquote = OP 498 499 visitBitand = _visitBinary 500 501 visitBitor = _visitBinary 502 503 visitBitxor = _visitBinary 504 505 visitBreak = NOP 506 507 visitCallFunc = OP 508 509 def visitClass(self, node): 510 511 """ 512 Register the class at the given 'node' subject to the restrictions 513 mentioned in the module docstring. 514 """ 515 516 if self.namespaces: 517 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 518 return None 519 else: 520 cls = Class(node.name, self.get_parent(), self, node) 521 522 # Visit the base class expressions, attempting to find concrete 523 # definitions of classes. 524 525 for base in node.bases: 526 expr = self.dispatch(base) 527 if isinstance(expr, Attr): 528 if expr.assignments != 1: 529 raise InspectError(self.full_name(), node, 530 "Base class %r for %r is not constant." % (base, cls.full_name())) 531 else: 532 cls.add_base(expr.get_value()) 533 else: # if expr is None: 534 raise InspectError(self.full_name(), node, 535 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 536 537 # NOTE: Potentially dubious measure to permit __init__ availability. 538 # If no bases exist, adopt the 'object' class. 539 540 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 541 expr = self.dispatch(compiler.ast.Name("object")) 542 cls.add_base(expr.get_value()) 543 544 # Make a back reference from the node for code generation. 545 546 node.unit = cls 547 548 # Make an entry for the class. 549 550 self.store(node.name, cls) 551 self.add_object(cls) 552 553 # Process the class body. 554 555 self.namespaces.append(cls) 556 self.dispatch(node.code) 557 self.namespaces.pop() 558 559 return cls 560 561 def visitCompare(self, node): 562 563 # Accounting. 564 # NOTE: Replicates some code in micropython.ast.visitCompare. 565 566 for op in node.ops: 567 op_name, next_node = op 568 methods = comparison_methods[op_name] 569 if methods is not None: 570 self.importer.use_name(methods[0]) 571 self.importer.use_name(methods[1]) 572 elif op_name.endswith("in"): 573 self.importer.use_name("__contains__") 574 575 return self.OP(node) 576 577 def visitConst(self, node): 578 579 # Register the constant, if necessary, returning the resulting object. 580 581 return self.importer.make_constant(node.value) 582 583 visitContinue = NOP 584 585 visitDecorators = NOP 586 587 visitDict = OP 588 589 visitDiscard = NOP 590 591 visitDiv = _visitBinary 592 593 visitEllipsis = NOP 594 595 visitExec = NOP 596 597 visitExpression = OP 598 599 visitFloorDiv = _visitBinary 600 601 def visitFor(self, node): 602 603 # Declare names which will be used by generated code. 604 605 self.importer.use_name("__iter__") 606 self.importer.use_name("next") 607 608 # Enter the loop. 609 610 self.in_loop = 1 611 self.NOP(node) 612 self.in_loop = 0 613 return None 614 615 def visitFrom(self, node): 616 module = self.importer.load(node.modname, 1) 617 618 if module is not None: 619 module.set_referenced() 620 621 #if module is None: 622 # print "Warning:", node.modname, "not imported." 623 624 for name, alias in node.names: 625 if name != "*": 626 if module is not None and module.namespace.has_key(name): 627 attr = module[name] 628 self.store(alias or name, attr) 629 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 630 self.importer.load(attr.get_value().name) 631 632 # Support the import of names from missing modules. 633 634 else: 635 self.store(alias or name, UnresolvedName(name, node.modname, self)) 636 else: 637 if module is not None: 638 for n in module.namespace.keys(): 639 attr = module[n] 640 self.store(n, attr) 641 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 642 self.importer.load(attr.get_value().name) 643 644 return None 645 646 def visitFunction(self, node): 647 return self._visitFunction(node, node.name) 648 649 visitGenExpr = OP 650 651 visitGenExprFor = NOP 652 653 visitGenExprIf = NOP 654 655 visitGenExprInner = NOP 656 657 def visitGetattr(self, node): 658 expr = self.dispatch(node.expr) 659 attrname = node.attrname 660 661 if isinstance(expr, Attr): 662 value = expr.get_value() 663 if isinstance(value, (Class, Module)): 664 attr = value.namespace.get(attrname) 665 elif isinstance(value, UnresolvedName): 666 attr = UnresolvedName(attrname, value.full_name(), self) 667 else: 668 attr = None 669 elif self.builtins is not None: 670 attr = self.builtins.get(attrname) 671 else: 672 attr = UnresolvedName(attrname, value.full_name(), self) 673 674 # Accounting. 675 676 if attr is not None: 677 attr.set_referenced() 678 679 self.importer.use_name(attrname) 680 681 return attr 682 683 def visitGlobal(self, node): 684 if self.namespaces: 685 for name in node.names: 686 ns = self.namespaces[-1] 687 if not ns.make_global(name): 688 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 689 690 # The name is recorded in an earlier process. 691 692 def visitIf(self, node): 693 for test, body in node.tests: 694 self.dispatch(test) 695 self.dispatch(body) 696 if node.else_ is not None: 697 self.dispatch(node.else_) 698 return None 699 700 visitIfExp = NOP 701 702 def visitImport(self, node): 703 for name, alias in node.names: 704 if alias is not None: 705 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 706 self.store(alias, module) 707 else: 708 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 709 self.store(name.split(".")[0], module) 710 module.set_referenced() 711 712 return None 713 714 visitInvert = _visitUnary 715 716 def visitKeyword(self, node): 717 self.dispatch(node.expr) 718 self.importer.make_constant(node.name) 719 self.keyword_names.add(node.name) 720 return None 721 722 def visitLambda(self, node): 723 return self._visitFunction(node, None) 724 725 visitLeftShift = _visitBinary 726 727 visitList = OP 728 729 visitListComp = OP 730 731 visitListCompFor = NOP 732 733 visitListCompIf = NOP 734 735 visitMod = _visitBinary 736 737 def visitModule(self, node): 738 739 # Make a back reference from the node for code generation. 740 741 node.unit = self 742 return self.dispatch(node.node) 743 744 visitMul = _visitBinary 745 746 def visitName(self, node): 747 name = node.name 748 749 if self.importer.predefined_constants.has_key(name): 750 attr = self.importer.get_predefined_constant(name) 751 elif self.namespaces and self.namespaces[-1].has_key(name): 752 attr = self.namespaces[-1][name] 753 elif self.has_key(name): 754 attr = self[name] 755 elif self.builtins is not None and self.builtins.has_key(name): 756 attr = self.builtins[name] 757 else: 758 attr = None 759 760 # Accounting. 761 762 if attr is not None: 763 attr.set_referenced() 764 765 self.importer.use_name(name) 766 767 return attr 768 769 visitNot = OP 770 771 visitOr = OP 772 773 visitPass = NOP 774 775 visitPower = _visitBinary 776 777 visitPrint = NOP 778 779 visitPrintnl = NOP 780 781 visitRaise = NOP 782 783 visitReturn = NOP 784 785 visitRightShift = _visitBinary 786 787 visitSlice = OP 788 789 visitSliceobj = OP 790 791 def visitStmt(self, node): 792 for n in node.nodes: 793 self.dispatch(n) 794 return None 795 796 visitSub = _visitBinary 797 798 def visitSubscript(self, node): 799 self.importer.use_name("__getitem__") 800 self.OP(node) 801 802 def visitTryExcept(self, node): 803 self.dispatch(node.body) 804 for name, var, n in node.handlers: 805 806 # Establish the local for the handler. 807 808 if var is not None: 809 self.dispatch(var) 810 if n is not None: 811 self.dispatch(n) 812 if node.else_ is not None: 813 self.dispatch(node.else_) 814 return None 815 816 visitTryFinally = NOP 817 818 visitTuple = OP 819 820 visitUnaryAdd = _visitUnary 821 822 visitUnarySub = _visitUnary 823 824 def visitWhile(self, node): 825 self.in_loop = 1 826 self.NOP(node) 827 self.in_loop = 0 828 return None 829 830 visitWith = NOP 831 832 visitYield = NOP 833 834 # vim: tabstop=4 expandtab shiftwidth=4