1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 if delete_all or not self.importer.uses_name(name): 202 del obj[name] 203 if attr.assignments == 1: 204 value = attr.get_value() 205 206 # Delete any unambiguous attribute value. 207 208 if value is not obj and value in self.all_objects: 209 self.all_objects.remove(value) 210 211 # Delete class contents. 212 213 if isinstance(value, Class): 214 self.vacuum_object(value, 1) 215 216 def finalise(self): 217 218 "Finalise the module." 219 220 for obj in self.all_objects: 221 if isinstance(obj, (Class, Function)): 222 obj.finalise_attributes() 223 224 def add_object(self, obj, any_scope=0): 225 226 """ 227 Record 'obj' if non-local or if the optional 'any_scope' is set to a 228 true value. 229 """ 230 231 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 232 self.all_objects.add(obj) 233 234 # Optimisation tests. 235 236 def should_optimise_unused_objects(self): 237 return "unused_objects" in self.optimisations 238 239 # Namespace methods. 240 241 def store(self, name, obj): 242 243 "Record attribute or local 'name', storing 'obj'." 244 245 if not self.namespaces: 246 self.set(name, obj, not self.in_loop) 247 else: 248 self.namespaces[-1].set(name, obj, not self.in_loop) 249 250 def store_lambda(self, obj): 251 252 "Store a lambda function 'obj'." 253 254 self.add_object(obj) 255 256 def store_module_attr(self, name, module): 257 258 """ 259 Record module attribute 'name' in the given 'module' using the current 260 expression. 261 """ 262 263 module.set(name, self.expr, 0) 264 265 def store_class_attr(self, name): 266 267 """ 268 Record class attribute 'name' in the current class using the current 269 expression. 270 """ 271 272 if self.in_method and self.namespaces[-2].has_key(name): 273 self.namespaces[-2].set(name, self.expr, 0) 274 return 1 275 276 return 0 277 278 def store_instance_attr(self, name): 279 280 "Record instance attribute 'name' in the current class." 281 282 if self.in_method: 283 284 # Current namespace is the function. 285 # Previous namespace is the class. 286 287 self.namespaces[-2].add_instance_attribute(name) 288 289 def get_parent(self): 290 291 "Return the parent (or most recent) namespace currently exposed." 292 293 return (self.namespaces[-1:] or [self])[0] 294 295 def use_name(self, name): 296 297 "Use the given 'name' within the current namespace/unit." 298 299 unit = self.get_parent() 300 self.importer.use_name(name, unit.name) 301 302 # Visitor methods. 303 304 def default(self, node, *args): 305 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 306 307 def dispatch(self, node, *args): 308 return ASTVisitor.dispatch(self, node, *args) 309 310 def NOP(self, node): 311 for n in node.getChildNodes(): 312 self.dispatch(n) 313 return None 314 315 def OP(self, node): 316 for n in node.getChildNodes(): 317 self.dispatch(n) 318 return Instance() 319 320 def _visitUnary(self, node): 321 322 "Accounting method for the unary operator 'node'." 323 324 method = unary_methods[node.__class__.__name__] 325 self.use_name(method) 326 return self.OP(node) 327 328 def _visitBinary(self, node): 329 330 "Accounting method for the binary operator 'node'." 331 332 left_method, right_method = binary_methods[node.__class__.__name__] 333 self.use_name(left_method) 334 self.use_name(right_method) 335 return self.OP(node) 336 337 def _visitFunction(self, node, name): 338 339 """ 340 Return a function object for the function defined by 'node' with the 341 given 'name'. If a lambda expression is being visited, 'name' should be 342 None. 343 """ 344 345 # Define the function object. 346 347 function = Function( 348 name, 349 self.get_parent(), 350 node.argnames, 351 node.defaults, 352 (node.flags & 4 != 0), 353 (node.flags & 8 != 0), 354 self, 355 node 356 ) 357 358 self.add_object(function, any_scope=1) 359 360 # Make a back reference from the node for code generation. 361 362 node.unit = function 363 364 # Process the defaults. 365 366 for n in node.defaults: 367 self.expr = self.dispatch(n) 368 function.store_default(self.expr) 369 370 self.functions.append((node, self.namespaces + [function])) 371 372 if name is not None: 373 self.store(name, function) 374 else: 375 self.store_lambda(function) 376 377 # Where defaults exist, an instance needs creating. Thus, it makes 378 # no sense to return a reference to the function here, since the 379 # recipient will not be referencing the function itself. 380 381 if node.defaults: 382 return Instance() # indicates no known target 383 384 return function 385 386 def _visitFunctionBody(self, node, namespaces): 387 388 "Enter the function." 389 390 # Current namespace is the function. 391 # Previous namespace is the class. 392 393 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 394 if namespaces[-1].name == "__init__": 395 self.in_init = 1 396 self.in_method = 1 397 398 self.namespaces = namespaces 399 self.dispatch(node.code) 400 401 self.in_init = 0 402 self.in_method = 0 403 404 # Specific handler methods. 405 406 visitAdd = _visitBinary 407 408 visitAnd = OP 409 410 visitAssert = NOP 411 412 def visitAssign(self, node): 413 self.expr = self.dispatch(node.expr) 414 for n in node.nodes: 415 self.dispatch(n) 416 return None 417 418 def visitAssAttr(self, node): 419 expr = self.dispatch(node.expr) 420 421 # Record the attribute on the presumed target. 422 423 if isinstance(expr, Attr): 424 if expr.name == "self": 425 if not self.store_class_attr(node.attrname): 426 self.store_instance_attr(node.attrname) 427 elif isinstance(expr.get_value(), Module): 428 self.store_module_attr(node.attrname, expr.get_value()) 429 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 430 431 # Note usage of the attribute. 432 433 node._attrnames = expr.use_attribute(node.attrname) 434 435 return None 436 437 def visitAssList(self, node): 438 439 # Declare names which will be used by generated code. 440 441 self.use_name("__getitem__") 442 443 # Process the assignment. 444 445 for i, n in enumerate(node.nodes): 446 self.dispatch(n) 447 self.importer.make_constant(i) # for __getitem__(i) at run-time 448 return None 449 450 def visitAssName(self, node): 451 self.store(node.name, self.expr) 452 self.use_name(node.name) 453 return None 454 455 visitAssTuple = visitAssList 456 457 def visitAugAssign(self, node): 458 459 # Accounting. 460 461 aug_method, (left_method, right_method) = augassign_methods[node.op] 462 self.use_name(aug_method) 463 self.use_name(left_method) 464 self.use_name(right_method) 465 466 # Process the assignment. 467 468 self.expr = self.dispatch(node.expr) 469 470 # NOTE: Similar to micropython.ast handler code. 471 # NOTE: Slices and subscripts not supported. 472 473 if isinstance(node.node, compiler.ast.Name): 474 self.visitAssName(node.node) 475 elif isinstance(node.node, compiler.ast.Getattr): 476 self.visitAssAttr(node.node) 477 else: 478 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 479 480 return None 481 482 visitBackquote = OP 483 484 visitBitand = _visitBinary 485 486 visitBitor = _visitBinary 487 488 visitBitxor = _visitBinary 489 490 visitBreak = NOP 491 492 visitCallFunc = OP 493 494 def visitClass(self, node): 495 496 """ 497 Register the class at the given 'node' subject to the restrictions 498 mentioned in the module docstring. 499 """ 500 501 if self.namespaces: 502 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 503 return None 504 else: 505 cls = Class(node.name, self.get_parent(), self, node) 506 507 # Visit the base class expressions, attempting to find concrete 508 # definitions of classes. 509 510 for base in node.bases: 511 expr = self.dispatch(base) 512 if isinstance(expr, Attr): 513 if expr.assignments != 1: 514 raise InspectError(self.full_name(), node, 515 "Base class %r for %r is not constant." % (base, cls.full_name())) 516 else: 517 cls.add_base(expr.get_value()) 518 else: # if expr is None: 519 raise InspectError(self.full_name(), node, 520 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 521 522 # NOTE: Potentially dubious measure to permit __init__ availability. 523 # If no bases exist, adopt the 'object' class. 524 525 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 526 expr = self.dispatch(compiler.ast.Name("object")) 527 cls.add_base(expr.get_value()) 528 529 # Make a back reference from the node for code generation. 530 531 node.unit = cls 532 533 # Make an entry for the class. 534 535 self.store(node.name, cls) 536 self.add_object(cls) 537 538 # Process the class body. 539 540 self.namespaces.append(cls) 541 self.dispatch(node.code) 542 self.namespaces.pop() 543 544 return cls 545 546 def visitCompare(self, node): 547 548 # Accounting. 549 # NOTE: Replicates some code in micropython.ast.visitCompare. 550 551 for op in node.ops: 552 op_name, next_node = op 553 methods = comparison_methods[op_name] 554 if methods is not None: 555 self.use_name(methods[0]) 556 self.use_name(methods[1]) 557 elif op_name.endswith("in"): 558 self.use_name("__contains__") 559 560 return self.OP(node) 561 562 def visitConst(self, node): 563 564 # Register the constant, if necessary, returning the resulting object. 565 566 self.use_name(self.importer.get_constant_type_name(node.value)) 567 return self.importer.make_constant(node.value) 568 569 visitContinue = NOP 570 571 visitDecorators = NOP 572 573 visitDict = OP 574 575 visitDiscard = NOP 576 577 visitDiv = _visitBinary 578 579 visitEllipsis = NOP 580 581 visitExec = NOP 582 583 visitExpression = OP 584 585 visitFloorDiv = _visitBinary 586 587 def visitFor(self, node): 588 589 # Declare names which will be used by generated code. 590 591 self.use_name("__iter__") 592 self.use_name("next") 593 594 # Enter the loop. 595 596 self.in_loop = 1 597 self.NOP(node) 598 self.in_loop = 0 599 return None 600 601 def visitFrom(self, node): 602 module = self.importer.load(node.modname, 1) 603 604 #if module is None: 605 # print "Warning:", node.modname, "not imported." 606 607 for name, alias in node.names: 608 if name != "*": 609 if module is not None and module.namespace.has_key(name): 610 attr = module[name] 611 self.store(alias or name, attr) 612 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 613 self.importer.load(attr.get_value().name) 614 615 # Support the import of names from missing modules. 616 617 else: 618 self.store(alias or name, UnresolvedName(name, node.modname, self)) 619 else: 620 if module is not None: 621 for n in module.namespace.keys(): 622 attr = module[n] 623 self.store(n, attr) 624 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 625 self.importer.load(attr.get_value().name) 626 627 return None 628 629 def visitFunction(self, node): 630 return self._visitFunction(node, node.name) 631 632 visitGenExpr = OP 633 634 visitGenExprFor = NOP 635 636 visitGenExprIf = NOP 637 638 visitGenExprInner = NOP 639 640 def visitGetattr(self, node): 641 expr = self.dispatch(node.expr) 642 attrname = node.attrname 643 644 # Attempt to identify the nature of the attribute. 645 646 if isinstance(expr, Attr): 647 value = expr.get_value() 648 if isinstance(value, (Class, Module)): 649 attr = value.namespace.get(attrname) 650 elif isinstance(value, UnresolvedName): 651 attr = UnresolvedName(attrname, value.full_name(), self) 652 else: 653 attr = None 654 655 # Note usage of the attribute. 656 657 node._attrnames = expr.use_attribute(node.attrname) 658 659 elif self.builtins is not None: 660 attr = self.builtins.get(attrname) 661 else: 662 attr = UnresolvedName(attrname, value.full_name(), self) 663 664 # Accounting. 665 666 self.use_name(attrname) 667 668 return attr 669 670 def visitGlobal(self, node): 671 if self.namespaces: 672 for name in node.names: 673 ns = self.namespaces[-1] 674 if not ns.make_global(name): 675 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 676 677 # The name is recorded in an earlier process. 678 679 def visitIf(self, node): 680 for test, body in node.tests: 681 self.dispatch(test) 682 self.dispatch(body) 683 if node.else_ is not None: 684 self.dispatch(node.else_) 685 return None 686 687 visitIfExp = NOP 688 689 def visitImport(self, node): 690 for name, alias in node.names: 691 if alias is not None: 692 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 693 self.store(alias, module) 694 else: 695 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 696 self.store(name.split(".")[0], module) 697 698 return None 699 700 visitInvert = _visitUnary 701 702 def visitKeyword(self, node): 703 self.dispatch(node.expr) 704 self.importer.make_constant(node.name) 705 self.keyword_names.add(node.name) 706 return None 707 708 def visitLambda(self, node): 709 self.use_name(None) # lambda functions have no names but are assumed to be invoked 710 return self._visitFunction(node, None) 711 712 visitLeftShift = _visitBinary 713 714 visitList = OP 715 716 visitListComp = OP 717 718 visitListCompFor = NOP 719 720 visitListCompIf = NOP 721 722 visitMod = _visitBinary 723 724 def visitModule(self, node): 725 726 # Make a back reference from the node for code generation. 727 728 node.unit = self 729 return self.dispatch(node.node) 730 731 visitMul = _visitBinary 732 733 def visitName(self, node): 734 name = node.name 735 736 if self.importer.predefined_constants.has_key(name): 737 attr = self.importer.get_predefined_constant(name) 738 elif self.namespaces and self.namespaces[-1].has_key(name): 739 attr = self.namespaces[-1][name] 740 elif self.has_key(name): 741 attr = self[name] 742 elif self.builtins is not None and self.builtins.has_key(name): 743 attr = self.builtins[name] 744 else: 745 attr = None 746 747 self.use_name(name) 748 return attr 749 750 visitNot = OP 751 752 visitOr = OP 753 754 visitPass = NOP 755 756 visitPower = _visitBinary 757 758 visitPrint = NOP 759 760 visitPrintnl = NOP 761 762 visitRaise = NOP 763 764 visitReturn = NOP 765 766 visitRightShift = _visitBinary 767 768 visitSlice = OP 769 770 visitSliceobj = OP 771 772 def visitStmt(self, node): 773 for n in node.nodes: 774 self.dispatch(n) 775 return None 776 777 visitSub = _visitBinary 778 779 def visitSubscript(self, node): 780 self.use_name("__getitem__") 781 self.OP(node) 782 783 def visitTryExcept(self, node): 784 self.dispatch(node.body) 785 for name, var, n in node.handlers: 786 787 # Establish the local for the handler. 788 789 if var is not None: 790 self.dispatch(var) 791 if n is not None: 792 self.dispatch(n) 793 if node.else_ is not None: 794 self.dispatch(node.else_) 795 return None 796 797 visitTryFinally = NOP 798 799 visitTuple = OP 800 801 visitUnaryAdd = _visitUnary 802 803 visitUnarySub = _visitUnary 804 805 def visitWhile(self, node): 806 self.in_loop = 1 807 self.NOP(node) 808 self.in_loop = 0 809 return None 810 811 visitWith = NOP 812 813 visitYield = NOP 814 815 # vim: tabstop=4 expandtab shiftwidth=4