1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # First, visit module-level code, recording global names. 133 134 processed = self.dispatch(module) 135 136 # Then, for each function, detect and record globals declared in those 137 # functions. 138 139 for node, namespaces in self.functions: 140 self.process_globals(node) 141 142 # Then, visit each function, recording other names. 143 144 for node, namespaces in self.functions: 145 self._visitFunctionBody(node, namespaces) 146 147 # Add references to other modules declared using the __all__ global. 148 149 if self.has_key("__all__"): 150 all = self["__all__"] 151 if isinstance(all, compiler.ast.List): 152 for n in all.nodes: 153 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 154 155 return processed 156 157 def process_globals(self, node): 158 159 """ 160 Within the given 'node', process global declarations, adjusting the 161 module namespace. 162 """ 163 164 for n in node.getChildNodes(): 165 if isinstance(n, compiler.ast.Global): 166 for name in n.names: 167 if not self.has_key(name): 168 self[name] = None 169 else: 170 self.process_globals(n) 171 172 def vacuum(self): 173 174 """ 175 Vacuum the module namespace, removing unreferenced objects and unused 176 names. 177 """ 178 179 # Remove unreferenced objects. 180 181 if self.should_optimise_unused_objects(): 182 183 for name, value in self.items(): 184 185 # Remove entries for unreferenced objects. 186 # This, due to the nature of the referenced attribute, assumes 187 # that only explicitly mentioned classes and functions are 188 # employed in the final program. 189 190 if isinstance(value, Attr): 191 192 # Only remove entries for classes and functions, not methods. 193 194 for attr_value in value.get_values(): 195 if (isinstance(attr_value, Function) and not attr_value.is_method() or 196 isinstance(attr_value, Class)) and not attr_value.referenced: 197 pass 198 else: 199 break 200 else: 201 del self[name] 202 203 all_objects = list(self.all_objects) 204 205 for obj in all_objects: 206 207 # Only remove entries for classes and functions, not methods. 208 209 if (isinstance(obj, Function) and not obj.is_method() or 210 isinstance(obj, Class)) and not obj.referenced: 211 212 self.all_objects.remove(obj) 213 obj_was_removed = 1 214 215 else: 216 obj_was_removed = 0 217 218 # Remove unused entries from classes plus associated methods. 219 220 if isinstance(obj, Class): 221 for name, attr in obj.class_attributes().items(): 222 223 # In removed classes, all explicitly defined methods can 224 # be removed. 225 226 # In other classes, methods can only be removed if they 227 # are the only assigned object to the class for a 228 # particular attribute and are unreferenced. 229 230 if obj_was_removed and isinstance(attr.get_value(), Function) and attr.get_value().is_method() or \ 231 name not in self.importer.names_used and \ 232 attr.assignments == 1 and isinstance(attr.get_value(), Function) and \ 233 attr.get_value().is_method() and not attr.get_value().referenced: 234 235 method = attr.get_value() 236 237 if method in self.all_objects: 238 self.all_objects.remove(method) 239 del obj[name] 240 241 def finalise(self): 242 243 "Finalise the module." 244 245 for obj in self.all_objects: 246 if isinstance(obj, (Class, Function)): 247 obj.finalise_attributes() 248 249 def add_object(self, obj, any_scope=0): 250 251 """ 252 Record 'obj' if non-local or if the optional 'any_scope' is set to a 253 true value. 254 """ 255 256 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 257 self.all_objects.add(obj) 258 259 # Optimisation tests. 260 261 def should_optimise_unused_objects(self): 262 return "unused_objects" in self.optimisations 263 264 # Namespace methods. 265 266 def store(self, name, obj): 267 268 "Record attribute or local 'name', storing 'obj'." 269 270 if not self.namespaces: 271 self.set(name, obj, not self.in_loop) 272 else: 273 self.namespaces[-1].set(name, obj, not self.in_loop) 274 275 def store_lambda(self, obj): 276 277 "Store a lambda function 'obj'." 278 279 self.add_object(obj) 280 281 def store_module_attr(self, name, module): 282 283 """ 284 Record module attribute 'name' in the given 'module' using the current 285 expression. 286 """ 287 288 module.set(name, self.expr, 0) 289 290 def store_class_attr(self, name): 291 292 """ 293 Record class attribute 'name' in the current class using the current 294 expression. 295 """ 296 297 if self.in_method and self.namespaces[-2].has_key(name): 298 self.namespaces[-2].set(name, self.expr, 0) 299 return 1 300 301 return 0 302 303 def store_instance_attr(self, name): 304 305 "Record instance attribute 'name' in the current class." 306 307 if self.in_method: 308 309 # Current namespace is the function. 310 # Previous namespace is the class. 311 312 self.namespaces[-2].add_instance_attribute(name) 313 314 def get_parent(self): 315 316 "Return the parent (or most recent) namespace currently exposed." 317 318 return (self.namespaces[-1:] or [self])[0] 319 320 # Visitor methods. 321 322 def default(self, node, *args): 323 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 324 325 def dispatch(self, node, *args): 326 return ASTVisitor.dispatch(self, node, *args) 327 328 def NOP(self, node): 329 for n in node.getChildNodes(): 330 self.dispatch(n) 331 return None 332 333 def OP(self, node): 334 for n in node.getChildNodes(): 335 self.dispatch(n) 336 return Instance() 337 338 def _visitUnary(self, node): 339 340 "Accounting method for the unary operator 'node'." 341 342 method = unary_methods[node.__class__.__name__] 343 self.importer.use_name(method) 344 return self.OP(node) 345 346 def _visitBinary(self, node): 347 348 "Accounting method for the binary operator 'node'." 349 350 left_method, right_method = binary_methods[node.__class__.__name__] 351 self.importer.use_name(left_method) 352 self.importer.use_name(right_method) 353 return self.OP(node) 354 355 def _visitFunction(self, node, name): 356 357 """ 358 Return a function object for the function defined by 'node' with the 359 given 'name'. If a lambda expression is being visited, 'name' should be 360 None. 361 """ 362 363 # Define the function object. 364 365 function = Function( 366 name, 367 self.get_parent(), 368 node.argnames, 369 node.defaults, 370 (node.flags & 4 != 0), 371 (node.flags & 8 != 0), 372 self, 373 node 374 ) 375 376 self.add_object(function, any_scope=1) 377 378 # Make a back reference from the node for code generation. 379 380 node.unit = function 381 382 # Process the defaults. 383 384 for n in node.defaults: 385 self.expr = self.dispatch(n) 386 function.store_default(self.expr) 387 388 self.functions.append((node, self.namespaces + [function])) 389 390 if name is not None: 391 self.store(name, function) 392 else: 393 self.store_lambda(function) 394 395 # Lambda functions are always assumed to be referenced. This is 396 # because other means of discovering the referencing of objects rely 397 # on the original names inherent in the definition of those objects. 398 399 function.set_referenced() 400 401 # Where defaults exist, an instance needs creating. Thus, it makes 402 # no sense to return a reference to the function here, since the 403 # recipient will not be referencing the function itself. 404 405 if node.defaults: 406 return Instance() # indicates no known target 407 408 return function 409 410 def _visitFunctionBody(self, node, namespaces): 411 412 "Enter the function." 413 414 # Current namespace is the function. 415 # Previous namespace is the class. 416 417 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 418 if namespaces[-1].name == "__init__": 419 self.in_init = 1 420 self.in_method = 1 421 422 self.namespaces = namespaces 423 self.dispatch(node.code) 424 425 self.in_init = 0 426 self.in_method = 0 427 428 # Specific handler methods. 429 430 visitAdd = _visitBinary 431 432 visitAnd = OP 433 434 visitAssert = NOP 435 436 def visitAssign(self, node): 437 self.expr = self.dispatch(node.expr) 438 for n in node.nodes: 439 self.dispatch(n) 440 return None 441 442 def visitAssAttr(self, node): 443 expr = self.dispatch(node.expr) 444 if isinstance(expr, Attr): 445 if expr.name == "self": 446 if not self.store_class_attr(node.attrname): 447 self.store_instance_attr(node.attrname) 448 elif isinstance(expr.get_value(), Module): 449 self.store_module_attr(node.attrname, expr.get_value()) 450 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 451 return None 452 453 def visitAssList(self, node): 454 455 # Declare names which will be used by generated code. 456 457 self.importer.use_name("__getitem__") 458 459 # Process the assignment. 460 461 for i, n in enumerate(node.nodes): 462 self.dispatch(n) 463 self.importer.make_constant(i) # for __getitem__(i) at run-time 464 return None 465 466 def visitAssName(self, node): 467 self.store(node.name, self.expr) 468 return None 469 470 visitAssTuple = visitAssList 471 472 def visitAugAssign(self, node): 473 474 # Accounting. 475 476 aug_method, (left_method, right_method) = augassign_methods[node.op] 477 self.importer.use_name(aug_method) 478 self.importer.use_name(left_method) 479 self.importer.use_name(right_method) 480 return self.OP(node) 481 482 visitBackquote = OP 483 484 visitBitand = _visitBinary 485 486 visitBitor = _visitBinary 487 488 visitBitxor = _visitBinary 489 490 visitBreak = NOP 491 492 visitCallFunc = OP 493 494 def visitClass(self, node): 495 496 """ 497 Register the class at the given 'node' subject to the restrictions 498 mentioned in the module docstring. 499 """ 500 501 if self.namespaces: 502 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 503 return None 504 else: 505 cls = Class(node.name, self.get_parent(), self, node) 506 507 # Visit the base class expressions, attempting to find concrete 508 # definitions of classes. 509 510 for base in node.bases: 511 expr = self.dispatch(base) 512 if isinstance(expr, Attr): 513 if expr.assignments != 1: 514 raise InspectError(self.full_name(), node, 515 "Base class %r for %r is not constant." % (base, cls.full_name())) 516 else: 517 cls.add_base(expr.get_value()) 518 else: # if expr is None: 519 raise InspectError(self.full_name(), node, 520 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 521 522 # NOTE: Potentially dubious measure to permit __init__ availability. 523 # If no bases exist, adopt the 'object' class. 524 525 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 526 expr = self.dispatch(compiler.ast.Name("object")) 527 cls.add_base(expr.get_value()) 528 529 # Make a back reference from the node for code generation. 530 531 node.unit = cls 532 533 # Make an entry for the class. 534 535 self.store(node.name, cls) 536 self.add_object(cls) 537 538 # Process the class body. 539 540 self.namespaces.append(cls) 541 self.dispatch(node.code) 542 self.namespaces.pop() 543 544 return cls 545 546 def visitCompare(self, node): 547 548 # Accounting. 549 # NOTE: Replicates some code in micropython.ast.visitCompare. 550 551 for op in node.ops: 552 op_name, next_node = op 553 methods = comparison_methods[op_name] 554 if methods is not None: 555 self.importer.use_name(methods[0]) 556 self.importer.use_name(methods[1]) 557 elif op_name.endswith("in"): 558 self.importer.use_name("__contains__") 559 560 return self.OP(node) 561 562 def visitConst(self, node): 563 564 # Register the constant, if necessary, returning the resulting object. 565 566 return self.importer.make_constant(node.value) 567 568 visitContinue = NOP 569 570 visitDecorators = NOP 571 572 visitDict = OP 573 574 visitDiscard = NOP 575 576 visitDiv = _visitBinary 577 578 visitEllipsis = NOP 579 580 visitExec = NOP 581 582 visitExpression = OP 583 584 visitFloorDiv = _visitBinary 585 586 def visitFor(self, node): 587 588 # Declare names which will be used by generated code. 589 590 self.importer.use_name("__iter__") 591 self.importer.use_name("next") 592 593 # Enter the loop. 594 595 self.in_loop = 1 596 self.NOP(node) 597 self.in_loop = 0 598 return None 599 600 def visitFrom(self, node): 601 module = self.importer.load(node.modname, 1) 602 603 if module is not None: 604 module.set_referenced() 605 606 #if module is None: 607 # print "Warning:", node.modname, "not imported." 608 609 for name, alias in node.names: 610 if name != "*": 611 if module is not None and module.namespace.has_key(name): 612 attr = module[name] 613 self.store(alias or name, attr) 614 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 615 self.importer.load(attr.get_value().name) 616 617 # Support the import of names from missing modules. 618 619 else: 620 self.store(alias or name, UnresolvedName(name, node.modname, self)) 621 else: 622 if module is not None: 623 for n in module.namespace.keys(): 624 attr = module[n] 625 self.store(n, attr) 626 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 627 self.importer.load(attr.get_value().name) 628 629 return None 630 631 def visitFunction(self, node): 632 return self._visitFunction(node, node.name) 633 634 visitGenExpr = OP 635 636 visitGenExprFor = NOP 637 638 visitGenExprIf = NOP 639 640 visitGenExprInner = NOP 641 642 def visitGetattr(self, node): 643 expr = self.dispatch(node.expr) 644 attrname = node.attrname 645 646 if isinstance(expr, Attr): 647 value = expr.get_value() 648 if isinstance(value, (Class, Module)): 649 attr = value.namespace.get(attrname) 650 elif isinstance(value, UnresolvedName): 651 attr = UnresolvedName(attrname, value.full_name(), self) 652 else: 653 attr = None 654 elif self.builtins is not None: 655 attr = self.builtins.get(attrname) 656 else: 657 attr = UnresolvedName(attrname, value.full_name(), self) 658 659 # Accounting. 660 661 if attr is not None: 662 attr.set_referenced() 663 664 self.importer.use_name(attrname) 665 666 return attr 667 668 def visitGlobal(self, node): 669 if self.namespaces: 670 for name in node.names: 671 ns = self.namespaces[-1] 672 if not ns.make_global(name): 673 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 674 675 # The name is recorded in an earlier process. 676 677 def visitIf(self, node): 678 for test, body in node.tests: 679 self.dispatch(test) 680 self.dispatch(body) 681 if node.else_ is not None: 682 self.dispatch(node.else_) 683 return None 684 685 visitIfExp = NOP 686 687 def visitImport(self, node): 688 for name, alias in node.names: 689 if alias is not None: 690 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 691 self.store(alias, module) 692 else: 693 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 694 self.store(name.split(".")[0], module) 695 module.set_referenced() 696 697 return None 698 699 visitInvert = _visitUnary 700 701 def visitKeyword(self, node): 702 self.dispatch(node.expr) 703 self.importer.make_constant(node.name) 704 self.keyword_names.add(node.name) 705 return None 706 707 def visitLambda(self, node): 708 return self._visitFunction(node, None) 709 710 visitLeftShift = _visitBinary 711 712 visitList = OP 713 714 visitListComp = OP 715 716 visitListCompFor = NOP 717 718 visitListCompIf = NOP 719 720 visitMod = _visitBinary 721 722 def visitModule(self, node): 723 724 # Make a back reference from the node for code generation. 725 726 node.unit = self 727 return self.dispatch(node.node) 728 729 visitMul = _visitBinary 730 731 def visitName(self, node): 732 name = node.name 733 734 if self.importer.predefined_constants.has_key(name): 735 attr = self.importer.get_predefined_constant(name) 736 elif self.namespaces and self.namespaces[-1].has_key(name): 737 attr = self.namespaces[-1][name] 738 elif self.has_key(name): 739 attr = self[name] 740 elif self.builtins is not None and self.builtins.has_key(name): 741 attr = self.builtins[name] 742 else: 743 attr = None 744 745 # Accounting. 746 747 if attr is not None: 748 attr.set_referenced() 749 750 self.importer.use_name(name) 751 752 return attr 753 754 visitNot = OP 755 756 visitOr = OP 757 758 visitPass = NOP 759 760 visitPower = _visitBinary 761 762 visitPrint = NOP 763 764 visitPrintnl = NOP 765 766 visitRaise = NOP 767 768 visitReturn = NOP 769 770 visitRightShift = _visitBinary 771 772 visitSlice = OP 773 774 visitSliceobj = OP 775 776 def visitStmt(self, node): 777 for n in node.nodes: 778 self.dispatch(n) 779 return None 780 781 visitSub = _visitBinary 782 783 def visitSubscript(self, node): 784 self.importer.use_name("__getitem__") 785 self.OP(node) 786 787 def visitTryExcept(self, node): 788 self.dispatch(node.body) 789 for name, var, n in node.handlers: 790 791 # Establish the local for the handler. 792 793 if var is not None: 794 self.dispatch(var) 795 if n is not None: 796 self.dispatch(n) 797 if node.else_ is not None: 798 self.dispatch(node.else_) 799 return None 800 801 visitTryFinally = NOP 802 803 visitTuple = OP 804 805 visitUnaryAdd = _visitUnary 806 807 visitUnarySub = _visitUnary 808 809 def visitWhile(self, node): 810 self.in_loop = 1 811 self.NOP(node) 812 self.in_loop = 0 813 return None 814 815 visitWith = NOP 816 817 visitYield = NOP 818 819 # vim: tabstop=4 expandtab shiftwidth=4