1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # First, visit module-level code, recording global names. 133 134 processed = self.dispatch(module) 135 136 # Then, for each function, detect and record globals declared in those 137 # functions. 138 139 for node, namespaces in self.functions: 140 self.process_globals(node) 141 142 # Then, visit each function, recording other names. 143 144 for node, namespaces in self.functions: 145 self._visitFunctionBody(node, namespaces) 146 147 # Add references to other modules declared using the __all__ global. 148 149 if self.has_key("__all__"): 150 all = self["__all__"] 151 if isinstance(all, compiler.ast.List): 152 for n in all.nodes: 153 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 154 155 return processed 156 157 def process_globals(self, node): 158 159 """ 160 Within the given 'node', process global declarations, adjusting the 161 module namespace. 162 """ 163 164 for n in node.getChildNodes(): 165 if isinstance(n, compiler.ast.Global): 166 for name in n.names: 167 if not self.has_key(name): 168 self[name] = None 169 else: 170 self.process_globals(n) 171 172 def vacuum(self): 173 174 """ 175 Vacuum the module namespace, removing unreferenced objects and unused 176 names. 177 """ 178 179 for name, value in self.items(): 180 181 if self.should_optimise_unused_objects(): 182 183 # Remove entries for unreferenced objects. 184 # This, due to the nature of the referenced attribute, assumes 185 # that only explicitly mentioned classes and functions are 186 # employed in the final program. 187 188 if isinstance(value, Attr): 189 190 # Only remove entries for classes and functions, not methods. 191 192 for attr_value in value.get_values(): 193 if (isinstance(attr_value, Function) and not attr_value.is_method() or 194 isinstance(attr_value, Class)) and not attr_value.referenced: 195 pass 196 else: 197 break 198 else: 199 del self[name] 200 201 # Remove unreferenced objects. 202 203 if self.should_optimise_unused_objects(): 204 205 all_objects = list(self.all_objects) 206 207 for obj in all_objects: 208 209 # Only remove entries for classes and functions, not methods. 210 211 if (isinstance(obj, Function) and not obj.is_method() or 212 isinstance(obj, Class)) and not obj.referenced: 213 214 self.all_objects.remove(obj) 215 216 # Remove unused entries from classes plus associated methods. 217 218 if isinstance(obj, Class): 219 for name, attr in obj.class_attributes().items(): 220 221 # Methods can only be deleted if they are the only 222 # assigned object to the class and are unreferenced. 223 224 if name not in self.importer.names_used and \ 225 attr.assignments == 1 and isinstance(attr.get_value(), Function) and \ 226 attr.get_value().is_method() and not attr.get_value().referenced: 227 228 self.all_objects.remove(attr.get_value()) 229 del obj[name] 230 231 def finalise(self): 232 233 "Finalise the module." 234 235 for obj in self.all_objects: 236 if isinstance(obj, (Class, Function)): 237 obj.finalise_attributes() 238 239 def add_object(self, obj, any_scope=0): 240 241 """ 242 Record 'obj' if non-local or if the optional 'any_scope' is set to a 243 true value. 244 """ 245 246 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 247 self.all_objects.add(obj) 248 249 # Optimisation tests. 250 251 def should_optimise_unused_objects(self): 252 return "unused_objects" in self.optimisations 253 254 # Namespace methods. 255 256 def store(self, name, obj): 257 258 "Record attribute or local 'name', storing 'obj'." 259 260 if not self.namespaces: 261 self.set(name, obj, not self.in_loop) 262 else: 263 self.namespaces[-1].set(name, obj, not self.in_loop) 264 265 def store_lambda(self, obj): 266 267 "Store a lambda function 'obj'." 268 269 self.add_object(obj) 270 271 def store_module_attr(self, name, module): 272 273 """ 274 Record module attribute 'name' in the given 'module' using the current 275 expression. 276 """ 277 278 module.set(name, self.expr, 0) 279 280 def store_class_attr(self, name): 281 282 """ 283 Record class attribute 'name' in the current class using the current 284 expression. 285 """ 286 287 if self.in_method and self.namespaces[-2].has_key(name): 288 self.namespaces[-2].set(name, self.expr, 0) 289 return 1 290 291 return 0 292 293 def store_instance_attr(self, name): 294 295 "Record instance attribute 'name' in the current class." 296 297 if self.in_method: 298 299 # Current namespace is the function. 300 # Previous namespace is the class. 301 302 self.namespaces[-2].add_instance_attribute(name) 303 304 def get_parent(self): 305 306 "Return the parent (or most recent) namespace currently exposed." 307 308 return (self.namespaces[-1:] or [self])[0] 309 310 # Visitor methods. 311 312 def default(self, node, *args): 313 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 314 315 def dispatch(self, node, *args): 316 return ASTVisitor.dispatch(self, node, *args) 317 318 def NOP(self, node): 319 for n in node.getChildNodes(): 320 self.dispatch(n) 321 return None 322 323 def OP(self, node): 324 for n in node.getChildNodes(): 325 self.dispatch(n) 326 return Instance() 327 328 def _visitUnary(self, node): 329 330 "Accounting method for the unary operator 'node'." 331 332 method = unary_methods[node.__class__.__name__] 333 self.importer.use_name(method) 334 return self.OP(node) 335 336 def _visitBinary(self, node): 337 338 "Accounting method for the binary operator 'node'." 339 340 left_method, right_method = binary_methods[node.__class__.__name__] 341 self.importer.use_name(left_method) 342 self.importer.use_name(right_method) 343 return self.OP(node) 344 345 def _visitFunction(self, node, name): 346 347 """ 348 Return a function object for the function defined by 'node' with the 349 given 'name'. If a lambda expression is being visited, 'name' should be 350 None. 351 """ 352 353 # Define the function object. 354 355 function = Function( 356 name, 357 self.get_parent(), 358 node.argnames, 359 node.defaults, 360 (node.flags & 4 != 0), 361 (node.flags & 8 != 0), 362 self, 363 node 364 ) 365 366 self.add_object(function, any_scope=1) 367 368 # Make a back reference from the node for code generation. 369 370 node.unit = function 371 372 # Process the defaults. 373 374 for n in node.defaults: 375 self.expr = self.dispatch(n) 376 function.store_default(self.expr) 377 378 self.functions.append((node, self.namespaces + [function])) 379 380 if name is not None: 381 self.store(name, function) 382 else: 383 self.store_lambda(function) 384 385 # Lambda functions are always assumed to be referenced. This is 386 # because other means of discovering the referencing of objects rely 387 # on the original names inherent in the definition of those objects. 388 389 function.set_referenced() 390 391 # Where defaults exist, an instance needs creating. Thus, it makes 392 # no sense to return a reference to the function here, since the 393 # recipient will not be referencing the function itself. 394 395 if node.defaults: 396 return Instance() # indicates no known target 397 398 return function 399 400 def _visitFunctionBody(self, node, namespaces): 401 402 "Enter the function." 403 404 # Current namespace is the function. 405 # Previous namespace is the class. 406 407 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 408 if namespaces[-1].name == "__init__": 409 self.in_init = 1 410 self.in_method = 1 411 412 self.namespaces = namespaces 413 self.dispatch(node.code) 414 415 self.in_init = 0 416 self.in_method = 0 417 418 # Specific handler methods. 419 420 visitAdd = _visitBinary 421 422 visitAnd = OP 423 424 visitAssert = NOP 425 426 def visitAssign(self, node): 427 self.expr = self.dispatch(node.expr) 428 for n in node.nodes: 429 self.dispatch(n) 430 return None 431 432 def visitAssAttr(self, node): 433 expr = self.dispatch(node.expr) 434 if isinstance(expr, Attr): 435 if expr.name == "self": 436 if not self.store_class_attr(node.attrname): 437 self.store_instance_attr(node.attrname) 438 elif isinstance(expr.get_value(), Module): 439 self.store_module_attr(node.attrname, expr.get_value()) 440 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 441 return None 442 443 def visitAssList(self, node): 444 445 # Declare names which will be used by generated code. 446 447 self.importer.use_name("__getitem__") 448 449 # Process the assignment. 450 451 for i, n in enumerate(node.nodes): 452 self.dispatch(n) 453 self.importer.make_constant(i) # for __getitem__(i) at run-time 454 return None 455 456 def visitAssName(self, node): 457 self.store(node.name, self.expr) 458 return None 459 460 visitAssTuple = visitAssList 461 462 def visitAugAssign(self, node): 463 464 # Accounting. 465 466 aug_method, (left_method, right_method) = augassign_methods[node.op] 467 self.importer.use_name(aug_method) 468 self.importer.use_name(left_method) 469 self.importer.use_name(right_method) 470 return self.OP(node) 471 472 visitBackquote = OP 473 474 visitBitand = _visitBinary 475 476 visitBitor = _visitBinary 477 478 visitBitxor = _visitBinary 479 480 visitBreak = NOP 481 482 visitCallFunc = OP 483 484 def visitClass(self, node): 485 486 """ 487 Register the class at the given 'node' subject to the restrictions 488 mentioned in the module docstring. 489 """ 490 491 if self.namespaces: 492 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 493 return None 494 else: 495 cls = Class(node.name, self.get_parent(), self, node) 496 497 # Visit the base class expressions, attempting to find concrete 498 # definitions of classes. 499 500 for base in node.bases: 501 expr = self.dispatch(base) 502 if isinstance(expr, Attr): 503 if expr.assignments != 1: 504 raise InspectError(self.full_name(), node, 505 "Base class %r for %r is not constant." % (base, cls.full_name())) 506 else: 507 cls.add_base(expr.get_value()) 508 else: # if expr is None: 509 raise InspectError(self.full_name(), node, 510 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 511 512 # NOTE: Potentially dubious measure to permit __init__ availability. 513 # If no bases exist, adopt the 'object' class. 514 515 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 516 expr = self.dispatch(compiler.ast.Name("object")) 517 cls.add_base(expr.get_value()) 518 519 # Make a back reference from the node for code generation. 520 521 node.unit = cls 522 523 # Make an entry for the class. 524 525 self.store(node.name, cls) 526 self.add_object(cls) 527 528 # Process the class body. 529 530 self.namespaces.append(cls) 531 self.dispatch(node.code) 532 self.namespaces.pop() 533 534 return cls 535 536 def visitCompare(self, node): 537 538 # Accounting. 539 # NOTE: Replicates some code in micropython.ast.visitCompare. 540 541 for op in node.ops: 542 op_name, next_node = op 543 methods = comparison_methods[op_name] 544 if methods is not None: 545 self.importer.use_name(methods[0]) 546 self.importer.use_name(methods[1]) 547 elif op_name.endswith("in"): 548 self.importer.use_name("__contains__") 549 550 return self.OP(node) 551 552 def visitConst(self, node): 553 554 # Register the constant, if necessary, returning the resulting object. 555 556 return self.importer.make_constant(node.value) 557 558 visitContinue = NOP 559 560 visitDecorators = NOP 561 562 visitDict = OP 563 564 visitDiscard = NOP 565 566 visitDiv = _visitBinary 567 568 visitEllipsis = NOP 569 570 visitExec = NOP 571 572 visitExpression = OP 573 574 visitFloorDiv = _visitBinary 575 576 def visitFor(self, node): 577 578 # Declare names which will be used by generated code. 579 580 self.importer.use_name("__iter__") 581 self.importer.use_name("next") 582 583 # Enter the loop. 584 585 self.in_loop = 1 586 self.NOP(node) 587 self.in_loop = 0 588 return None 589 590 def visitFrom(self, node): 591 module = self.importer.load(node.modname, 1) 592 593 if module is not None: 594 module.set_referenced() 595 596 #if module is None: 597 # print "Warning:", node.modname, "not imported." 598 599 for name, alias in node.names: 600 if name != "*": 601 if module is not None and module.namespace.has_key(name): 602 attr = module[name] 603 self.store(alias or name, attr) 604 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 605 self.importer.load(attr.get_value().name) 606 607 # Support the import of names from missing modules. 608 609 else: 610 self.store(alias or name, UnresolvedName(name, node.modname, self)) 611 else: 612 if module is not None: 613 for n in module.namespace.keys(): 614 attr = module[n] 615 self.store(n, attr) 616 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 617 self.importer.load(attr.get_value().name) 618 619 return None 620 621 def visitFunction(self, node): 622 return self._visitFunction(node, node.name) 623 624 visitGenExpr = OP 625 626 visitGenExprFor = NOP 627 628 visitGenExprIf = NOP 629 630 visitGenExprInner = NOP 631 632 def visitGetattr(self, node): 633 expr = self.dispatch(node.expr) 634 attrname = node.attrname 635 636 if isinstance(expr, Attr): 637 value = expr.get_value() 638 if isinstance(value, (Class, Module)): 639 attr = value.namespace.get(attrname) 640 elif isinstance(value, UnresolvedName): 641 attr = UnresolvedName(attrname, value.full_name(), self) 642 else: 643 attr = None 644 elif self.builtins is not None: 645 attr = self.builtins.get(attrname) 646 else: 647 attr = UnresolvedName(attrname, value.full_name(), self) 648 649 # Accounting. 650 651 if attr is not None: 652 attr.set_referenced() 653 654 self.importer.use_name(attrname) 655 656 return attr 657 658 def visitGlobal(self, node): 659 if self.namespaces: 660 for name in node.names: 661 ns = self.namespaces[-1] 662 if not ns.make_global(name): 663 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 664 665 # The name is recorded in an earlier process. 666 667 def visitIf(self, node): 668 for test, body in node.tests: 669 self.dispatch(test) 670 self.dispatch(body) 671 if node.else_ is not None: 672 self.dispatch(node.else_) 673 return None 674 675 visitIfExp = NOP 676 677 def visitImport(self, node): 678 for name, alias in node.names: 679 if alias is not None: 680 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 681 self.store(alias, module) 682 else: 683 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 684 self.store(name.split(".")[0], module) 685 module.set_referenced() 686 687 return None 688 689 visitInvert = _visitUnary 690 691 def visitKeyword(self, node): 692 self.dispatch(node.expr) 693 self.importer.make_constant(node.name) 694 self.keyword_names.add(node.name) 695 return None 696 697 def visitLambda(self, node): 698 return self._visitFunction(node, None) 699 700 visitLeftShift = _visitBinary 701 702 visitList = OP 703 704 visitListComp = OP 705 706 visitListCompFor = NOP 707 708 visitListCompIf = NOP 709 710 visitMod = _visitBinary 711 712 def visitModule(self, node): 713 714 # Make a back reference from the node for code generation. 715 716 node.unit = self 717 return self.dispatch(node.node) 718 719 visitMul = _visitBinary 720 721 def visitName(self, node): 722 name = node.name 723 724 if self.importer.predefined_constants.has_key(name): 725 attr = self.importer.get_predefined_constant(name) 726 elif self.namespaces and self.namespaces[-1].has_key(name): 727 attr = self.namespaces[-1][name] 728 elif self.has_key(name): 729 attr = self[name] 730 elif self.builtins is not None and self.builtins.has_key(name): 731 attr = self.builtins[name] 732 else: 733 attr = None 734 735 # Accounting. 736 737 if attr is not None: 738 attr.set_referenced() 739 740 self.importer.use_name(name) 741 742 return attr 743 744 visitNot = OP 745 746 visitOr = OP 747 748 visitPass = NOP 749 750 visitPower = _visitBinary 751 752 visitPrint = NOP 753 754 visitPrintnl = NOP 755 756 visitRaise = NOP 757 758 visitReturn = NOP 759 760 visitRightShift = _visitBinary 761 762 visitSlice = OP 763 764 visitSliceobj = OP 765 766 def visitStmt(self, node): 767 for n in node.nodes: 768 self.dispatch(n) 769 return None 770 771 visitSub = _visitBinary 772 773 def visitSubscript(self, node): 774 self.importer.use_name("__getitem__") 775 self.OP(node) 776 777 def visitTryExcept(self, node): 778 self.dispatch(node.body) 779 for name, var, n in node.handlers: 780 781 # Establish the local for the handler. 782 783 if var is not None: 784 self.dispatch(var) 785 if n is not None: 786 self.dispatch(n) 787 if node.else_ is not None: 788 self.dispatch(node.else_) 789 return None 790 791 visitTryFinally = NOP 792 793 visitTuple = OP 794 795 visitUnaryAdd = _visitUnary 796 797 visitUnarySub = _visitUnary 798 799 def visitWhile(self, node): 800 self.in_loop = 1 801 self.NOP(node) 802 self.in_loop = 0 803 return None 804 805 visitWith = NOP 806 807 visitYield = NOP 808 809 # vim: tabstop=4 expandtab shiftwidth=4