1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # Add __name__ to the namespace by adding an explicit assignment to the 133 # module. 134 135 module.node.nodes.insert(0, compiler.ast.Assign( 136 [compiler.ast.AssName("__name__", 0)], 137 compiler.ast.Const(self.full_name()) 138 )) 139 140 # First, visit module-level code, recording global names. 141 142 processed = self.dispatch(module) 143 144 # Then, for each function, detect and record globals declared in those 145 # functions. 146 147 for node, namespaces in self.functions: 148 self.process_globals(node) 149 150 # Then, visit each function, recording other names. 151 152 for node, namespaces in self.functions: 153 self._visitFunctionBody(node, namespaces) 154 155 # Add references to other modules declared using the __all__ global. 156 157 if self.has_key("__all__"): 158 all = self["__all__"] 159 if isinstance(all, compiler.ast.List): 160 for n in all.nodes: 161 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 162 163 return processed 164 165 def process_globals(self, node): 166 167 """ 168 Within the given 'node', process global declarations, adjusting the 169 module namespace. 170 """ 171 172 for n in node.getChildNodes(): 173 if isinstance(n, compiler.ast.Global): 174 for name in n.names: 175 if not self.has_key(name): 176 self[name] = None 177 else: 178 self.process_globals(n) 179 180 def vacuum(self): 181 182 """ 183 Vacuum the module namespace, removing unreferenced objects and unused 184 names. 185 """ 186 187 if self.should_optimise_unused_objects(): 188 self.vacuum_object(self) 189 190 all_objects = list(self.all_objects) 191 192 for obj in all_objects: 193 if isinstance(obj, Class): 194 self.vacuum_object(obj) 195 196 def vacuum_object(self, obj, delete_all=0): 197 198 "Vacuum the given object 'obj'." 199 200 for name, attr in obj.items(): 201 if delete_all or not self.importer.uses_name(name): 202 del obj[name] 203 if attr.assignments == 1: 204 value = attr.get_value() 205 206 # Delete any unambiguous attribute value. 207 208 if value is not obj and value in self.all_objects: 209 self.all_objects.remove(value) 210 211 # Delete class contents. 212 213 if isinstance(value, Class): 214 self.vacuum_object(value, 1) 215 216 def finalise(self): 217 218 "Finalise the module." 219 220 for obj in self.all_objects: 221 if isinstance(obj, (Class, Function)): 222 obj.finalise_attributes() 223 224 def add_object(self, obj, any_scope=0): 225 226 """ 227 Record 'obj' if non-local or if the optional 'any_scope' is set to a 228 true value. 229 """ 230 231 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 232 self.all_objects.add(obj) 233 234 # Optimisation tests. 235 236 def should_optimise_unused_objects(self): 237 return "unused_objects" in self.optimisations 238 239 # Namespace methods. 240 241 def store(self, name, obj): 242 243 "Record attribute or local 'name', storing 'obj'." 244 245 if not self.namespaces: 246 self.set(name, obj, not self.in_loop) 247 else: 248 self.namespaces[-1].set(name, obj, not self.in_loop) 249 250 def store_lambda(self, obj): 251 252 "Store a lambda function 'obj'." 253 254 self.add_object(obj) 255 256 def store_module_attr(self, name, module): 257 258 """ 259 Record module attribute 'name' in the given 'module' using the current 260 expression. 261 """ 262 263 module.set(name, self.expr, 0) 264 265 def store_class_attr(self, name): 266 267 """ 268 Record class attribute 'name' in the current class using the current 269 expression. 270 """ 271 272 if self.in_method and self.namespaces[-2].has_key(name): 273 self.namespaces[-2].set(name, self.expr, 0) 274 return 1 275 276 return 0 277 278 def store_instance_attr(self, name): 279 280 "Record instance attribute 'name' in the current class." 281 282 if self.in_method: 283 284 # Current namespace is the function. 285 # Previous namespace is the class. 286 287 self.namespaces[-2].add_instance_attribute(name) 288 289 def get_parent(self): 290 291 "Return the parent (or most recent) namespace currently exposed." 292 293 return (self.namespaces[-1:] or [self])[0] 294 295 def use_name(self, name): 296 297 "Use the given 'name' within the current namespace/unit." 298 299 unit = self.get_parent() 300 self.importer.use_name(name, unit.name) 301 302 # Visitor methods. 303 304 def default(self, node, *args): 305 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 306 307 def dispatch(self, node, *args): 308 return ASTVisitor.dispatch(self, node, *args) 309 310 def NOP(self, node): 311 for n in node.getChildNodes(): 312 self.dispatch(n) 313 return None 314 315 def OP(self, node): 316 for n in node.getChildNodes(): 317 self.dispatch(n) 318 return Instance() 319 320 def _visitUnary(self, node): 321 322 "Accounting method for the unary operator 'node'." 323 324 method = unary_methods[node.__class__.__name__] 325 self.use_name(method) 326 return self.OP(node) 327 328 def _visitBinary(self, node): 329 330 "Accounting method for the binary operator 'node'." 331 332 left_method, right_method = binary_methods[node.__class__.__name__] 333 self.use_name(left_method) 334 self.use_name(right_method) 335 return self.OP(node) 336 337 def _visitFunction(self, node, name): 338 339 """ 340 Return a function object for the function defined by 'node' with the 341 given 'name'. If a lambda expression is being visited, 'name' should be 342 None. 343 """ 344 345 # Define the function object. 346 347 function = Function( 348 name, 349 self.get_parent(), 350 node.argnames, 351 node.defaults, 352 (node.flags & 4 != 0), 353 (node.flags & 8 != 0), 354 self, 355 node 356 ) 357 358 self.add_object(function, any_scope=1) 359 360 # Make a back reference from the node for code generation. 361 362 node.unit = function 363 364 # Process the defaults. 365 366 for n in node.defaults: 367 self.expr = self.dispatch(n) 368 function.store_default(self.expr) 369 370 self.functions.append((node, self.namespaces + [function])) 371 372 if name is not None: 373 self.store(name, function) 374 else: 375 self.store_lambda(function) 376 377 # Where defaults exist, an instance needs creating. Thus, it makes 378 # no sense to return a reference to the function here, since the 379 # recipient will not be referencing the function itself. 380 381 if node.defaults: 382 return Instance() # indicates no known target 383 384 return function 385 386 def _visitFunctionBody(self, node, namespaces): 387 388 "Enter the function." 389 390 # Current namespace is the function. 391 # Previous namespace is the class. 392 393 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 394 if namespaces[-1].name == "__init__": 395 self.in_init = 1 396 self.in_method = 1 397 398 self.namespaces = namespaces 399 self.dispatch(node.code) 400 401 self.in_init = 0 402 self.in_method = 0 403 404 # Specific handler methods. 405 406 visitAdd = _visitBinary 407 408 visitAnd = OP 409 410 visitAssert = NOP 411 412 def visitAssign(self, node): 413 self.expr = self.dispatch(node.expr) 414 for n in node.nodes: 415 self.dispatch(n) 416 return None 417 418 def visitAssAttr(self, node): 419 expr = self.dispatch(node.expr) 420 if isinstance(expr, Attr): 421 if expr.name == "self": 422 if not self.store_class_attr(node.attrname): 423 self.store_instance_attr(node.attrname) 424 elif isinstance(expr.get_value(), Module): 425 self.store_module_attr(node.attrname, expr.get_value()) 426 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 427 return None 428 429 def visitAssList(self, node): 430 431 # Declare names which will be used by generated code. 432 433 self.use_name("__getitem__") 434 435 # Process the assignment. 436 437 for i, n in enumerate(node.nodes): 438 self.dispatch(n) 439 self.importer.make_constant(i) # for __getitem__(i) at run-time 440 return None 441 442 def visitAssName(self, node): 443 self.store(node.name, self.expr) 444 self.use_name(node.name) 445 return None 446 447 visitAssTuple = visitAssList 448 449 def visitAugAssign(self, node): 450 451 # Accounting. 452 453 aug_method, (left_method, right_method) = augassign_methods[node.op] 454 self.use_name(aug_method) 455 self.use_name(left_method) 456 self.use_name(right_method) 457 458 # Process the assignment. 459 460 self.expr = self.dispatch(node.expr) 461 462 # NOTE: Similar to micropython.ast handler code. 463 # NOTE: Slices and subscripts not supported. 464 465 if isinstance(node.node, compiler.ast.Name): 466 self.visitAssName(node.node) 467 elif isinstance(node.node, compiler.ast.Getattr): 468 self.visitAssAttr(node.node) 469 else: 470 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 471 472 return None 473 474 visitBackquote = OP 475 476 visitBitand = _visitBinary 477 478 visitBitor = _visitBinary 479 480 visitBitxor = _visitBinary 481 482 visitBreak = NOP 483 484 visitCallFunc = OP 485 486 def visitClass(self, node): 487 488 """ 489 Register the class at the given 'node' subject to the restrictions 490 mentioned in the module docstring. 491 """ 492 493 if self.namespaces: 494 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 495 return None 496 else: 497 cls = Class(node.name, self.get_parent(), self, node) 498 499 # Visit the base class expressions, attempting to find concrete 500 # definitions of classes. 501 502 for base in node.bases: 503 expr = self.dispatch(base) 504 if isinstance(expr, Attr): 505 if expr.assignments != 1: 506 raise InspectError(self.full_name(), node, 507 "Base class %r for %r is not constant." % (base, cls.full_name())) 508 else: 509 cls.add_base(expr.get_value()) 510 else: # if expr is None: 511 raise InspectError(self.full_name(), node, 512 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 513 514 # NOTE: Potentially dubious measure to permit __init__ availability. 515 # If no bases exist, adopt the 'object' class. 516 517 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 518 expr = self.dispatch(compiler.ast.Name("object")) 519 cls.add_base(expr.get_value()) 520 521 # Make a back reference from the node for code generation. 522 523 node.unit = cls 524 525 # Make an entry for the class. 526 527 self.store(node.name, cls) 528 self.add_object(cls) 529 530 # Process the class body. 531 532 self.namespaces.append(cls) 533 self.dispatch(node.code) 534 self.namespaces.pop() 535 536 return cls 537 538 def visitCompare(self, node): 539 540 # Accounting. 541 # NOTE: Replicates some code in micropython.ast.visitCompare. 542 543 for op in node.ops: 544 op_name, next_node = op 545 methods = comparison_methods[op_name] 546 if methods is not None: 547 self.use_name(methods[0]) 548 self.use_name(methods[1]) 549 elif op_name.endswith("in"): 550 self.use_name("__contains__") 551 552 return self.OP(node) 553 554 def visitConst(self, node): 555 556 # Register the constant, if necessary, returning the resulting object. 557 558 self.use_name(self.importer.get_constant_type_name(node.value)) 559 return self.importer.make_constant(node.value) 560 561 visitContinue = NOP 562 563 visitDecorators = NOP 564 565 visitDict = OP 566 567 visitDiscard = NOP 568 569 visitDiv = _visitBinary 570 571 visitEllipsis = NOP 572 573 visitExec = NOP 574 575 visitExpression = OP 576 577 visitFloorDiv = _visitBinary 578 579 def visitFor(self, node): 580 581 # Declare names which will be used by generated code. 582 583 self.use_name("__iter__") 584 self.use_name("next") 585 586 # Enter the loop. 587 588 self.in_loop = 1 589 self.NOP(node) 590 self.in_loop = 0 591 return None 592 593 def visitFrom(self, node): 594 module = self.importer.load(node.modname, 1) 595 596 #if module is None: 597 # print "Warning:", node.modname, "not imported." 598 599 for name, alias in node.names: 600 if name != "*": 601 if module is not None and module.namespace.has_key(name): 602 attr = module[name] 603 self.store(alias or name, attr) 604 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 605 self.importer.load(attr.get_value().name) 606 607 # Support the import of names from missing modules. 608 609 else: 610 self.store(alias or name, UnresolvedName(name, node.modname, self)) 611 else: 612 if module is not None: 613 for n in module.namespace.keys(): 614 attr = module[n] 615 self.store(n, attr) 616 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 617 self.importer.load(attr.get_value().name) 618 619 return None 620 621 def visitFunction(self, node): 622 return self._visitFunction(node, node.name) 623 624 visitGenExpr = OP 625 626 visitGenExprFor = NOP 627 628 visitGenExprIf = NOP 629 630 visitGenExprInner = NOP 631 632 def visitGetattr(self, node): 633 expr = self.dispatch(node.expr) 634 attrname = node.attrname 635 636 if isinstance(expr, Attr): 637 value = expr.get_value() 638 if isinstance(value, (Class, Module)): 639 attr = value.namespace.get(attrname) 640 elif isinstance(value, UnresolvedName): 641 attr = UnresolvedName(attrname, value.full_name(), self) 642 else: 643 attr = None 644 elif self.builtins is not None: 645 attr = self.builtins.get(attrname) 646 else: 647 attr = UnresolvedName(attrname, value.full_name(), self) 648 649 # Accounting. 650 651 self.use_name(attrname) 652 653 return attr 654 655 def visitGlobal(self, node): 656 if self.namespaces: 657 for name in node.names: 658 ns = self.namespaces[-1] 659 if not ns.make_global(name): 660 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 661 662 # The name is recorded in an earlier process. 663 664 def visitIf(self, node): 665 for test, body in node.tests: 666 self.dispatch(test) 667 self.dispatch(body) 668 if node.else_ is not None: 669 self.dispatch(node.else_) 670 return None 671 672 visitIfExp = NOP 673 674 def visitImport(self, node): 675 for name, alias in node.names: 676 if alias is not None: 677 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 678 self.store(alias, module) 679 else: 680 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 681 self.store(name.split(".")[0], module) 682 683 return None 684 685 visitInvert = _visitUnary 686 687 def visitKeyword(self, node): 688 self.dispatch(node.expr) 689 self.importer.make_constant(node.name) 690 self.keyword_names.add(node.name) 691 return None 692 693 def visitLambda(self, node): 694 self.use_name(None) # lambda functions have no names but are assumed to be invoked 695 return self._visitFunction(node, None) 696 697 visitLeftShift = _visitBinary 698 699 visitList = OP 700 701 visitListComp = OP 702 703 visitListCompFor = NOP 704 705 visitListCompIf = NOP 706 707 visitMod = _visitBinary 708 709 def visitModule(self, node): 710 711 # Make a back reference from the node for code generation. 712 713 node.unit = self 714 return self.dispatch(node.node) 715 716 visitMul = _visitBinary 717 718 def visitName(self, node): 719 name = node.name 720 721 if self.importer.predefined_constants.has_key(name): 722 attr = self.importer.get_predefined_constant(name) 723 elif self.namespaces and self.namespaces[-1].has_key(name): 724 attr = self.namespaces[-1][name] 725 elif self.has_key(name): 726 attr = self[name] 727 elif self.builtins is not None and self.builtins.has_key(name): 728 attr = self.builtins[name] 729 else: 730 attr = None 731 732 self.use_name(name) 733 return attr 734 735 visitNot = OP 736 737 visitOr = OP 738 739 visitPass = NOP 740 741 visitPower = _visitBinary 742 743 visitPrint = NOP 744 745 visitPrintnl = NOP 746 747 visitRaise = NOP 748 749 visitReturn = NOP 750 751 visitRightShift = _visitBinary 752 753 visitSlice = OP 754 755 visitSliceobj = OP 756 757 def visitStmt(self, node): 758 for n in node.nodes: 759 self.dispatch(n) 760 return None 761 762 visitSub = _visitBinary 763 764 def visitSubscript(self, node): 765 self.use_name("__getitem__") 766 self.OP(node) 767 768 def visitTryExcept(self, node): 769 self.dispatch(node.body) 770 for name, var, n in node.handlers: 771 772 # Establish the local for the handler. 773 774 if var is not None: 775 self.dispatch(var) 776 if n is not None: 777 self.dispatch(n) 778 if node.else_ is not None: 779 self.dispatch(node.else_) 780 return None 781 782 visitTryFinally = NOP 783 784 visitTuple = OP 785 786 visitUnaryAdd = _visitUnary 787 788 visitUnarySub = _visitUnary 789 790 def visitWhile(self, node): 791 self.in_loop = 1 792 self.NOP(node) 793 self.in_loop = 0 794 return None 795 796 visitWith = NOP 797 798 visitYield = NOP 799 800 # vim: tabstop=4 expandtab shiftwidth=4