1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 self.functions = [] 118 119 def parse(self, filename): 120 121 "Parse the file having the given 'filename'." 122 123 module = compiler.parseFile(filename) 124 self.process(module) 125 126 def process(self, module): 127 128 "Process the given 'module'." 129 130 self.astnode = self.module = module 131 132 # First, visit module-level code, recording global names. 133 134 processed = self.dispatch(module) 135 136 # Then, for each function, detect and record globals declared in those 137 # functions. 138 139 for node, namespaces in self.functions: 140 self.process_globals(node) 141 142 # Then, visit each function, recording other names. 143 144 for node, namespaces in self.functions: 145 self._visitFunctionBody(node, namespaces) 146 147 # Add references to other modules declared using the __all__ global. 148 149 if self.has_key("__all__"): 150 all = self["__all__"] 151 if isinstance(all, compiler.ast.List): 152 for n in all.nodes: 153 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 154 155 return processed 156 157 def process_globals(self, node): 158 159 """ 160 Within the given 'node', process global declarations, adjusting the 161 module namespace. 162 """ 163 164 for n in node.getChildNodes(): 165 if isinstance(n, compiler.ast.Global): 166 for name in n.names: 167 if not self.has_key(name): 168 self[name] = None 169 else: 170 self.process_globals(n) 171 172 def vacuum(self): 173 174 """ 175 Vacuum the module namespace, removing unreferenced objects and unused 176 names. 177 """ 178 179 if self.should_optimise_unused_objects(): 180 self.vacuum_object(self) 181 182 all_objects = list(self.all_objects) 183 184 for obj in all_objects: 185 if isinstance(obj, Class): 186 self.vacuum_object(obj) 187 188 def vacuum_object(self, obj, delete_all=0): 189 190 "Vacuum the given object 'obj'." 191 192 for name, attr in obj.items(): 193 if delete_all or not self.importer.uses_name(name): 194 del obj[name] 195 if attr.assignments == 1: 196 value = attr.get_value() 197 198 # Delete any unambiguous attribute value. 199 200 if value is not obj and value in self.all_objects: 201 self.all_objects.remove(value) 202 203 # Delete class contents. 204 205 if isinstance(value, Class): 206 self.vacuum_object(value, 1) 207 208 def finalise(self): 209 210 "Finalise the module." 211 212 for obj in self.all_objects: 213 if isinstance(obj, (Class, Function)): 214 obj.finalise_attributes() 215 216 def add_object(self, obj, any_scope=0): 217 218 """ 219 Record 'obj' if non-local or if the optional 'any_scope' is set to a 220 true value. 221 """ 222 223 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 224 self.all_objects.add(obj) 225 226 # Optimisation tests. 227 228 def should_optimise_unused_objects(self): 229 return "unused_objects" in self.optimisations 230 231 # Namespace methods. 232 233 def store(self, name, obj): 234 235 "Record attribute or local 'name', storing 'obj'." 236 237 if not self.namespaces: 238 self.set(name, obj, not self.in_loop) 239 else: 240 self.namespaces[-1].set(name, obj, not self.in_loop) 241 242 def store_lambda(self, obj): 243 244 "Store a lambda function 'obj'." 245 246 self.add_object(obj) 247 248 def store_module_attr(self, name, module): 249 250 """ 251 Record module attribute 'name' in the given 'module' using the current 252 expression. 253 """ 254 255 module.set(name, self.expr, 0) 256 257 def store_class_attr(self, name): 258 259 """ 260 Record class attribute 'name' in the current class using the current 261 expression. 262 """ 263 264 if self.in_method and self.namespaces[-2].has_key(name): 265 self.namespaces[-2].set(name, self.expr, 0) 266 return 1 267 268 return 0 269 270 def store_instance_attr(self, name): 271 272 "Record instance attribute 'name' in the current class." 273 274 if self.in_method: 275 276 # Current namespace is the function. 277 # Previous namespace is the class. 278 279 self.namespaces[-2].add_instance_attribute(name) 280 281 def get_parent(self): 282 283 "Return the parent (or most recent) namespace currently exposed." 284 285 return (self.namespaces[-1:] or [self])[0] 286 287 def use_name(self, name): 288 289 "Use the given 'name' within the current namespace/unit." 290 291 unit = self.get_parent() 292 self.importer.use_name(name, unit.name) 293 294 # Visitor methods. 295 296 def default(self, node, *args): 297 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 298 299 def dispatch(self, node, *args): 300 return ASTVisitor.dispatch(self, node, *args) 301 302 def NOP(self, node): 303 for n in node.getChildNodes(): 304 self.dispatch(n) 305 return None 306 307 def OP(self, node): 308 for n in node.getChildNodes(): 309 self.dispatch(n) 310 return Instance() 311 312 def _visitUnary(self, node): 313 314 "Accounting method for the unary operator 'node'." 315 316 method = unary_methods[node.__class__.__name__] 317 self.use_name(method) 318 return self.OP(node) 319 320 def _visitBinary(self, node): 321 322 "Accounting method for the binary operator 'node'." 323 324 left_method, right_method = binary_methods[node.__class__.__name__] 325 self.use_name(left_method) 326 self.use_name(right_method) 327 return self.OP(node) 328 329 def _visitFunction(self, node, name): 330 331 """ 332 Return a function object for the function defined by 'node' with the 333 given 'name'. If a lambda expression is being visited, 'name' should be 334 None. 335 """ 336 337 # Define the function object. 338 339 function = Function( 340 name, 341 self.get_parent(), 342 node.argnames, 343 node.defaults, 344 (node.flags & 4 != 0), 345 (node.flags & 8 != 0), 346 self, 347 node 348 ) 349 350 self.add_object(function, any_scope=1) 351 352 # Make a back reference from the node for code generation. 353 354 node.unit = function 355 356 # Process the defaults. 357 358 for n in node.defaults: 359 self.expr = self.dispatch(n) 360 function.store_default(self.expr) 361 362 self.functions.append((node, self.namespaces + [function])) 363 364 if name is not None: 365 self.store(name, function) 366 else: 367 self.store_lambda(function) 368 369 # Where defaults exist, an instance needs creating. Thus, it makes 370 # no sense to return a reference to the function here, since the 371 # recipient will not be referencing the function itself. 372 373 if node.defaults: 374 return Instance() # indicates no known target 375 376 return function 377 378 def _visitFunctionBody(self, node, namespaces): 379 380 "Enter the function." 381 382 # Current namespace is the function. 383 # Previous namespace is the class. 384 385 if len(namespaces) > 1 and isinstance(namespaces[-2], Class): 386 if namespaces[-1].name == "__init__": 387 self.in_init = 1 388 self.in_method = 1 389 390 self.namespaces = namespaces 391 self.dispatch(node.code) 392 393 self.in_init = 0 394 self.in_method = 0 395 396 # Specific handler methods. 397 398 visitAdd = _visitBinary 399 400 visitAnd = OP 401 402 visitAssert = NOP 403 404 def visitAssign(self, node): 405 self.expr = self.dispatch(node.expr) 406 for n in node.nodes: 407 self.dispatch(n) 408 return None 409 410 def visitAssAttr(self, node): 411 expr = self.dispatch(node.expr) 412 if isinstance(expr, Attr): 413 if expr.name == "self": 414 if not self.store_class_attr(node.attrname): 415 self.store_instance_attr(node.attrname) 416 elif isinstance(expr.get_value(), Module): 417 self.store_module_attr(node.attrname, expr.get_value()) 418 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 419 return None 420 421 def visitAssList(self, node): 422 423 # Declare names which will be used by generated code. 424 425 self.use_name("__getitem__") 426 427 # Process the assignment. 428 429 for i, n in enumerate(node.nodes): 430 self.dispatch(n) 431 self.importer.make_constant(i) # for __getitem__(i) at run-time 432 return None 433 434 def visitAssName(self, node): 435 self.store(node.name, self.expr) 436 self.use_name(node.name) 437 return None 438 439 visitAssTuple = visitAssList 440 441 def visitAugAssign(self, node): 442 443 # Accounting. 444 445 aug_method, (left_method, right_method) = augassign_methods[node.op] 446 self.use_name(aug_method) 447 self.use_name(left_method) 448 self.use_name(right_method) 449 450 # Process the assignment. 451 452 self.expr = self.dispatch(node.expr) 453 454 # NOTE: Similar to micropython.ast handler code. 455 # NOTE: Slices and subscripts not supported. 456 457 if isinstance(node.node, compiler.ast.Name): 458 self.visitAssName(node.node) 459 elif isinstance(node.node, compiler.ast.Getattr): 460 self.visitAssAttr(node.node) 461 else: 462 raise InspectError(self.full_name(), node, "AugAssign(Slice or Subscript)") 463 464 return None 465 466 visitBackquote = OP 467 468 visitBitand = _visitBinary 469 470 visitBitor = _visitBinary 471 472 visitBitxor = _visitBinary 473 474 visitBreak = NOP 475 476 visitCallFunc = OP 477 478 def visitClass(self, node): 479 480 """ 481 Register the class at the given 'node' subject to the restrictions 482 mentioned in the module docstring. 483 """ 484 485 if self.namespaces: 486 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 487 return None 488 else: 489 cls = Class(node.name, self.get_parent(), self, node) 490 491 # Visit the base class expressions, attempting to find concrete 492 # definitions of classes. 493 494 for base in node.bases: 495 expr = self.dispatch(base) 496 if isinstance(expr, Attr): 497 if expr.assignments != 1: 498 raise InspectError(self.full_name(), node, 499 "Base class %r for %r is not constant." % (base, cls.full_name())) 500 else: 501 cls.add_base(expr.get_value()) 502 else: # if expr is None: 503 raise InspectError(self.full_name(), node, 504 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 505 506 # NOTE: Potentially dubious measure to permit __init__ availability. 507 # If no bases exist, adopt the 'object' class. 508 509 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 510 expr = self.dispatch(compiler.ast.Name("object")) 511 cls.add_base(expr.get_value()) 512 513 # Make a back reference from the node for code generation. 514 515 node.unit = cls 516 517 # Make an entry for the class. 518 519 self.store(node.name, cls) 520 self.add_object(cls) 521 522 # Process the class body. 523 524 self.namespaces.append(cls) 525 self.dispatch(node.code) 526 self.namespaces.pop() 527 528 return cls 529 530 def visitCompare(self, node): 531 532 # Accounting. 533 # NOTE: Replicates some code in micropython.ast.visitCompare. 534 535 for op in node.ops: 536 op_name, next_node = op 537 methods = comparison_methods[op_name] 538 if methods is not None: 539 self.use_name(methods[0]) 540 self.use_name(methods[1]) 541 elif op_name.endswith("in"): 542 self.use_name("__contains__") 543 544 return self.OP(node) 545 546 def visitConst(self, node): 547 548 # Register the constant, if necessary, returning the resulting object. 549 550 self.use_name(self.importer.get_constant_type_name(node.value)) 551 return self.importer.make_constant(node.value) 552 553 visitContinue = NOP 554 555 visitDecorators = NOP 556 557 visitDict = OP 558 559 visitDiscard = NOP 560 561 visitDiv = _visitBinary 562 563 visitEllipsis = NOP 564 565 visitExec = NOP 566 567 visitExpression = OP 568 569 visitFloorDiv = _visitBinary 570 571 def visitFor(self, node): 572 573 # Declare names which will be used by generated code. 574 575 self.use_name("__iter__") 576 self.use_name("next") 577 578 # Enter the loop. 579 580 self.in_loop = 1 581 self.NOP(node) 582 self.in_loop = 0 583 return None 584 585 def visitFrom(self, node): 586 module = self.importer.load(node.modname, 1) 587 588 #if module is None: 589 # print "Warning:", node.modname, "not imported." 590 591 for name, alias in node.names: 592 if name != "*": 593 if module is not None and module.namespace.has_key(name): 594 attr = module[name] 595 self.store(alias or name, attr) 596 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 597 self.importer.load(attr.get_value().name) 598 599 # Support the import of names from missing modules. 600 601 else: 602 self.store(alias or name, UnresolvedName(name, node.modname, self)) 603 else: 604 if module is not None: 605 for n in module.namespace.keys(): 606 attr = module[n] 607 self.store(n, attr) 608 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 609 self.importer.load(attr.get_value().name) 610 611 return None 612 613 def visitFunction(self, node): 614 return self._visitFunction(node, node.name) 615 616 visitGenExpr = OP 617 618 visitGenExprFor = NOP 619 620 visitGenExprIf = NOP 621 622 visitGenExprInner = NOP 623 624 def visitGetattr(self, node): 625 expr = self.dispatch(node.expr) 626 attrname = node.attrname 627 628 if isinstance(expr, Attr): 629 value = expr.get_value() 630 if isinstance(value, (Class, Module)): 631 attr = value.namespace.get(attrname) 632 elif isinstance(value, UnresolvedName): 633 attr = UnresolvedName(attrname, value.full_name(), self) 634 else: 635 attr = None 636 elif self.builtins is not None: 637 attr = self.builtins.get(attrname) 638 else: 639 attr = UnresolvedName(attrname, value.full_name(), self) 640 641 # Accounting. 642 643 self.use_name(attrname) 644 645 return attr 646 647 def visitGlobal(self, node): 648 if self.namespaces: 649 for name in node.names: 650 ns = self.namespaces[-1] 651 if not ns.make_global(name): 652 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 653 654 # The name is recorded in an earlier process. 655 656 def visitIf(self, node): 657 for test, body in node.tests: 658 self.dispatch(test) 659 self.dispatch(body) 660 if node.else_ is not None: 661 self.dispatch(node.else_) 662 return None 663 664 visitIfExp = NOP 665 666 def visitImport(self, node): 667 for name, alias in node.names: 668 if alias is not None: 669 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 670 self.store(alias, module) 671 else: 672 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 673 self.store(name.split(".")[0], module) 674 675 return None 676 677 visitInvert = _visitUnary 678 679 def visitKeyword(self, node): 680 self.dispatch(node.expr) 681 self.importer.make_constant(node.name) 682 self.keyword_names.add(node.name) 683 return None 684 685 def visitLambda(self, node): 686 self.use_name(None) # lambda functions have no names but are assumed to be invoked 687 return self._visitFunction(node, None) 688 689 visitLeftShift = _visitBinary 690 691 visitList = OP 692 693 visitListComp = OP 694 695 visitListCompFor = NOP 696 697 visitListCompIf = NOP 698 699 visitMod = _visitBinary 700 701 def visitModule(self, node): 702 703 # Make a back reference from the node for code generation. 704 705 node.unit = self 706 return self.dispatch(node.node) 707 708 visitMul = _visitBinary 709 710 def visitName(self, node): 711 name = node.name 712 713 if self.importer.predefined_constants.has_key(name): 714 attr = self.importer.get_predefined_constant(name) 715 elif self.namespaces and self.namespaces[-1].has_key(name): 716 attr = self.namespaces[-1][name] 717 elif self.has_key(name): 718 attr = self[name] 719 elif self.builtins is not None and self.builtins.has_key(name): 720 attr = self.builtins[name] 721 else: 722 attr = None 723 724 self.use_name(name) 725 return attr 726 727 visitNot = OP 728 729 visitOr = OP 730 731 visitPass = NOP 732 733 visitPower = _visitBinary 734 735 visitPrint = NOP 736 737 visitPrintnl = NOP 738 739 visitRaise = NOP 740 741 visitReturn = NOP 742 743 visitRightShift = _visitBinary 744 745 visitSlice = OP 746 747 visitSliceobj = OP 748 749 def visitStmt(self, node): 750 for n in node.nodes: 751 self.dispatch(n) 752 return None 753 754 visitSub = _visitBinary 755 756 def visitSubscript(self, node): 757 self.use_name("__getitem__") 758 self.OP(node) 759 760 def visitTryExcept(self, node): 761 self.dispatch(node.body) 762 for name, var, n in node.handlers: 763 764 # Establish the local for the handler. 765 766 if var is not None: 767 self.dispatch(var) 768 if n is not None: 769 self.dispatch(n) 770 if node.else_ is not None: 771 self.dispatch(node.else_) 772 return None 773 774 visitTryFinally = NOP 775 776 visitTuple = OP 777 778 visitUnaryAdd = _visitUnary 779 780 visitUnarySub = _visitUnary 781 782 def visitWhile(self, node): 783 self.in_loop = 1 784 self.NOP(node) 785 self.in_loop = 0 786 return None 787 788 visitWith = NOP 789 790 visitYield = NOP 791 792 # vim: tabstop=4 expandtab shiftwidth=4