1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 118 def parse(self, filename): 119 120 "Parse the file having the given 'filename'." 121 122 module = compiler.parseFile(filename) 123 self.process(module) 124 125 def process(self, module): 126 127 "Process the given 'module'." 128 129 self.astnode = self.module = module 130 processed = self.dispatch(module) 131 if self.has_key("__all__"): 132 all = self["__all__"] 133 if isinstance(all, compiler.ast.List): 134 for n in all.nodes: 135 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 136 return processed 137 138 def vacuum(self): 139 140 """ 141 Vacuum the module namespace, removing unreferenced objects and unused 142 names. 143 """ 144 145 for name, value in self.items(): 146 147 if self.should_optimise_unused_objects(): 148 149 # Remove entries for unreferenced objects. 150 # This, due to the nature of the referenced attribute, assumes 151 # that only explicitly mentioned classes and functions are 152 # employed in the final program. 153 154 if isinstance(value, Attr): 155 156 # Only remove entries for classes and functions, not methods. 157 158 for attr_value in value.get_values(): 159 if (isinstance(attr_value, Function) and not attr_value.is_method() or 160 isinstance(attr_value, Class)) and not attr_value.referenced: 161 pass 162 else: 163 break 164 else: 165 del self[name] 166 167 # Complain about globals not initialised at the module level. 168 169 if isinstance(value, Global): 170 print "Warning: global %r in module %r not initialised at the module level." % (name, self.name) 171 172 # Remove unreferenced objects. 173 174 if self.should_optimise_unused_objects(): 175 176 all_objects = list(self.all_objects) 177 178 for obj in all_objects: 179 180 # Only remove entries for classes and functions, not methods. 181 182 if (isinstance(obj, Function) and not obj.is_method() or 183 isinstance(obj, Class)) and not obj.referenced: 184 185 self.all_objects.remove(obj) 186 187 # Remove unused entries from classes plus associated methods. 188 189 if isinstance(obj, Class): 190 for name, attr in obj.class_attributes().items(): 191 192 # Methods can only be deleted if they are the only 193 # assigned object to the class and are unreferenced. 194 195 if name not in self.importer.names_used and \ 196 attr.assignments == 1 and isinstance(attr.get_value(), Function) and \ 197 attr.get_value().is_method() and not attr.get_value().referenced: 198 199 self.all_objects.remove(attr.get_value()) 200 del obj[name] 201 202 def finalise(self): 203 204 "Finalise the module." 205 206 for obj in self.all_objects: 207 if isinstance(obj, (Class, Function)): 208 obj.finalise_attributes() 209 210 def add_object(self, obj, any_scope=0): 211 212 """ 213 Record 'obj' if non-local or if the optional 'any_scope' is set to a 214 true value. 215 """ 216 217 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 218 self.all_objects.add(obj) 219 220 # Optimisation tests. 221 222 def should_optimise_unused_objects(self): 223 return "unused_objects" in self.optimisations 224 225 # Namespace methods. 226 227 def store(self, name, obj): 228 229 "Record attribute or local 'name', storing 'obj'." 230 231 if not self.namespaces: 232 self.set(name, obj, not self.in_loop) 233 else: 234 self.namespaces[-1].set(name, obj, not self.in_loop) 235 236 def store_lambda(self, obj): 237 238 "Store a lambda function 'obj'." 239 240 self.add_object(obj) 241 242 def store_module_attr(self, name, module): 243 244 """ 245 Record module attribute 'name' in the given 'module' using the current 246 expression. 247 """ 248 249 module.set(name, self.expr, 0) 250 251 def store_class_attr(self, name): 252 253 """ 254 Record class attribute 'name' in the current class using the current 255 expression. 256 """ 257 258 if self.in_method and self.namespaces[-2].has_key(name): 259 self.namespaces[-2].set(name, self.expr, 0) 260 return 1 261 262 return 0 263 264 def store_instance_attr(self, name): 265 266 "Record instance attribute 'name' in the current class." 267 268 if self.in_method: 269 270 # Current namespace is the function. 271 # Previous namespace is the class. 272 273 self.namespaces[-2].add_instance_attribute(name) 274 275 def get_parent(self): 276 277 "Return the parent (or most recent) namespace currently exposed." 278 279 return (self.namespaces[-1:] or [self])[0] 280 281 # Visitor methods. 282 283 def default(self, node, *args): 284 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 285 286 def dispatch(self, node, *args): 287 return ASTVisitor.dispatch(self, node, *args) 288 289 def NOP(self, node): 290 for n in node.getChildNodes(): 291 self.dispatch(n) 292 return None 293 294 def OP(self, node): 295 for n in node.getChildNodes(): 296 self.dispatch(n) 297 return Instance() 298 299 def _visitUnary(self, node): 300 301 "Accounting method for the unary operator 'node'." 302 303 method = unary_methods[node.__class__.__name__] 304 self.importer.use_name(method) 305 return self.OP(node) 306 307 def _visitBinary(self, node): 308 309 "Accounting method for the binary operator 'node'." 310 311 left_method, right_method = binary_methods[node.__class__.__name__] 312 self.importer.use_name(left_method) 313 self.importer.use_name(right_method) 314 return self.OP(node) 315 316 def _visitFunction(self, node, name): 317 318 """ 319 Return a function object for the function defined by 'node' with the 320 given 'name'. If a lambda expression is being visited, 'name' should be 321 None. 322 """ 323 324 function = Function( 325 name, 326 self.get_parent(), 327 node.argnames, 328 node.defaults, 329 (node.flags & 4 != 0), 330 (node.flags & 8 != 0), 331 self, 332 node 333 ) 334 335 # Make a back reference from the node for code generation. 336 337 node.unit = function 338 339 # Process the defaults. 340 341 for n in node.defaults: 342 self.expr = self.dispatch(n) 343 function.store_default(self.expr) 344 345 # Enter the function. 346 347 self.namespaces.append(function) 348 349 # Current namespace is the function. 350 # Previous namespace is the class. 351 352 if len(self.namespaces) > 1 and isinstance(self.namespaces[-2], Class): 353 if name == "__init__": 354 self.in_init = 1 355 self.in_method = 1 356 357 self.dispatch(node.code) 358 self.in_init = 0 359 self.in_method = 0 360 self.namespaces.pop() 361 362 if name is not None: 363 self.store(name, function) 364 else: 365 self.store_lambda(function) 366 367 # Lambda functions are always assumed to be referenced. This is 368 # because other means of discovering the referencing of objects rely 369 # on the original names inherent in the definition of those objects. 370 371 function.set_referenced() 372 373 # Where defaults exist, an instance needs creating. Thus, it makes 374 # no sense to return a reference to the function here, since the 375 # recipient will not be referencing the function itself. 376 377 if node.defaults: 378 return Instance() # indicates no known target 379 380 self.add_object(function, any_scope=1) 381 return function 382 383 # Specific handler methods. 384 385 visitAdd = _visitBinary 386 387 visitAnd = OP 388 389 visitAssert = NOP 390 391 def visitAssign(self, node): 392 self.expr = self.dispatch(node.expr) 393 for n in node.nodes: 394 self.dispatch(n) 395 return None 396 397 def visitAssAttr(self, node): 398 expr = self.dispatch(node.expr) 399 if isinstance(expr, Attr): 400 if expr.name == "self": 401 if not self.store_class_attr(node.attrname): 402 self.store_instance_attr(node.attrname) 403 elif isinstance(expr.get_value(), Module): 404 self.store_module_attr(node.attrname, expr.get_value()) 405 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.get_value().name) 406 return None 407 408 def visitAssList(self, node): 409 410 # Declare names which will be used by generated code. 411 412 self.importer.use_name("__getitem__") 413 414 # Process the assignment. 415 416 for i, n in enumerate(node.nodes): 417 self.dispatch(n) 418 self.importer.make_constant(i) # for __getitem__(i) at run-time 419 return None 420 421 def visitAssName(self, node): 422 self.store(node.name, self.expr) 423 return None 424 425 visitAssTuple = visitAssList 426 427 def visitAugAssign(self, node): 428 429 # Accounting. 430 431 aug_method, (left_method, right_method) = augassign_methods[node.op] 432 self.importer.use_name(aug_method) 433 self.importer.use_name(left_method) 434 self.importer.use_name(right_method) 435 return self.OP(node) 436 437 visitBackquote = OP 438 439 visitBitand = _visitBinary 440 441 visitBitor = _visitBinary 442 443 visitBitxor = _visitBinary 444 445 visitBreak = NOP 446 447 visitCallFunc = OP 448 449 def visitClass(self, node): 450 451 """ 452 Register the class at the given 'node' subject to the restrictions 453 mentioned in the module docstring. 454 """ 455 456 if self.namespaces: 457 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 458 return None 459 else: 460 cls = Class(node.name, self.get_parent(), self, node) 461 462 # Visit the base class expressions, attempting to find concrete 463 # definitions of classes. 464 465 for base in node.bases: 466 expr = self.dispatch(base) 467 if isinstance(expr, Attr): 468 if expr.assignments != 1: 469 raise InspectError(self.full_name(), node, 470 "Base class %r for %r is not constant." % (base, cls.full_name())) 471 else: 472 cls.add_base(expr.get_value()) 473 else: # if expr is None: 474 raise InspectError(self.full_name(), node, 475 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 476 477 # NOTE: Potentially dubious measure to permit __init__ availability. 478 # If no bases exist, adopt the 'object' class. 479 480 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 481 expr = self.dispatch(compiler.ast.Name("object")) 482 cls.add_base(expr.get_value()) 483 484 # Make a back reference from the node for code generation. 485 486 node.unit = cls 487 488 # Make an entry for the class. 489 490 self.store(node.name, cls) 491 self.add_object(cls) 492 493 # Process the class body. 494 495 self.namespaces.append(cls) 496 self.dispatch(node.code) 497 self.namespaces.pop() 498 499 return cls 500 501 def visitCompare(self, node): 502 503 # Accounting. 504 # NOTE: Replicates some code in micropython.ast.visitCompare. 505 506 for op in node.ops: 507 op_name, next_node = op 508 methods = comparison_methods[op_name] 509 if methods is not None: 510 self.importer.use_name(methods[0]) 511 self.importer.use_name(methods[1]) 512 elif op_name.endswith("in"): 513 self.importer.use_name("__contains__") 514 515 return self.OP(node) 516 517 def visitConst(self, node): 518 519 # Accounting. 520 521 name = node.value.__class__.__name__ 522 if self.builtins is not None and self.builtins.has_key(name): 523 attr = self.builtins[name] 524 attr.set_referenced() 525 526 # Register the constant, if necessary, returning the resulting object. 527 528 return self.importer.make_constant(node.value) 529 530 visitContinue = NOP 531 532 visitDecorators = NOP 533 534 visitDict = OP 535 536 visitDiscard = NOP 537 538 visitDiv = _visitBinary 539 540 visitEllipsis = NOP 541 542 visitExec = NOP 543 544 visitExpression = OP 545 546 visitFloorDiv = _visitBinary 547 548 def visitFor(self, node): 549 550 # Declare names which will be used by generated code. 551 552 self.importer.use_name("__iter__") 553 self.importer.use_name("next") 554 555 # Enter the loop. 556 557 self.in_loop = 1 558 self.NOP(node) 559 self.in_loop = 0 560 return None 561 562 def visitFrom(self, node): 563 module = self.importer.load(node.modname, 1) 564 565 if module is not None: 566 module.set_referenced() 567 568 #if module is None: 569 # print "Warning:", node.modname, "not imported." 570 571 for name, alias in node.names: 572 if name != "*": 573 if module is not None and module.namespace.has_key(name): 574 attr = module[name] 575 self.store(alias or name, attr) 576 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 577 self.importer.load(attr.get_value().name) 578 579 # Support the import of names from missing modules. 580 581 else: 582 self.store(alias or name, UnresolvedName(name, node.modname, self)) 583 else: 584 if module is not None: 585 for n in module.namespace.keys(): 586 attr = module[n] 587 self.store(n, attr) 588 if isinstance(attr.get_value(), Module) and not attr.get_value().loaded: 589 self.importer.load(attr.get_value().name) 590 591 return None 592 593 def visitFunction(self, node): 594 return self._visitFunction(node, node.name) 595 596 visitGenExpr = OP 597 598 visitGenExprFor = NOP 599 600 visitGenExprIf = NOP 601 602 visitGenExprInner = NOP 603 604 def visitGetattr(self, node): 605 expr = self.dispatch(node.expr) 606 attrname = node.attrname 607 608 if isinstance(expr, Attr): 609 value = expr.get_value() 610 if isinstance(value, (Class, Module)): 611 attr = value.namespace.get(attrname) 612 elif isinstance(value, UnresolvedName): 613 attr = UnresolvedName(attrname, value.full_name(), self) 614 else: 615 attr = None 616 elif self.builtins is not None: 617 attr = self.builtins.get(attrname) 618 else: 619 attr = UnresolvedName(attrname, value.full_name(), self) 620 621 # Accounting. 622 623 if attr is not None: 624 attr.set_referenced() 625 626 self.importer.use_name(attrname) 627 628 return attr 629 630 def visitGlobal(self, node): 631 if self.namespaces: 632 for name in node.names: 633 ns = self.namespaces[-1] 634 if not ns.make_global(name): 635 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 636 637 # Record a global entry for the name in the module. 638 639 if not self.has_key(name): 640 self[name] = Global() 641 642 def visitIf(self, node): 643 for test, body in node.tests: 644 self.dispatch(test) 645 self.dispatch(body) 646 if node.else_ is not None: 647 self.dispatch(node.else_) 648 return None 649 650 visitIfExp = NOP 651 652 def visitImport(self, node): 653 for name, alias in node.names: 654 if alias is not None: 655 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 656 self.store(alias, module) 657 else: 658 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 659 self.store(name.split(".")[0], module) 660 module.set_referenced() 661 662 return None 663 664 visitInvert = _visitUnary 665 666 def visitKeyword(self, node): 667 self.dispatch(node.expr) 668 self.importer.make_constant(node.name) 669 self.keyword_names.add(node.name) 670 return None 671 672 def visitLambda(self, node): 673 return self._visitFunction(node, None) 674 675 visitLeftShift = _visitBinary 676 677 visitList = OP 678 679 visitListComp = OP 680 681 visitListCompFor = NOP 682 683 visitListCompIf = NOP 684 685 visitMod = _visitBinary 686 687 def visitModule(self, node): 688 689 # Make a back reference from the node for code generation. 690 691 node.unit = self 692 return self.dispatch(node.node) 693 694 visitMul = _visitBinary 695 696 def visitName(self, node): 697 name = node.name 698 699 if self.importer.predefined_constants.has_key(name): 700 attr = self.importer.get_predefined_constant(name) 701 elif self.namespaces and self.namespaces[-1].has_key(name): 702 attr = self.namespaces[-1][name] 703 elif self.has_key(name): 704 attr = self[name] 705 elif self.builtins is not None and self.builtins.has_key(name): 706 attr = self.builtins[name] 707 else: 708 attr = None 709 710 # Accounting. 711 712 if attr is not None: 713 attr.set_referenced() 714 715 self.importer.use_name(name) 716 717 return attr 718 719 visitNot = OP 720 721 visitOr = OP 722 723 visitPass = NOP 724 725 visitPower = _visitBinary 726 727 visitPrint = NOP 728 729 visitPrintnl = NOP 730 731 visitRaise = NOP 732 733 visitReturn = NOP 734 735 visitRightShift = _visitBinary 736 737 visitSlice = OP 738 739 visitSliceobj = OP 740 741 def visitStmt(self, node): 742 for n in node.nodes: 743 self.dispatch(n) 744 return None 745 746 visitSub = _visitBinary 747 748 visitSubscript = OP 749 750 def visitTryExcept(self, node): 751 self.dispatch(node.body) 752 for name, var, n in node.handlers: 753 754 # Establish the local for the handler. 755 756 if var is not None: 757 self.dispatch(var) 758 if n is not None: 759 self.dispatch(n) 760 if node.else_ is not None: 761 self.dispatch(node.else_) 762 return None 763 764 visitTryFinally = NOP 765 766 visitTuple = OP 767 768 visitUnaryAdd = _visitUnary 769 770 visitUnarySub = _visitUnary 771 772 def visitWhile(self, node): 773 self.in_loop = 1 774 self.NOP(node) 775 self.in_loop = 0 776 return None 777 778 visitWith = NOP 779 780 visitYield = NOP 781 782 class Global: 783 784 """ 785 A reference to an object assigned to a global from outside the module 786 top-level. 787 """ 788 789 pass 790 791 # vim: tabstop=4 expandtab shiftwidth=4