1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 118 def parse(self, filename): 119 120 "Parse the file having the given 'filename'." 121 122 module = compiler.parseFile(filename) 123 self.process(module) 124 125 def process(self, module): 126 127 "Process the given 'module'." 128 129 self.astnode = self.module = module 130 processed = self.dispatch(module) 131 if self.has_key("__all__"): 132 all = self["__all__"] 133 if isinstance(all, compiler.ast.List): 134 for n in all.nodes: 135 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 136 return processed 137 138 def vacuum(self): 139 140 """ 141 Vacuum the module namespace, removing unreferenced objects and unused 142 names. 143 """ 144 145 for name, value in self.items(): 146 147 if self.should_optimise_unused_objects(): 148 149 # Remove entries for unreferenced objects. 150 # This, due to the nature of the referenced attribute, assumes 151 # that only explicitly mentioned classes and functions are 152 # employed in the final program. 153 154 if isinstance(value, Attr): 155 attr_value = value.value 156 157 # Only remove entries for classes and functions, not methods. 158 159 if attr_value is not None: 160 for attr_value in value.assignment_values: 161 if (isinstance(attr_value, Function) and not attr_value.is_method() or 162 isinstance(attr_value, Class)) and not attr_value.referenced: 163 pass 164 else: 165 break 166 else: 167 del self[name] 168 169 # Complain about globals not initialised at the module level. 170 171 if isinstance(value, Global): 172 print "Warning: global %r in module %r not initialised at the module level." % (name, self.name) 173 174 # Remove unreferenced objects. 175 176 if self.should_optimise_unused_objects(): 177 178 all_objects = list(self.all_objects) 179 180 for obj in all_objects: 181 182 # Only remove entries for classes and functions, not methods. 183 184 if (isinstance(obj, Function) and not obj.is_method() or 185 isinstance(obj, Class)) and not obj.referenced: 186 187 self.all_objects.remove(obj) 188 189 # Remove unused entries from classes plus associated methods. 190 191 if isinstance(obj, Class): 192 for name, attr in obj.class_attributes().items(): 193 194 # Methods can only be deleted if they are the only 195 # assigned object to the class and are unreferenced. 196 197 if name not in self.importer.names_used and \ 198 attr.assignments == 1 and isinstance(attr.value, Function) and \ 199 attr.value.is_method() and not attr.value.referenced: 200 201 self.all_objects.remove(attr.value) 202 del obj[name] 203 204 def finalise(self): 205 206 "Finalise the module." 207 208 for obj in self.all_objects: 209 if isinstance(obj, (Class, Function)): 210 obj.finalise_attributes() 211 212 def add_object(self, obj, any_scope=0): 213 214 """ 215 Record 'obj' if non-local or if the optional 'any_scope' is set to a 216 true value. 217 """ 218 219 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 220 self.all_objects.add(obj) 221 222 # Optimisation tests. 223 224 def should_optimise_unused_objects(self): 225 return "unused_objects" in self.optimisations 226 227 # Namespace methods. 228 229 def store(self, name, obj): 230 231 "Record attribute or local 'name', storing 'obj'." 232 233 if not self.namespaces: 234 self.set(name, obj, not self.in_loop) 235 else: 236 self.namespaces[-1].set(name, obj, not self.in_loop) 237 238 def store_lambda(self, obj): 239 240 "Store a lambda function 'obj'." 241 242 self.add_object(obj) 243 244 def store_module_attr(self, name, module): 245 246 """ 247 Record module attribute 'name' in the given 'module' using the current 248 expression. 249 """ 250 251 if isinstance(self.expr, Attr): 252 assigned_value = self.expr.value 253 else: 254 assigned_value = self.expr 255 256 module.set(name, assigned_value, 0) 257 258 def store_class_attr(self, name): 259 260 """ 261 Record class attribute 'name' in the current class using the current 262 expression. 263 """ 264 265 if self.in_method and self.namespaces[-2].has_key(name): 266 267 if isinstance(self.expr, Attr): 268 assigned_value = self.expr.value 269 else: 270 assigned_value = self.expr 271 272 self.namespaces[-2].set(name, assigned_value, 0) 273 return 1 274 275 return 0 276 277 def store_instance_attr(self, name): 278 279 "Record instance attribute 'name' in the current class." 280 281 if self.in_method: 282 283 # Current namespace is the function. 284 # Previous namespace is the class. 285 286 self.namespaces[-2].add_instance_attribute(name) 287 288 def get_parent(self): 289 290 "Return the parent (or most recent) namespace currently exposed." 291 292 return (self.namespaces[-1:] or [self])[0] 293 294 # Visitor methods. 295 296 def default(self, node, *args): 297 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 298 299 def dispatch(self, node, *args): 300 return ASTVisitor.dispatch(self, node, *args) 301 302 def NOP(self, node): 303 for n in node.getChildNodes(): 304 self.dispatch(n) 305 return None 306 307 def OP(self, node): 308 for n in node.getChildNodes(): 309 self.dispatch(n) 310 return Instance() 311 312 def _visitUnary(self, node): 313 314 "Accounting method for the unary operator 'node'." 315 316 method = unary_methods[node.__class__.__name__] 317 self.importer.use_name(method) 318 return self.OP(node) 319 320 def _visitBinary(self, node): 321 322 "Accounting method for the binary operator 'node'." 323 324 left_method, right_method = binary_methods[node.__class__.__name__] 325 self.importer.use_name(left_method) 326 self.importer.use_name(right_method) 327 return self.OP(node) 328 329 def _visitFunction(self, node, name): 330 331 """ 332 Return a function object for the function defined by 'node' with the 333 given 'name'. If a lambda expression is being visited, 'name' should be 334 None. 335 """ 336 337 function = Function( 338 name, 339 self.get_parent(), 340 node.argnames, 341 node.defaults, 342 (node.flags & 4 != 0), 343 (node.flags & 8 != 0), 344 self, 345 node 346 ) 347 348 # Make a back reference from the node for code generation. 349 350 node.unit = function 351 352 # Process the defaults. 353 354 for n in node.defaults: 355 self.expr = self.dispatch(n) 356 if isinstance(self.expr, Attr): 357 function.store_default(self.expr.value) 358 else: 359 function.store_default(self.expr) 360 361 # Enter the function. 362 363 self.namespaces.append(function) 364 365 # Current namespace is the function. 366 # Previous namespace is the class. 367 368 if len(self.namespaces) > 1 and isinstance(self.namespaces[-2], Class): 369 if name == "__init__": 370 self.in_init = 1 371 self.in_method = 1 372 373 self.dispatch(node.code) 374 self.in_init = 0 375 self.in_method = 0 376 self.namespaces.pop() 377 378 if name is not None: 379 self.store(name, function) 380 else: 381 self.store_lambda(function) 382 383 # Lambda functions are always assumed to be referenced. This is 384 # because other means of discovering the referencing of objects rely 385 # on the original names inherent in the definition of those objects. 386 387 function.set_referenced() 388 389 # Where defaults exist, an instance needs creating. Thus, it makes 390 # no sense to return a reference to the function here, since the 391 # recipient will not be referencing the function itself. 392 393 if node.defaults: 394 return Instance() # indicates no known target 395 396 self.add_object(function, any_scope=1) 397 return function 398 399 # Specific handler methods. 400 401 visitAdd = _visitBinary 402 403 visitAnd = OP 404 405 visitAssert = NOP 406 407 def visitAssign(self, node): 408 self.expr = self.dispatch(node.expr) 409 for n in node.nodes: 410 self.dispatch(n) 411 return None 412 413 def visitAssAttr(self, node): 414 expr = self.dispatch(node.expr) 415 if isinstance(expr, Attr): 416 if expr.name == "self": 417 if not self.store_class_attr(node.attrname): 418 self.store_instance_attr(node.attrname) 419 elif isinstance(expr.value, Module): 420 self.store_module_attr(node.attrname, expr.value) 421 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.value.name) 422 return None 423 424 def visitAssList(self, node): 425 for i, n in enumerate(node.nodes): 426 self.dispatch(n) 427 self.importer.make_constant(i) # for __getitem__(i) at run-time 428 return None 429 430 def visitAssName(self, node): 431 self.store(node.name, self.expr) 432 return None 433 434 visitAssTuple = visitAssList 435 436 def visitAugAssign(self, node): 437 438 # Accounting. 439 440 aug_method, (left_method, right_method) = augassign_methods[node.op] 441 self.importer.use_name(aug_method) 442 self.importer.use_name(left_method) 443 self.importer.use_name(right_method) 444 return self.OP(node) 445 446 visitBackquote = OP 447 448 visitBitand = _visitBinary 449 450 visitBitor = _visitBinary 451 452 visitBitxor = _visitBinary 453 454 visitBreak = NOP 455 456 visitCallFunc = OP 457 458 def visitClass(self, node): 459 460 """ 461 Register the class at the given 'node' subject to the restrictions 462 mentioned in the module docstring. 463 """ 464 465 if self.namespaces: 466 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 467 return None 468 else: 469 cls = Class(node.name, self.get_parent(), self, node) 470 471 # Visit the base class expressions, attempting to find concrete 472 # definitions of classes. 473 474 for base in node.bases: 475 expr = self.dispatch(base) 476 if isinstance(expr, Attr): 477 if expr.assignments != 1: 478 raise InspectError(self.full_name(), node, 479 "Base class %r for %r is not constant." % (base, cls.full_name())) 480 else: 481 cls.add_base(expr.value) 482 else: # if expr is None: 483 raise InspectError(self.full_name(), node, 484 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 485 486 # NOTE: Potentially dubious measure to permit __init__ availability. 487 # If no bases exist, adopt the 'object' class. 488 489 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 490 expr = self.dispatch(compiler.ast.Name("object")) 491 cls.add_base(expr.value) 492 493 # Make a back reference from the node for code generation. 494 495 node.unit = cls 496 497 # Make an entry for the class. 498 499 self.store(node.name, cls) 500 self.add_object(cls) 501 502 # Process the class body. 503 504 self.namespaces.append(cls) 505 self.dispatch(node.code) 506 self.namespaces.pop() 507 508 return cls 509 510 def visitCompare(self, node): 511 512 # Accounting. 513 # NOTE: Replicates some code in micropython.ast.visitCompare. 514 515 for op in node.ops: 516 op_name, next_node = op 517 methods = comparison_methods[op_name] 518 if methods is not None: 519 self.importer.use_name(methods[0]) 520 self.importer.use_name(methods[1]) 521 elif op_name.endswith("in"): 522 self.importer.use_name("__contains__") 523 524 return self.OP(node) 525 526 def visitConst(self, node): 527 528 # Accounting. 529 530 name = node.value.__class__.__name__ 531 if self.builtins is not None and self.builtins.has_key(name): 532 attr = self.builtins[name] 533 attr.set_referenced() 534 535 # Register the constant, if necessary, returning the resulting object. 536 537 return self.importer.make_constant(node.value) 538 539 visitContinue = NOP 540 541 visitDecorators = NOP 542 543 visitDict = OP 544 545 visitDiscard = NOP 546 547 visitDiv = _visitBinary 548 549 visitEllipsis = NOP 550 551 visitExec = NOP 552 553 visitExpression = OP 554 555 visitFloorDiv = _visitBinary 556 557 def visitFor(self, node): 558 self.in_loop = 1 559 self.NOP(node) 560 self.in_loop = 0 561 return None 562 563 def visitFrom(self, node): 564 module = self.importer.load(node.modname, 1) 565 566 if module is not None: 567 module.set_referenced() 568 569 #if module is None: 570 # print "Warning:", node.modname, "not imported." 571 572 for name, alias in node.names: 573 if name != "*": 574 if module is not None and module.namespace.has_key(name): 575 attr = module[name] 576 self.store(alias or name, attr.value) 577 if isinstance(attr.value, Module) and not attr.value.loaded: 578 self.importer.load(attr.value.name) 579 580 # Support the import of names from missing modules. 581 582 else: 583 self.store(alias or name, UnresolvedName(name, node.modname, self)) 584 else: 585 if module is not None: 586 for n in module.namespace.keys(): 587 attr = module[n] 588 self.store(n, attr.value) 589 if isinstance(attr.value, Module) and not attr.value.loaded: 590 self.importer.load(attr.value.name) 591 592 return None 593 594 def visitFunction(self, node): 595 return self._visitFunction(node, node.name) 596 597 visitGenExpr = OP 598 599 visitGenExprFor = NOP 600 601 visitGenExprIf = NOP 602 603 visitGenExprInner = NOP 604 605 def visitGetattr(self, node): 606 expr = self.dispatch(node.expr) 607 attrname = node.attrname 608 609 if isinstance(expr, Attr): 610 value = expr.value 611 if isinstance(value, (Class, Module)): 612 attr = value.namespace.get(attrname) 613 elif isinstance(value, UnresolvedName): 614 attr = UnresolvedName(attrname, value.full_name(), self) 615 else: 616 attr = None 617 elif self.builtins is not None: 618 attr = self.builtins.get(attrname) 619 else: 620 attr = UnresolvedName(attrname, value.full_name(), self) 621 622 # Accounting. 623 624 if attr is not None: 625 attr.set_referenced() 626 627 self.importer.use_name(attrname) 628 629 return attr 630 631 def visitGlobal(self, node): 632 if self.namespaces: 633 for name in node.names: 634 ns = self.namespaces[-1] 635 if not ns.make_global(name): 636 raise InspectError(ns.full_name(), node, "Name %r is global and local in %r" % (name, ns.full_name())) 637 638 # Record a global entry for the name in the module. 639 640 if not self.has_key(name): 641 self[name] = Global() 642 643 def visitIf(self, node): 644 for test, body in node.tests: 645 self.dispatch(body) 646 if node.else_ is not None: 647 self.dispatch(node.else_) 648 return None 649 650 visitIfExp = NOP 651 652 def visitImport(self, node): 653 for name, alias in node.names: 654 if alias is not None: 655 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 656 self.store(alias, module) 657 else: 658 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 659 self.store(name.split(".")[0], module) 660 module.set_referenced() 661 662 return None 663 664 visitInvert = _visitUnary 665 666 def visitKeyword(self, node): 667 self.dispatch(node.expr) 668 self.importer.make_constant(node.name) 669 self.keyword_names.add(node.name) 670 return None 671 672 def visitLambda(self, node): 673 return self._visitFunction(node, None) 674 675 visitLeftShift = _visitBinary 676 677 visitList = OP 678 679 visitListComp = OP 680 681 visitListCompFor = NOP 682 683 visitListCompIf = NOP 684 685 visitMod = _visitBinary 686 687 def visitModule(self, node): 688 689 # Make a back reference from the node for code generation. 690 691 node.unit = self 692 return self.dispatch(node.node) 693 694 visitMul = _visitBinary 695 696 def visitName(self, node): 697 name = node.name 698 699 if self.importer.predefined_constants.has_key(name): 700 attr = self.importer.get_predefined_constant(name) 701 elif self.namespaces and self.namespaces[-1].has_key(name): 702 attr = self.namespaces[-1][name] 703 elif self.has_key(name): 704 attr = self[name] 705 elif self.builtins is not None and self.builtins.has_key(name): 706 attr = self.builtins[name] 707 else: 708 attr = None 709 710 # Accounting. 711 712 if attr is not None: 713 attr.set_referenced() 714 715 self.importer.use_name(name) 716 717 return attr 718 719 visitNot = OP 720 721 visitOr = OP 722 723 visitPass = NOP 724 725 visitPower = _visitBinary 726 727 visitPrint = NOP 728 729 visitPrintnl = NOP 730 731 visitRaise = NOP 732 733 visitReturn = NOP 734 735 visitRightShift = _visitBinary 736 737 visitSlice = OP 738 739 visitSliceobj = OP 740 741 def visitStmt(self, node): 742 for n in node.nodes: 743 self.dispatch(n) 744 return None 745 746 visitSub = _visitBinary 747 748 visitSubscript = OP 749 750 def visitTryExcept(self, node): 751 self.dispatch(node.body) 752 for name, var, n in node.handlers: 753 754 # Establish the local for the handler. 755 756 if var is not None: 757 self.dispatch(var) 758 if n is not None: 759 self.dispatch(n) 760 if node.else_ is not None: 761 self.dispatch(node.else_) 762 return None 763 764 visitTryFinally = NOP 765 766 visitTuple = OP 767 768 visitUnaryAdd = _visitUnary 769 770 visitUnarySub = _visitUnary 771 772 def visitWhile(self, node): 773 self.in_loop = 1 774 self.NOP(node) 775 self.in_loop = 0 776 return None 777 778 visitWith = NOP 779 780 visitYield = NOP 781 782 class Global: 783 784 """ 785 A reference to an object assigned to a global from outside the module 786 top-level. 787 """ 788 789 pass 790 791 # vim: tabstop=4 expandtab shiftwidth=4