1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 118 def parse(self, filename): 119 120 "Parse the file having the given 'filename'." 121 122 module = compiler.parseFile(filename) 123 self.process(module) 124 125 def process(self, module): 126 127 "Process the given 'module'." 128 129 self.astnode = self.module = module 130 processed = self.dispatch(module) 131 if self.has_key("__all__"): 132 all = self["__all__"] 133 if isinstance(all, compiler.ast.List): 134 for n in all.nodes: 135 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 136 return processed 137 138 def vacuum(self): 139 140 """ 141 Vacuum the module namespace, removing unreferenced objects and unused 142 names. 143 """ 144 145 for name, value in self.items(): 146 147 if self.should_optimise_unused_objects(): 148 149 # Remove unused names. 150 151 if name not in self.importer.names_used: 152 del self[name] 153 154 # Remove entries for unreferenced objects. 155 # This, due to the nature of the referenced attribute, assumes 156 # that only explicitly mentioned classes and functions are 157 # employed in the final program. 158 159 elif isinstance(value, Attr): 160 attr_value = value.value 161 162 # Only remove entries for classes and functions, not methods. 163 164 for attr_value in value.assignment_values: 165 if (isinstance(attr_value, Function) and not attr_value.is_method() or 166 isinstance(attr_value, Class)) and not attr_value.referenced: 167 pass 168 else: 169 break 170 else: 171 del self[name] 172 173 # Complain about globals not initialised at the module level. 174 175 if isinstance(value, Global): 176 print "Warning: global %r in module %r not initialised at the module level." % (name, self.name) 177 178 # Remove unreferenced objects. 179 180 if self.should_optimise_unused_objects(): 181 182 all_objects = list(self.all_objects) 183 184 for obj in all_objects: 185 186 # Only remove entries for classes and functions, not methods. 187 188 if (isinstance(obj, Function) and not obj.is_method() or 189 isinstance(obj, Class)) and not obj.referenced: 190 191 self.all_objects.remove(obj) 192 193 # Remove unused entries from classes plus associated methods. 194 195 if isinstance(obj, Class): 196 for name, attr in obj.class_attributes().items(): 197 198 if name not in self.importer.names_used: 199 del obj[name] 200 201 # Methods can only be deleted if they are the only 202 # assigned object to the class and are unreferenced. 203 204 if attr.assignments == 1 and isinstance(attr.value, Function) and \ 205 attr.value.is_method() and not attr.value.referenced: 206 207 self.all_objects.remove(attr.value) 208 209 def finalise(self): 210 211 "Finalise the module." 212 213 for obj in self.all_objects: 214 if isinstance(obj, (Class, Function)): 215 obj.finalise_attributes() 216 217 def add_object(self, obj, any_scope=0): 218 219 """ 220 Record 'obj' if non-local or if the optional 'any_scope' is set to a 221 true value. 222 """ 223 224 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 225 self.all_objects.add(obj) 226 227 # Optimisation tests. 228 229 def should_optimise_unused_objects(self): 230 return "unused_objects" in self.optimisations 231 232 # Namespace methods. 233 234 def store(self, name, obj): 235 236 "Record attribute or local 'name', storing 'obj'." 237 238 if not self.namespaces: 239 self.set(name, obj, not self.in_loop) 240 else: 241 self.namespaces[-1].set(name, obj, not self.in_loop) 242 243 def store_lambda(self, obj): 244 245 "Store a lambda function 'obj'." 246 247 self.add_object(obj) 248 249 def store_module_attr(self, name, module): 250 251 """ 252 Record module attribute 'name' in the given 'module' using the current 253 expression. 254 """ 255 256 if isinstance(self.expr, Attr): 257 assigned_value = self.expr.value 258 else: 259 assigned_value = self.expr 260 261 module.set(name, assigned_value, 0) 262 263 def store_class_attr(self, name): 264 265 """ 266 Record class attribute 'name' in the current class using the current 267 expression. 268 """ 269 270 if self.in_method and self.namespaces[-2].has_key(name): 271 272 if isinstance(self.expr, Attr): 273 assigned_value = self.expr.value 274 else: 275 assigned_value = self.expr 276 277 self.namespaces[-2].set(name, assigned_value, 0) 278 return 1 279 280 return 0 281 282 def store_instance_attr(self, name): 283 284 "Record instance attribute 'name' in the current class." 285 286 if self.in_method: 287 288 # Current namespace is the function. 289 # Previous namespace is the class. 290 291 self.namespaces[-2].add_instance_attribute(name) 292 293 def get_parent(self): 294 295 "Return the parent (or most recent) namespace currently exposed." 296 297 return (self.namespaces[-1:] or [self])[0] 298 299 # Visitor methods. 300 301 def default(self, node, *args): 302 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 303 304 def dispatch(self, node, *args): 305 return ASTVisitor.dispatch(self, node, *args) 306 307 def NOP(self, node): 308 for n in node.getChildNodes(): 309 self.dispatch(n) 310 return None 311 312 def OP(self, node): 313 for n in node.getChildNodes(): 314 self.dispatch(n) 315 return Instance() 316 317 def _visitUnary(self, node): 318 319 "Accounting method for the unary operator 'node'." 320 321 method = unary_methods[node.__class__.__name__] 322 self.importer.use_name(method) 323 return self.OP(node) 324 325 def _visitBinary(self, node): 326 327 "Accounting method for the binary operator 'node'." 328 329 left_method, right_method = binary_methods[node.__class__.__name__] 330 self.importer.use_name(left_method) 331 self.importer.use_name(right_method) 332 return self.OP(node) 333 334 def _visitFunction(self, node, name): 335 336 """ 337 Return a function object for the function defined by 'node' with the 338 given 'name'. If a lambda expression is being visited, 'name' should be 339 None. 340 """ 341 342 function = Function( 343 name, 344 self.get_parent(), 345 node.argnames, 346 node.defaults, 347 (node.flags & 4 != 0), 348 (node.flags & 8 != 0), 349 self, 350 node 351 ) 352 353 # Make a back reference from the node for code generation. 354 355 node.unit = function 356 357 # Process the defaults. 358 359 for n in node.defaults: 360 self.expr = self.dispatch(n) 361 if isinstance(self.expr, Attr): 362 function.store_default(self.expr.value) 363 else: 364 function.store_default(self.expr) 365 366 # Enter the function. 367 368 self.namespaces.append(function) 369 370 # Current namespace is the function. 371 # Previous namespace is the class. 372 373 if len(self.namespaces) > 1 and isinstance(self.namespaces[-2], Class): 374 if name == "__init__": 375 self.in_init = 1 376 self.in_method = 1 377 378 self.dispatch(node.code) 379 self.in_init = 0 380 self.in_method = 0 381 self.namespaces.pop() 382 383 if name is not None: 384 self.store(name, function) 385 else: 386 self.store_lambda(function) 387 388 # Lambda functions are always assumed to be referenced. This is 389 # because other means of discovering the referencing of objects rely 390 # on the original names inherent in the definition of those objects. 391 392 function.set_referenced() 393 394 # Where defaults exist, an instance needs creating. Thus, it makes 395 # no sense to return a reference to the function here, since the 396 # recipient will not be referencing the function itself. 397 398 if node.defaults: 399 return Instance() # indicates no known target 400 401 self.add_object(function, any_scope=1) 402 return function 403 404 # Specific handler methods. 405 406 visitAdd = _visitBinary 407 408 visitAnd = OP 409 410 visitAssert = NOP 411 412 def visitAssign(self, node): 413 self.expr = self.dispatch(node.expr) 414 for n in node.nodes: 415 self.dispatch(n) 416 return None 417 418 def visitAssAttr(self, node): 419 expr = self.dispatch(node.expr) 420 if isinstance(expr, Attr): 421 if expr.name == "self": 422 if not self.store_class_attr(node.attrname): 423 self.store_instance_attr(node.attrname) 424 elif isinstance(expr.value, Module): 425 self.store_module_attr(node.attrname, expr.value) 426 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.value.name) 427 return None 428 429 def visitAssList(self, node): 430 for i, n in enumerate(node.nodes): 431 self.dispatch(n) 432 self.importer.make_constant(i) # for __getitem__(i) at run-time 433 return None 434 435 def visitAssName(self, node): 436 self.store(node.name, self.expr) 437 return None 438 439 visitAssTuple = visitAssList 440 441 def visitAugAssign(self, node): 442 443 # Accounting. 444 445 aug_method, (left_method, right_method) = augassign_methods[node.op] 446 self.importer.use_name(aug_method) 447 self.importer.use_name(left_method) 448 self.importer.use_name(right_method) 449 return self.OP(node) 450 451 visitBackquote = OP 452 453 visitBitand = _visitBinary 454 455 visitBitor = _visitBinary 456 457 visitBitxor = _visitBinary 458 459 visitBreak = NOP 460 461 visitCallFunc = OP 462 463 def visitClass(self, node): 464 465 """ 466 Register the class at the given 'node' subject to the restrictions 467 mentioned in the module docstring. 468 """ 469 470 if self.namespaces: 471 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 472 return None 473 else: 474 cls = Class(node.name, self.get_parent(), self, node) 475 476 # Visit the base class expressions, attempting to find concrete 477 # definitions of classes. 478 479 for base in node.bases: 480 expr = self.dispatch(base) 481 if isinstance(expr, Attr): 482 if expr.assignments != 1: 483 raise InspectError(self.full_name(), node, 484 "Base class %r for %r is not constant." % (base, cls.full_name())) 485 else: 486 cls.add_base(expr.value) 487 else: # if expr is None: 488 raise InspectError(self.full_name(), node, 489 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 490 491 # NOTE: Potentially dubious measure to permit __init__ availability. 492 # If no bases exist, adopt the 'object' class. 493 494 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 495 expr = self.dispatch(compiler.ast.Name("object")) 496 cls.add_base(expr.value) 497 498 # Make a back reference from the node for code generation. 499 500 node.unit = cls 501 502 # Make an entry for the class. 503 504 self.store(node.name, cls) 505 self.add_object(cls) 506 507 # Process the class body. 508 509 self.namespaces.append(cls) 510 self.dispatch(node.code) 511 self.namespaces.pop() 512 513 return cls 514 515 def visitCompare(self, node): 516 517 # Accounting. 518 # NOTE: Replicates some code in micropython.ast.visitCompare. 519 520 for op in node.ops: 521 op_name, next_node = op 522 methods = comparison_methods[op_name] 523 if methods is not None: 524 self.importer.use_name(methods[0]) 525 self.importer.use_name(methods[1]) 526 elif op_name.endswith("in"): 527 self.importer.use_name("__contains__") 528 529 return self.OP(node) 530 531 def visitConst(self, node): 532 return self.importer.make_constant(node.value) 533 534 visitContinue = NOP 535 536 visitDecorators = NOP 537 538 visitDict = OP 539 540 visitDiscard = NOP 541 542 visitDiv = _visitBinary 543 544 visitEllipsis = NOP 545 546 visitExec = NOP 547 548 visitExpression = OP 549 550 visitFloorDiv = _visitBinary 551 552 def visitFor(self, node): 553 self.in_loop = 1 554 self.NOP(node) 555 self.in_loop = 0 556 return None 557 558 def visitFrom(self, node): 559 module = self.importer.load(node.modname, 1) 560 561 if module is not None: 562 module.set_referenced() 563 564 #if module is None: 565 # print "Warning:", node.modname, "not imported." 566 567 for name, alias in node.names: 568 if name != "*": 569 if module is not None and module.namespace.has_key(name): 570 attr = module[name] 571 self.store(alias or name, attr.value) 572 if isinstance(attr.value, Module) and not attr.value.loaded: 573 self.importer.load(attr.value.name) 574 575 # Support the import of names from missing modules. 576 577 else: 578 self.store(alias or name, UnresolvedName(name, node.modname, self)) 579 else: 580 if module is not None: 581 for n in module.namespace.keys(): 582 attr = module[n] 583 self.store(n, attr.value) 584 if isinstance(attr.value, Module) and not attr.value.loaded: 585 self.importer.load(attr.value.name) 586 587 return None 588 589 def visitFunction(self, node): 590 return self._visitFunction(node, node.name) 591 592 visitGenExpr = OP 593 594 visitGenExprFor = NOP 595 596 visitGenExprIf = NOP 597 598 visitGenExprInner = NOP 599 600 def visitGetattr(self, node): 601 expr = self.dispatch(node.expr) 602 attrname = node.attrname 603 604 if isinstance(expr, Attr): 605 value = expr.value 606 if isinstance(value, (Class, Module)): 607 attr = value.namespace.get(attrname) 608 elif isinstance(value, UnresolvedName): 609 attr = UnresolvedName(attrname, value.full_name(), self) 610 else: 611 attr = None 612 elif self.builtins is not None: 613 attr = self.builtins.get(attrname) 614 else: 615 attr = UnresolvedName(attrname, value.full_name(), self) 616 617 # Accounting. 618 619 if attr is not None: 620 attr.set_referenced() 621 622 self.importer.use_name(attrname) 623 624 return attr 625 626 def visitGlobal(self, node): 627 if self.namespaces: 628 for name in node.names: 629 self.namespaces[-1].make_global(name) 630 631 # Record a global entry for the name in the module. 632 633 if not self.has_key(name): 634 self[name] = Global() 635 636 def visitIf(self, node): 637 for test, body in node.tests: 638 self.dispatch(body) 639 if node.else_ is not None: 640 self.dispatch(node.else_) 641 return None 642 643 visitIfExp = NOP 644 645 def visitImport(self, node): 646 for name, alias in node.names: 647 if alias is not None: 648 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 649 self.store(alias, module) 650 else: 651 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 652 self.store(name.split(".")[0], module) 653 module.set_referenced() 654 655 return None 656 657 visitInvert = _visitUnary 658 659 def visitKeyword(self, node): 660 self.dispatch(node.expr) 661 self.importer.make_constant(node.name) 662 self.keyword_names.add(node.name) 663 return None 664 665 def visitLambda(self, node): 666 return self._visitFunction(node, None) 667 668 visitLeftShift = _visitBinary 669 670 visitList = OP 671 672 visitListComp = OP 673 674 visitListCompFor = NOP 675 676 visitListCompIf = NOP 677 678 visitMod = _visitBinary 679 680 def visitModule(self, node): 681 682 # Make a back reference from the node for code generation. 683 684 node.unit = self 685 return self.dispatch(node.node) 686 687 visitMul = _visitBinary 688 689 def visitName(self, node): 690 name = node.name 691 692 if self.importer.predefined_constants.has_key(name): 693 attr = self.importer.get_predefined_constant(name) 694 elif self.namespaces and self.namespaces[-1].has_key(name): 695 attr = self.namespaces[-1][name] 696 elif self.has_key(name): 697 attr = self[name] 698 elif self.builtins is not None and self.builtins.has_key(name): 699 attr = self.builtins[name] 700 else: 701 attr = None 702 703 # Accounting. 704 705 if attr is not None: 706 attr.set_referenced() 707 708 self.importer.use_name(name) 709 710 return attr 711 712 visitNot = OP 713 714 visitOr = OP 715 716 visitPass = NOP 717 718 visitPower = _visitBinary 719 720 visitPrint = NOP 721 722 visitPrintnl = NOP 723 724 visitRaise = NOP 725 726 visitReturn = NOP 727 728 visitRightShift = _visitBinary 729 730 visitSlice = OP 731 732 visitSliceobj = OP 733 734 def visitStmt(self, node): 735 for n in node.nodes: 736 self.dispatch(n) 737 return None 738 739 visitSub = _visitBinary 740 741 visitSubscript = OP 742 743 def visitTryExcept(self, node): 744 self.dispatch(node.body) 745 for name, var, n in node.handlers: 746 747 # Establish the local for the handler. 748 749 if var is not None: 750 self.dispatch(var) 751 if n is not None: 752 self.dispatch(n) 753 if node.else_ is not None: 754 self.dispatch(node.else_) 755 return None 756 757 visitTryFinally = NOP 758 759 visitTuple = OP 760 761 visitUnaryAdd = _visitUnary 762 763 visitUnarySub = _visitUnary 764 765 def visitWhile(self, node): 766 self.in_loop = 1 767 self.NOP(node) 768 self.in_loop = 0 769 return None 770 771 visitWith = NOP 772 773 visitYield = NOP 774 775 class Global: 776 777 """ 778 A reference to an object assigned to a global from outside the module 779 top-level. 780 """ 781 782 pass 783 784 # vim: tabstop=4 expandtab shiftwidth=4