1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007, 2008 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The results of inspecting a module are as follows: 24 25 Constants 26 --------- 27 28 All constants identified within the code shall be registered. 29 30 Classes 31 ------- 32 33 All global classes shall be registered; local classes (within functions) or 34 nested classes (within classes) are not currently registered. 35 36 Base classes must be detected and constant. 37 38 All classes without bases are made to inherit from __builtins__.object in order 39 to support some standard methods. 40 41 Functions 42 --------- 43 44 All functions and lambda definitions shall be registered. 45 46 Namespaces 47 ---------- 48 49 Modules define their own "global" namespace, within which classes, functions 50 and lambda definitions establish a hierarchy of namespaces. 51 52 Only local, global and built-in namespaces are recognised; closures are not 53 supported. 54 55 Assignments 56 ----------- 57 58 Name assignment and attribute assignment involving modules and classes cause 59 names to be associated with values within namespaces. 60 61 Any assignments within loops are considered to cause the targets of such 62 assignments to provide non-constant values. 63 64 Assignments to names are only really considered to cause the targets of such 65 assignments to provide constant values if the targets reside in the module 66 namespace or in class namespaces, subject to the above conditions. 67 68 Assignments to names within functions are not generally considered to cause the 69 targets of such assignments to provide constant values since functions can be 70 invoked many times with different inputs. However, there may be benefits in 71 considering a local to be constant within a single invocation. 72 """ 73 74 from micropython.common import * 75 from micropython.data import * 76 import compiler.ast 77 from compiler.visitor import ASTVisitor 78 79 # Program visitors. 80 81 class InspectedModule(ASTVisitor, Module): 82 83 """ 84 An inspected module, providing core details via the Module superclass, but 85 capable of being used as an AST visitor. 86 """ 87 88 def __init__(self, name, importer): 89 90 """ 91 Initialise this visitor with a module 'name' and an 'importer' which is 92 used to provide access to other modules when required. 93 """ 94 95 ASTVisitor.__init__(self) 96 Module.__init__(self, name) 97 self.visitor = self 98 99 # Import machinery links. 100 101 self.importer = importer 102 self.optimisations = importer.optimisations 103 self.builtins = self.importer.modules.get("__builtins__") 104 self.loaded = 0 105 106 # Current expression state. 107 108 self.expr = None 109 110 # Namespace state. 111 112 self.in_init = 0 # Find instance attributes in __init__ methods. 113 self.in_method = 0 # Find instance attributes in all methods. 114 self.in_loop = 0 # Note loop "membership", affecting assignments. 115 self.namespaces = [] 116 self.module = None 117 118 def parse(self, filename): 119 120 "Parse the file having the given 'filename'." 121 122 module = compiler.parseFile(filename) 123 self.process(module) 124 125 def process(self, module): 126 127 "Process the given 'module'." 128 129 self.astnode = self.module = module 130 processed = self.dispatch(module) 131 if self.has_key("__all__"): 132 all = self["__all__"] 133 if isinstance(all, compiler.ast.List): 134 for n in all.nodes: 135 self.store(n.value, self.importer.add_module(self.name + "." + n.value)) 136 return processed 137 138 def vacuum(self): 139 140 """ 141 Vacuum the module namespace, removing unreferenced objects and unused 142 names. 143 """ 144 145 for name, value in self.items(): 146 147 if self.should_optimise_unused_objects(): 148 149 # Remove unused names. 150 151 if name not in self.importer.names_used: 152 del self[name] 153 154 # Remove entries for unreferenced objects. 155 # This, due to the nature of the referenced attribute, assumes 156 # that only explicitly mentioned classes and functions are 157 # employed in the final program. 158 159 elif isinstance(value, Attr): 160 attr_value = value.value 161 162 # Only remove entries for classes and functions, not methods. 163 164 for attr_value in value.assignment_values: 165 if (isinstance(attr_value, Function) and not attr_value.is_method() or 166 isinstance(attr_value, Class)) and not attr_value.referenced: 167 pass 168 else: 169 break 170 else: 171 del self[name] 172 173 # Complain about globals not initialised at the module level. 174 175 if isinstance(value, Global): 176 print "Warning: global %r in module %r not initialised at the module level." % (name, self.name) 177 178 # Remove unreferenced objects. 179 180 if self.should_optimise_unused_objects(): 181 182 all_objects = list(self.all_objects) 183 184 for obj in all_objects: 185 186 # Only remove entries for classes and functions, not methods. 187 188 if (isinstance(obj, Function) and not obj.is_method() or 189 isinstance(obj, Class)) and not obj.referenced: 190 191 self.all_objects.remove(obj) 192 193 # Remove unused entries from classes plus associated methods. 194 195 if isinstance(obj, Class): 196 for name, attr in obj.class_attributes().items(): 197 198 if name not in self.importer.names_used: 199 del obj[name] 200 201 # Methods can only be deleted if they are the only 202 # assigned object to the class and are unreferenced. 203 204 if attr.assignments == 1 and isinstance(attr.value, Function) and \ 205 attr.value.is_method() and not attr.value.referenced: 206 207 self.all_objects.remove(attr.value) 208 209 def finalise(self): 210 211 "Finalise the module." 212 213 for obj in self.all_objects: 214 if isinstance(obj, (Class, Function)): 215 obj.finalise_attributes() 216 217 def add_object(self, obj, any_scope=0): 218 219 """ 220 Record 'obj' if non-local or if the optional 'any_scope' is set to a 221 true value. 222 """ 223 224 if any_scope or not (self.namespaces and isinstance(self.namespaces[-1], Function)): 225 self.all_objects.add(obj) 226 227 # Optimisation tests. 228 229 def should_optimise_unused_objects(self): 230 return "unused_objects" in self.optimisations 231 232 # Namespace methods. 233 234 def store(self, name, obj): 235 236 "Record attribute or local 'name', storing 'obj'." 237 238 if not self.namespaces: 239 self.set(name, obj, not self.in_loop) 240 else: 241 self.namespaces[-1].set(name, obj, not self.in_loop) 242 243 def store_lambda(self, obj): 244 245 "Store a lambda function 'obj'." 246 247 self.add_object(obj) 248 249 def store_module_attr(self, name, module): 250 251 """ 252 Record module attribute 'name' in the given 'module' using the current 253 expression. 254 """ 255 256 if isinstance(self.expr, Attr): 257 assigned_value = self.expr.value 258 else: 259 assigned_value = self.expr 260 261 module.set(name, assigned_value, 0) 262 263 def store_class_attr(self, name): 264 265 """ 266 Record class attribute 'name' in the current class using the current 267 expression. 268 """ 269 270 if self.in_method and self.namespaces[-2].has_key(name): 271 272 if isinstance(self.expr, Attr): 273 assigned_value = self.expr.value 274 else: 275 assigned_value = self.expr 276 277 self.namespaces[-2].set(name, assigned_value, 0) 278 return 1 279 280 return 0 281 282 def store_instance_attr(self, name): 283 284 "Record instance attribute 'name' in the current class." 285 286 if self.in_method: 287 288 # Current namespace is the function. 289 # Previous namespace is the class. 290 291 self.namespaces[-2].add_instance_attribute(name) 292 293 def get_parent(self): 294 295 "Return the parent (or most recent) namespace currently exposed." 296 297 return (self.namespaces[-1:] or [self])[0] 298 299 # Visitor methods. 300 301 def default(self, node, *args): 302 raise InspectError(self.full_name(), node, "Node class %r is not supported." % node.__class__) 303 304 def dispatch(self, node, *args): 305 return ASTVisitor.dispatch(self, node, *args) 306 307 def NOP(self, node): 308 for n in node.getChildNodes(): 309 self.dispatch(n) 310 return None 311 312 def OP(self, node): 313 for n in node.getChildNodes(): 314 self.dispatch(n) 315 return Instance() 316 317 def _visitFunction(self, node, name): 318 319 """ 320 Return a function object for the function defined by 'node' with the 321 given 'name'. If a lambda expression is being visited, 'name' should be 322 None. 323 """ 324 325 function = Function( 326 name, 327 self.get_parent(), 328 node.argnames, 329 node.defaults, 330 (node.flags & 4 != 0), 331 (node.flags & 8 != 0), 332 self, 333 node 334 ) 335 336 # Make a back reference from the node for code generation. 337 338 node.unit = function 339 340 # Process the defaults. 341 342 for n in node.defaults: 343 self.expr = self.dispatch(n) 344 if isinstance(self.expr, Attr): 345 function.store_default(self.expr.value) 346 else: 347 function.store_default(self.expr) 348 349 # Enter the function. 350 351 self.namespaces.append(function) 352 353 # Current namespace is the function. 354 # Previous namespace is the class. 355 356 if len(self.namespaces) > 1 and isinstance(self.namespaces[-2], Class): 357 if name == "__init__": 358 self.in_init = 1 359 self.in_method = 1 360 361 self.dispatch(node.code) 362 self.in_init = 0 363 self.in_method = 0 364 self.namespaces.pop() 365 366 if name is not None: 367 self.store(name, function) 368 else: 369 self.store_lambda(function) 370 371 # Lambda functions are always assumed to be referenced. This is 372 # because other means of discovering the referencing of objects rely 373 # on the original names inherent in the definition of those objects. 374 375 function.set_referenced() 376 377 # Where defaults exist, an instance needs creating. Thus, it makes 378 # no sense to return a reference to the function here, since the 379 # recipient will not be referencing the function itself. 380 381 if node.defaults: 382 return Instance() # indicates no known target 383 384 self.add_object(function, any_scope=1) 385 return function 386 387 visitAdd = OP 388 389 visitAnd = OP 390 391 visitAssert = NOP 392 393 def visitAssign(self, node): 394 self.expr = self.dispatch(node.expr) 395 for n in node.nodes: 396 self.dispatch(n) 397 return None 398 399 def visitAssAttr(self, node): 400 expr = self.dispatch(node.expr) 401 if isinstance(expr, Attr): 402 if expr.name == "self": 403 if not self.store_class_attr(node.attrname): 404 self.store_instance_attr(node.attrname) 405 elif isinstance(expr.value, Module): 406 self.store_module_attr(node.attrname, expr.value) 407 print "Warning: attribute %r of module %r set outside the module." % (node.attrname, expr.value.name) 408 return None 409 410 def visitAssList(self, node): 411 for i, n in enumerate(node.nodes): 412 self.dispatch(n) 413 self.importer.make_constant(i) # for __getitem__(i) at run-time 414 return None 415 416 def visitAssName(self, node): 417 self.store(node.name, self.expr) 418 return None 419 420 visitAssTuple = visitAssList 421 422 visitAugAssign = OP 423 424 visitBackquote = OP 425 426 visitBitand = OP 427 428 visitBitor = OP 429 430 visitBitxor = OP 431 432 visitBreak = NOP 433 434 visitCallFunc = OP 435 436 def visitClass(self, node): 437 438 """ 439 Register the class at the given 'node' subject to the restrictions 440 mentioned in the module docstring. 441 """ 442 443 if self.namespaces: 444 print "Class %r in %r is not global: ignored." % (node.name, self.namespaces[-1].full_name()) 445 return None 446 else: 447 cls = Class(node.name, self.get_parent(), self, node) 448 449 # Visit the base class expressions, attempting to find concrete 450 # definitions of classes. 451 452 for base in node.bases: 453 expr = self.dispatch(base) 454 if isinstance(expr, Attr): 455 if expr.assignments != 1: 456 raise InspectError(self.full_name(), node, 457 "Base class %r for %r is not constant." % (base, cls.full_name())) 458 else: 459 cls.add_base(expr.value) 460 else: # if expr is None: 461 raise InspectError(self.full_name(), node, 462 "Base class %r for %r is not found: it may be hidden in some way." % (base, cls.full_name())) 463 464 # NOTE: Potentially dubious measure to permit __init__ availability. 465 # If no bases exist, adopt the 'object' class. 466 467 if not node.bases and not (self.name == "__builtins__" and node.name == "object") : 468 expr = self.dispatch(compiler.ast.Name("object")) 469 cls.add_base(expr.value) 470 471 # Make a back reference from the node for code generation. 472 473 node.unit = cls 474 475 # Make an entry for the class. 476 477 self.store(node.name, cls) 478 self.add_object(cls) 479 480 # Process the class body. 481 482 self.namespaces.append(cls) 483 self.dispatch(node.code) 484 self.namespaces.pop() 485 486 return cls 487 488 visitCompare = OP 489 490 def visitConst(self, node): 491 return self.importer.make_constant(node.value) 492 493 visitContinue = NOP 494 495 visitDecorators = NOP 496 497 visitDict = OP 498 499 visitDiscard = NOP 500 501 visitDiv = OP 502 503 visitEllipsis = NOP 504 505 visitExec = NOP 506 507 visitExpression = OP 508 509 visitFloorDiv = OP 510 511 def visitFor(self, node): 512 self.in_loop = 1 513 self.NOP(node) 514 self.in_loop = 0 515 return None 516 517 def visitFrom(self, node): 518 module = self.importer.load(node.modname, 1) 519 520 if module is not None: 521 module.set_referenced() 522 523 #if module is None: 524 # print "Warning:", node.modname, "not imported." 525 526 for name, alias in node.names: 527 if name != "*": 528 if module is not None and module.namespace.has_key(name): 529 attr = module[name] 530 self.store(alias or name, attr.value) 531 if isinstance(attr.value, Module) and not attr.value.loaded: 532 self.importer.load(attr.value.name) 533 534 # Support the import of names from missing modules. 535 536 else: 537 self.store(alias or name, UnresolvedName(name, node.modname, self)) 538 else: 539 if module is not None: 540 for n in module.namespace.keys(): 541 attr = module[n] 542 self.store(n, attr.value) 543 if isinstance(attr.value, Module) and not attr.value.loaded: 544 self.importer.load(attr.value.name) 545 546 return None 547 548 def visitFunction(self, node): 549 return self._visitFunction(node, node.name) 550 551 visitGenExpr = OP 552 553 visitGenExprFor = NOP 554 555 visitGenExprIf = NOP 556 557 visitGenExprInner = NOP 558 559 def visitGetattr(self, node): 560 expr = self.dispatch(node.expr) 561 attrname = node.attrname 562 563 if isinstance(expr, Attr): 564 value = expr.value 565 if isinstance(value, (Class, Module)): 566 attr = value.namespace.get(attrname) 567 elif isinstance(value, UnresolvedName): 568 attr = UnresolvedName(attrname, value.full_name(), self) 569 else: 570 attr = None 571 elif self.builtins is not None: 572 attr = self.builtins.get(attrname) 573 else: 574 attr = UnresolvedName(attrname, value.full_name(), self) 575 576 # Accounting. 577 578 if attr is not None: 579 attr.set_referenced() 580 581 self.importer.use_name(attrname) 582 583 return attr 584 585 def visitGlobal(self, node): 586 if self.namespaces: 587 for name in node.names: 588 self.namespaces[-1].make_global(name) 589 590 # Record a global entry for the name in the module. 591 592 if not self.has_key(name): 593 self[name] = Global() 594 595 def visitIf(self, node): 596 for test, body in node.tests: 597 self.dispatch(body) 598 if node.else_ is not None: 599 self.dispatch(node.else_) 600 return None 601 602 visitIfExp = NOP 603 604 def visitImport(self, node): 605 for name, alias in node.names: 606 if alias is not None: 607 module = self.importer.load(name, 1) or UnresolvedName(None, name, self) 608 self.store(alias, module) 609 else: 610 module = self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self) 611 self.store(name.split(".")[0], module) 612 module.set_referenced() 613 614 return None 615 616 visitInvert = OP 617 618 def visitKeyword(self, node): 619 self.dispatch(node.expr) 620 self.importer.make_constant(node.name) 621 self.keyword_names.add(node.name) 622 return None 623 624 def visitLambda(self, node): 625 return self._visitFunction(node, None) 626 627 visitLeftShift = OP 628 629 visitList = OP 630 631 visitListComp = OP 632 633 visitListCompFor = NOP 634 635 visitListCompIf = NOP 636 637 visitMod = OP 638 639 def visitModule(self, node): 640 641 # Make a back reference from the node for code generation. 642 643 node.unit = self 644 return self.dispatch(node.node) 645 646 visitMul = OP 647 648 def visitName(self, node): 649 name = node.name 650 651 if self.importer.predefined_constants.has_key(name): 652 attr = self.importer.get_predefined_constant(name) 653 elif self.namespaces and self.namespaces[-1].has_key(name): 654 attr = self.namespaces[-1][name] 655 elif self.has_key(name): 656 attr = self[name] 657 elif self.builtins is not None and self.builtins.has_key(name): 658 attr = self.builtins[name] 659 else: 660 attr = None 661 662 # Accounting. 663 664 if attr is not None: 665 attr.set_referenced() 666 667 self.importer.use_name(name) 668 669 return attr 670 671 visitNot = OP 672 673 visitOr = OP 674 675 visitPass = NOP 676 677 visitPower = OP 678 679 visitPrint = NOP 680 681 visitPrintnl = NOP 682 683 visitRaise = NOP 684 685 visitReturn = NOP 686 687 visitRightShift = OP 688 689 visitSlice = OP 690 691 visitSliceobj = OP 692 693 def visitStmt(self, node): 694 for n in node.nodes: 695 self.dispatch(n) 696 return None 697 698 visitSub = OP 699 700 visitSubscript = OP 701 702 def visitTryExcept(self, node): 703 self.dispatch(node.body) 704 for name, var, n in node.handlers: 705 706 # Establish the local for the handler. 707 708 if var is not None: 709 self.dispatch(var) 710 if n is not None: 711 self.dispatch(n) 712 if node.else_ is not None: 713 self.dispatch(node.else_) 714 return None 715 716 visitTryFinally = NOP 717 718 visitTuple = OP 719 720 visitUnaryAdd = OP 721 722 visitUnarySub = OP 723 724 def visitWhile(self, node): 725 self.in_loop = 1 726 self.NOP(node) 727 self.in_loop = 0 728 return None 729 730 visitWith = NOP 731 732 visitYield = NOP 733 734 class Global: 735 736 """ 737 A reference to an object assigned to a global from outside the module 738 top-level. 739 """ 740 741 pass 742 743 # vim: tabstop=4 expandtab shiftwidth=4