1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The central classes in this module are the following: 24 25 * Class 26 * Function 27 * Module 28 * InspectedModule (derived from Module) 29 30 All of the above support the Naming interface either explicitly or through 31 general conformance, meaning that all can be asked to provide their 'full_name' 32 using the method of that name. 33 34 Additionally, all of the above also support a dictionary interface in order to 35 access names within their defined scopes. Specific methods also exist in order 36 to distinguish between certain kinds of attributes: 37 38 * Class: (class|all_class|instance|all)_attributes 39 * Function: parameters, locals 40 * Module: module_attributes 41 42 These specific methods are useful in certain situations. 43 44 The above classes also provide a 'node' attribute, indicating the AST node where 45 each such object is defined. 46 """ 47 48 import compiler.ast 49 from compiler.visitor import ASTVisitor 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class InspectError(Exception): 56 57 "An inspection error." 58 59 pass 60 61 # Mix-ins and abstract classes. 62 63 class NamespaceDict: 64 65 "A mix-in providing dictionary methods." 66 67 def __init__(self, global_namespace=None): 68 self.namespace = {} 69 self.globals = set() 70 self.global_namespace = global_namespace 71 72 # Number of assignments per name. 73 74 self.assignments = {} 75 76 def __getitem__(self, name): 77 return self.namespace[name] 78 79 def get(self, name, default=None): 80 return self.namespace.get(name, default) 81 82 def __setitem__(self, name, value): 83 if name not in self.globals: 84 self.namespace[name] = value 85 86 # Record the number of assignments to each name. 87 # NOTE: Insist on assignments with known values. 88 89 if value is not None: 90 if not self.assignments.has_key(name): 91 self.assignments[name] = 1 92 else: 93 self.assignments[name] += 1 94 else: 95 self.global_namespace[name] = value 96 97 def __delitem__(self, name): 98 del self.namespace[name] 99 100 def has_key(self, name): 101 return self.namespace.has_key(name) 102 103 def keys(self): 104 return self.namespace.keys() 105 106 def values(self): 107 return self.namespace.values() 108 109 def items(self): 110 return self.namespace.items() 111 112 def make_global(self, name): 113 if not self.namespace.has_key(name): 114 self.globals.add(name) 115 else: 116 raise InspectError, "Name %r is global and local in %r" % (name, self) 117 118 class Naming: 119 120 "A mix-in providing naming conveniences." 121 122 def full_name(self): 123 if self.name is not None: 124 return self.parent_name + "." + self.name 125 else: 126 return self.parent_name 127 128 # Program data structures. 129 130 class Attr: 131 132 "An attribute entry." 133 134 def __init__(self, position, parent, value=None): 135 self.position = position 136 self.parent = parent 137 self.value = value 138 139 def __repr__(self): 140 return "Attr(%d, %r, %r)" % (self.position, self.parent, self.value) 141 142 class Const: 143 144 "A constant object." 145 146 def __init__(self, value): 147 self.value = value 148 149 # Image generation details. 150 151 self.location = None 152 153 def __repr__(self): 154 return "Const(%r, location=%r)" % (self.value, self.location) 155 156 def __eq__(self, other): 157 return self.value == other.value 158 159 def __hash__(self): 160 return hash(self.value) 161 162 class Class(NamespaceDict, Naming): 163 164 "An inspected class." 165 166 def __init__(self, name, parent_name, global_namespace=None, node=None): 167 NamespaceDict.__init__(self, global_namespace) 168 self.name = name 169 self.parent_name = parent_name 170 self.node = node 171 172 self.bases = [] 173 self.instattr = set() # instance attributes 174 175 self.all_instattr = None # cache for instance_attributes 176 self.all_instattr_names = None # from all_instattr 177 self.classattr = None # cache for class_attributes 178 self.classattr_names = None # from classattr 179 self.all_classattr = None # cache for all_class_attributes 180 self.all_classattr_names = None # from all_classattr 181 self.allattr = None # cache for all_attributes 182 self.allattr_names = None # from allattr 183 184 # Image generation details. 185 186 self.location = None 187 self.code_location = None 188 189 def __repr__(self): 190 return "Class(%r, %r, location=%r)" % (self.name, self.parent_name, self.location) 191 192 def add_base(self, base): 193 self.bases.append(base) 194 195 def add_instance_attribute(self, name): 196 self.instattr.add(name) 197 198 def class_attribute_names(self): 199 200 "Return the attribute names provided by this class only." 201 202 if self.classattr_names is None: 203 self.class_attributes() 204 return self.classattr_names 205 206 def class_attributes(self): 207 208 "Return class attributes provided by this class only." 209 210 if self.classattr is None: 211 self.classattr = {} 212 self.classattr_names = self.keys() 213 214 for i, name in enumerate(self.classattr_names): 215 self.classattr[name] = Attr(i, self, self[name]) 216 217 return self.classattr 218 219 def all_class_attribute_names(self): 220 221 "Return the attribute names provided by classes in this hierarchy." 222 223 if self.all_classattr_names is None: 224 self.all_class_attributes() 225 return self.all_classattr_names 226 227 def all_class_attributes(self): 228 229 "Return all class attributes, indicating the class which provides them." 230 231 if self.all_classattr is None: 232 self.all_classattr = {} 233 234 reversed_bases = self.bases[:] 235 reversed_bases.reverse() 236 for cls in reversed_bases: 237 self.all_classattr.update(cls.all_class_attributes()) 238 239 # Record attributes provided by this class, along with their 240 # positions. 241 242 self.all_classattr.update(self.class_attributes()) 243 244 return self.all_classattr 245 246 def instance_attribute_names(self): 247 248 "Return the instance attribute names provided by the class." 249 250 if self.all_instattr_names is None: 251 self.instance_attributes() 252 return self.all_instattr_names 253 254 def instance_attributes(self): 255 256 "Return instance-only attributes for instances of this class." 257 258 if self.all_instattr is None: 259 self.all_instattr = {} 260 instattr = set() 261 262 reversed_bases = self.bases[:] 263 reversed_bases.reverse() 264 for cls in reversed_bases: 265 instattr.update(cls.instance_attributes().keys()) 266 267 # Record instance attributes provided by this class and its bases, 268 # along with their positions. 269 270 instattr.update(self.instattr) 271 272 for i, name in enumerate(instattr): 273 self.all_instattr[name] = Attr(i, None) 274 275 self.all_instattr_names = self.all_instattr.keys() 276 277 return self.all_instattr 278 279 def all_attribute_names(self): 280 281 """ 282 Return the names of all attributes provided by instances of this class. 283 """ 284 285 self.allattr_names = self.allattr_names or self.all_attributes().keys() 286 return self.allattr_names 287 288 def all_attributes(self): 289 290 """ 291 Return all attributes for an instance, indicating either the class which 292 provides them or that the instance itself provides them. 293 """ 294 295 if self.allattr is None: 296 self.allattr = {} 297 self.allattr.update(self.all_class_attributes()) 298 for i, name in enumerate(self.instance_attribute_names()): 299 if self.allattr.has_key(name): 300 print "Instance attribute %r in %r overrides class attribute." % (name, self) 301 self.allattr[name] = Attr(i, None) 302 return self.allattr 303 304 class Function(NamespaceDict, Naming): 305 306 "An inspected function." 307 308 def __init__(self, name, parent_name, argnames, has_star, has_dstar, global_namespace=None, node=None): 309 NamespaceDict.__init__(self, global_namespace) 310 self.name = name 311 self.parent_name = parent_name 312 self.argnames = argnames 313 self.has_star = has_star 314 self.has_dstar = has_dstar 315 self.node = node 316 317 self.localnames = None # cache for locals 318 self.alllocalnames = None # cache for all_locals 319 320 # Add parameters to the namespace. 321 322 for name in argnames: 323 self[name] = None 324 325 # Image generation details. 326 327 self.location = None 328 self.code_location = None 329 330 def __repr__(self): 331 return "Function(%r, %r, %r, %r, %r, location=%r)" % ( 332 self.name, self.parent_name, self.argnames, self.has_star, self.has_dstar, self.location 333 ) 334 335 def make_global(self, name): 336 if name not in self.argnames and not self.has_key(name): 337 self.globals.add(name) 338 else: 339 raise InspectError, "Name %r is global and local in %r" % (name, self) 340 341 def parameters(self): 342 343 """ 344 Return a dictionary mapping parameter names to their position in the 345 parameter list. 346 """ 347 348 parameters = {} 349 for i, name in enumerate(self.argnames): 350 parameters[name] = i 351 return parameters 352 353 def locals(self): 354 355 "Return a dictionary mapping names to local details." 356 357 if self.localnames is None: 358 self.localnames = {} 359 start = len(self.argnames) 360 for i, name in enumerate(self.keys()): 361 self.localnames[name] = Attr(start + i, None) 362 return self.localnames 363 364 def all_locals(self): 365 366 "Return a dictionary mapping names to local and parameter details." 367 368 if self.alllocalnames is None: 369 self.alllocalnames = {} 370 self.alllocalnames.update(self.locals()) 371 for i, name in enumerate(self.argnames): 372 self.alllocalnames[name] = Attr(i, None) 373 return self.alllocalnames 374 375 class UnresolvedName(NamespaceDict, Naming): 376 377 "A module, class or function which was mentioned but could not be imported." 378 379 def __init__(self, name, parent_name, global_namespace=None): 380 NamespaceDict.__init__(self, global_namespace) 381 self.name = name 382 self.parent_name = parent_name 383 384 def all_class_attributes(self): 385 return {} 386 387 def instance_attributes(self): 388 return {} 389 390 def __repr__(self): 391 return "UnresolvedName(%r, %r)" % (self.name, self.parent_name) 392 393 class Module(NamespaceDict): 394 395 "An inspected module's core details." 396 397 def __init__(self, name): 398 NamespaceDict.__init__(self, self) 399 self.name = name 400 401 # Module attributes. 402 403 self.modattr = None # cache for module_attributes 404 self.modattr_names = None # from modattr 405 406 # Complete lists of classes and functions. 407 408 self.all_objects = [] 409 410 # Constant records. 411 412 self.constant_values = {} 413 self.constant_list = None # cache for constants 414 415 # Image generation details. 416 417 self.location = None 418 self.code_location = None 419 420 # Original location details. 421 422 self.node = None 423 424 def full_name(self): 425 return self.name 426 427 def __repr__(self): 428 return "Module(%r, location=%r)" % (self.name, self.location) 429 430 # Attribute methods. 431 432 def module_attribute_names(self): 433 434 "Return the module attribute names provided by the module." 435 436 if self.modattr_names is None: 437 self.module_attributes() 438 return self.modattr_names 439 440 def module_attributes(self): 441 442 "Return a dictionary mapping names to module attributes." 443 444 if self.modattr is None: 445 self.modattr = {} 446 self.modattr_names = self.keys() 447 for i, name in enumerate(self.modattr_names): 448 self.modattr[name] = Attr(i, self, self[name]) 449 450 return self.modattr 451 452 def constants(self): 453 454 "Return a list of constants." 455 456 if self.constant_list is None: 457 self.constant_list = list(self.constant_values.values()) 458 459 return self.constant_list 460 461 # Program visitors. 462 463 class InspectedModule(ASTVisitor, Module): 464 465 """ 466 An inspected module, providing core details via the Module superclass, but 467 capable of being used as an AST visitor. 468 """ 469 470 def __init__(self, name, importer=None): 471 ASTVisitor.__init__(self) 472 Module.__init__(self, name) 473 self.visitor = self 474 475 self.importer = importer 476 self.loaded = 0 477 478 # Current expression state. 479 480 self.expr = None 481 482 # Namespace state. 483 484 self.in_init = 0 485 self.namespaces = [] 486 self.module = None 487 488 def parse(self, filename): 489 490 "Parse the file having the given 'filename'." 491 492 module = compiler.parseFile(filename) 493 self.process(module) 494 495 def process(self, module): 496 497 "Process the given 'module'." 498 499 self.node = self.module = module 500 processed = self.dispatch(module) 501 if self.has_key("__all__"): 502 all = self["__all__"] 503 if isinstance(all, compiler.ast.List): 504 for n in all.nodes: 505 self[n.value] = self.importer.add_module(self.name + "." + n.value) 506 return processed 507 508 def vacuum(self): 509 510 "Vacuum the module namespace, removing unloaded module references." 511 512 for name, value in self.items(): 513 if isinstance(value, Module) and not value.loaded: 514 del self[name] 515 516 # Complain about globals not initialised at the module level. 517 518 if isinstance(value, Global): 519 print "Warning: global %r in module %r not initialised at the module level." % (name, self.name) 520 521 # Namespace methods. 522 523 def store(self, name, obj): 524 525 "Record attribute or local 'name', storing 'obj'." 526 527 if not self.namespaces: 528 self[name] = obj 529 else: 530 self.namespaces[-1][name] = obj 531 532 # Record all non-local objects. 533 534 if not (self.namespaces and isinstance(self.namespaces[-1], Function)): 535 self.all_objects.append(obj) 536 537 def store_attr(self, name): 538 539 "Record instance attribute 'name' in the current class." 540 541 if self.in_init: 542 543 # Current namespace is the function. 544 # Previous namespace is the class. 545 546 self.namespaces[-2].add_instance_attribute(name) 547 548 def get_parent(self): 549 return (self.namespaces[-1:] or [self])[0] 550 551 # Visitor methods. 552 553 def default(self, node, *args): 554 raise InspectError, node.__class__ 555 556 def dispatch(self, node, *args): 557 return ASTVisitor.dispatch(self, node, *args) 558 559 def NOP(self, node): 560 for n in node.getChildNodes(): 561 self.dispatch(n) 562 return None 563 564 visitAdd = NOP 565 566 visitAnd = NOP 567 568 visitAssert = NOP 569 570 def visitAssign(self, node): 571 self.expr = self.dispatch(node.expr) 572 for n in node.nodes: 573 self.dispatch(n) 574 return None 575 576 def visitAssAttr(self, node): 577 expr = self.dispatch(node.expr) 578 if expr is not None and isinstance(expr, Self): 579 self.store_attr(node.attrname) 580 return None 581 582 def visitAssList(self, node): 583 for n in node.nodes: 584 self.dispatch(n) 585 return None 586 587 def visitAssName(self, node): 588 self.store(node.name, self.expr) 589 return None 590 591 visitAssTuple = visitAssList 592 593 visitAugAssign = NOP 594 595 visitBackquote = NOP 596 597 visitBitand = NOP 598 599 visitBitor = NOP 600 601 visitBitxor = NOP 602 603 visitBreak = NOP 604 605 visitCallFunc = NOP 606 607 def visitClass(self, node): 608 if self.namespaces: 609 print "Class %r in %r is not global: ignored." % (node.name, self) 610 else: 611 cls = Class(node.name, self.get_parent().full_name(), self, node) 612 for base in node.bases: 613 base_ref = self.dispatch(base) 614 if base_ref is None: 615 raise InspectError, "Base class %r for class %r in %r is not found: it may be hidden in some way." % (base, self, cls) 616 cls.add_base(base_ref) 617 618 # Make a back reference from the node for code generation. 619 620 node.cls = cls 621 622 # Make an entry for the class. 623 624 self.store(node.name, cls) 625 626 self.namespaces.append(cls) 627 self.dispatch(node.code) 628 self.namespaces.pop() 629 630 return None 631 632 visitCompare = NOP 633 634 def visitConst(self, node): 635 const = Const(node.value) 636 self.constant_values[node.value] = const 637 return const 638 639 visitContinue = NOP 640 641 visitDecorators = NOP 642 643 visitDict = NOP 644 645 visitDiscard = NOP 646 647 visitDiv = NOP 648 649 visitEllipsis = NOP 650 651 visitExec = NOP 652 653 visitExpression = NOP 654 655 visitFloorDiv = NOP 656 657 visitFor = NOP 658 659 def visitFrom(self, node): 660 if self.importer is None: 661 raise InspectError, "Please use the micropython.Importer class for code which uses the 'from' statement." 662 663 module = self.importer.load(node.modname, 1) 664 665 if module is None: 666 print "Warning:", node.modname, "not imported." 667 668 for name, alias in node.names: 669 if name != "*": 670 if module is not None and module.namespace.has_key(name): 671 attr = module[name] 672 self.store(alias or name, attr) 673 if isinstance(attr, Module) and not attr.loaded: 674 self.importer.load(attr.name) 675 676 # Support the import of names from missing modules. 677 678 else: 679 self.store(alias or name, UnresolvedName(name, node.modname, self)) 680 else: 681 if module is not None: 682 for n in module.namespace.keys(): 683 attr = module[n] 684 self.store(n, attr) 685 if isinstance(attr, Module) and not attr.loaded: 686 self.importer.load(attr.name) 687 688 return None 689 690 def visitFunction(self, node): 691 function = Function( 692 node.name, 693 self.get_parent().full_name(), 694 node.argnames, 695 (node.flags & 4 != 0), 696 (node.flags & 8 != 0), 697 self, 698 node 699 ) 700 701 # Make a back reference from the node for code generation. 702 703 node.function = function 704 705 self.namespaces.append(function) 706 707 # Current namespace is the function. 708 # Previous namespace is the class. 709 710 if node.name == "__init__" and isinstance(self.namespaces[-2], Class): 711 self.in_init = 1 712 713 self.dispatch(node.code) 714 self.in_init = 0 715 self.namespaces.pop() 716 717 self.store(node.name, function) 718 return None 719 720 visitGenExpr = NOP 721 722 visitGenExprFor = NOP 723 724 visitGenExprIf = NOP 725 726 visitGenExprInner = NOP 727 728 def visitGetattr(self, node): 729 expr = self.dispatch(node.expr) 730 if expr is not None: 731 if isinstance(expr, Module): 732 return expr.namespace.get(node.attrname) 733 elif isinstance(expr, UnresolvedName): 734 return UnresolvedName(node.attrname, expr.full_name(), self) 735 return builtins.get(node.attrname) 736 737 def visitGlobal(self, node): 738 if self.namespaces: 739 for name in node.names: 740 self.namespaces[-1].make_global(name) 741 if not self.has_key(name): 742 self[name] = Global() 743 744 def visitIf(self, node): 745 for test, body in node.tests: 746 self.dispatch(body) 747 if node.else_ is not None: 748 self.dispatch(node.else_) 749 return None 750 751 visitIfExp = NOP 752 753 def visitImport(self, node): 754 if self.importer is None: 755 raise InspectError, "Please use the micropython.Importer class for code which uses the 'import' statement." 756 757 for name, alias in node.names: 758 if alias is not None: 759 self.store(alias, self.importer.load(name, 1) or UnresolvedName(None, name, self)) 760 else: 761 self.store(name.split(".")[0], self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self)) 762 763 return None 764 765 visitInvert = NOP 766 767 visitKeyword = NOP 768 769 visitLambda = NOP 770 771 visitLeftShift = NOP 772 773 visitList = NOP 774 775 visitListComp = NOP 776 777 visitListCompFor = NOP 778 779 visitListCompIf = NOP 780 781 visitMod = NOP 782 783 def visitModule(self, node): 784 return self.dispatch(node.node) 785 786 visitMul = NOP 787 788 def visitName(self, node): 789 name = node.name 790 if name == "self": 791 return Self() 792 elif self.has_key(name): 793 return self[name] 794 elif builtins.has_key(name): 795 return builtins[name] 796 else: 797 return None 798 799 visitNot = NOP 800 801 visitOr = NOP 802 803 visitPass = NOP 804 805 visitPower = NOP 806 807 visitPrint = NOP 808 809 visitPrintnl = NOP 810 811 visitRaise = NOP 812 813 visitReturn = NOP 814 815 visitRightShift = NOP 816 817 visitSlice = NOP 818 819 visitSliceobj = NOP 820 821 def visitStmt(self, node): 822 for n in node.nodes: 823 self.dispatch(n) 824 return None 825 826 visitSub = NOP 827 828 visitSubscript = NOP 829 830 def visitTryExcept(self, node): 831 self.dispatch(node.body) 832 for name, var, n in node.handlers: 833 self.dispatch(n) 834 if node.else_ is not None: 835 self.dispatch(node.else_) 836 return None 837 838 visitTryFinally = NOP 839 840 visitTuple = NOP 841 842 visitUnaryAdd = NOP 843 844 visitUnarySub = NOP 845 846 visitWhile = NOP 847 848 visitWith = NOP 849 850 visitYield = NOP 851 852 class Self: 853 854 "A reference to an object within a method." 855 856 pass 857 858 class Global: 859 860 """ 861 A reference to an object assigned to a global from outside the module 862 top-level. 863 """ 864 865 pass 866 867 # Built-in types initialisation. 868 869 class Builtins(Module): 870 871 "The special built-in types module." 872 873 def __init__(self): 874 Module.__init__(self, "__builtins__") 875 876 for key in ['ArithmeticError', 'AssertionError', 'AttributeError', 877 'BaseException', 'DeprecationWarning', 'EOFError', 'Ellipsis', 878 'EnvironmentError', 'Exception', 'False', 'FloatingPointError', 879 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 880 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', 881 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', 882 'None', 'NotImplemented', 'NotImplementedError', 'OSError', 883 'OverflowError', 'PendingDeprecationWarning', 'ReferenceError', 884 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration', 885 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 886 'TabError', 'True', 'TypeError', 'UnboundLocalError', 887 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 888 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', 889 'ValueError', 'Warning', 'ZeroDivisionError', 890 'bool', 'buffer', 'complex', 'dict', 'file', 'float', 'int', 'list', 891 'long', 'object', 'slice', 'str', 'tuple', 'type', 'unicode', 892 'xrange']: 893 self[key] = Class(key, self.full_name(), self) 894 895 for key in ['id', 'len']: 896 self[key] = Function(key, self.full_name(), ['arg'], 0, 0, self) 897 898 builtins = Builtins() 899 900 # vim: tabstop=4 expandtab shiftwidth=4