1 #!/usr/bin/env python 2 3 """ 4 Inspect source files, obtaining details of classes and attributes. 5 6 Copyright (C) 2007 Paul Boddie <paul@boddie.org.uk> 7 8 This program is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free Software 10 Foundation; either version 3 of the License, or (at your option) any later 11 version. 12 13 This program is distributed in the hope that it will be useful, but WITHOUT 14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 15 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 16 details. 17 18 You should have received a copy of the GNU General Public License along with 19 this program. If not, see <http://www.gnu.org/licenses/>. 20 21 -------- 22 23 The central classes in this module are the following: 24 25 * Class 26 * Function 27 * Module 28 * InspectedModule (derived from Module) 29 30 All of the above support the Naming interface either explicitly or through 31 general conformance, meaning that all can be asked to provide their 'full_name' 32 using the method of that name. 33 34 Additionally, all of the above also support a dictionary interface in order to 35 access names within their defined scopes. Specific methods also exist in order 36 to distinguish between certain kinds of attributes: 37 38 * Class: (class|all_class|instance|all)_attributes 39 * Function: parameters, locals 40 * Module: module_attributes 41 42 These specific methods are useful in certain situations. 43 44 The above classes also provide a 'node' attribute, indicating the AST node where 45 each such object is defined. 46 """ 47 48 import compiler.ast 49 from compiler.visitor import ASTVisitor 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class InspectError(Exception): 56 57 "An inspection error." 58 59 pass 60 61 # Mix-ins and abstract classes. 62 63 class NamespaceDict: 64 65 "A mix-in providing dictionary methods." 66 67 def __init__(self, global_namespace=None): 68 self.namespace = {} 69 self.globals = set() 70 self.global_namespace = global_namespace 71 72 # Number of assignments per name. 73 74 self.assignments = {} 75 76 def __getitem__(self, name): 77 return self.namespace[name] 78 79 def get(self, name, default=None): 80 return self.namespace.get(name, default) 81 82 def __setitem__(self, name, value): 83 if name not in self.globals: 84 self.namespace[name] = value 85 86 # Record the number of assignments to each name. 87 # NOTE: Insist on assignments with known values. 88 89 if value is not None: 90 if not self.assignments.has_key(name): 91 self.assignments[name] = 1 92 else: 93 self.assignments[name] += 1 94 else: 95 self.global_namespace[name] = value 96 97 def __delitem__(self, name): 98 del self.namespace[name] 99 100 def has_key(self, name): 101 return self.namespace.has_key(name) 102 103 def keys(self): 104 return self.namespace.keys() 105 106 def values(self): 107 return self.namespace.values() 108 109 def items(self): 110 return self.namespace.items() 111 112 def make_global(self, name): 113 if not self.namespace.has_key(name): 114 self.globals.add(name) 115 else: 116 raise InspectError, "Name %r is global and local in %r" % (name, self) 117 118 class Naming: 119 120 "A mix-in providing naming conveniences." 121 122 def full_name(self): 123 if self.name is not None: 124 return self.parent_name + "." + self.name 125 else: 126 return self.parent_name 127 128 # Program data structures. 129 130 class Attr: 131 132 "An attribute entry." 133 134 def __init__(self, position, parent, assignments=None, value=None): 135 self.position = position 136 self.parent = parent 137 self.assignments = assignments 138 self.value = value 139 140 def __repr__(self): 141 return "Attr(%d, %r, %r, %r)" % (self.position, self.parent, self.assignments, self.value) 142 143 class Const: 144 145 "A constant object." 146 147 def __init__(self, value): 148 self.value = value 149 150 # Image generation details. 151 152 self.location = None 153 154 def __repr__(self): 155 return "Const(%r, location=%r)" % (self.value, self.location) 156 157 def __eq__(self, other): 158 return self.value == other.value 159 160 def __hash__(self): 161 return hash(self.value) 162 163 class Class(NamespaceDict, Naming): 164 165 "An inspected class." 166 167 def __init__(self, name, parent_name, global_namespace=None, node=None): 168 NamespaceDict.__init__(self, global_namespace) 169 self.name = name 170 self.parent_name = parent_name 171 self.node = node 172 173 self.bases = [] 174 self.instattr = set() # instance attributes 175 176 self.all_instattr = None # cache for instance_attributes 177 self.all_instattr_names = None # from all_instattr 178 self.classattr = None # cache for class_attributes 179 self.classattr_names = None # from classattr 180 self.all_classattr = None # cache for all_class_attributes 181 self.all_classattr_names = None # from all_classattr 182 self.allattr = None # cache for all_attributes 183 self.allattr_names = None # from allattr 184 185 # Image generation details. 186 187 self.location = None 188 self.code_location = None 189 190 def __repr__(self): 191 return "Class(%r, %r, location=%r)" % (self.name, self.parent_name, self.location) 192 193 def add_base(self, base): 194 self.bases.append(base) 195 196 def add_instance_attribute(self, name): 197 self.instattr.add(name) 198 199 def class_attribute_names(self): 200 201 "Return the attribute names provided by this class only." 202 203 if self.classattr_names is None: 204 self.class_attributes() 205 return self.classattr_names 206 207 def class_attributes(self): 208 209 "Return class attributes provided by this class only." 210 211 if self.classattr is None: 212 self.classattr = {} 213 self.classattr_names = self.keys() 214 215 for i, name in enumerate(self.classattr_names): 216 self.classattr[name] = Attr(i, self, self.assignments[name], self[name]) 217 218 return self.classattr 219 220 def all_class_attribute_names(self): 221 222 "Return the attribute names provided by classes in this hierarchy." 223 224 if self.all_classattr_names is None: 225 self.all_class_attributes() 226 return self.all_classattr_names 227 228 def all_class_attributes(self): 229 230 "Return all class attributes, indicating the class which provides them." 231 232 if self.all_classattr is None: 233 self.all_classattr = {} 234 235 reversed_bases = self.bases[:] 236 reversed_bases.reverse() 237 for cls in reversed_bases: 238 self.all_classattr.update(cls.all_class_attributes()) 239 240 # Record attributes provided by this class, along with their 241 # positions. 242 243 self.all_classattr.update(self.class_attributes()) 244 245 return self.all_classattr 246 247 def instance_attribute_names(self): 248 249 "Return the instance attribute names provided by the class." 250 251 if self.all_instattr_names is None: 252 self.instance_attributes() 253 return self.all_instattr_names 254 255 def instance_attributes(self): 256 257 "Return instance-only attributes for instances of this class." 258 259 if self.all_instattr is None: 260 self.all_instattr = {} 261 instattr = set() 262 263 reversed_bases = self.bases[:] 264 reversed_bases.reverse() 265 for cls in reversed_bases: 266 instattr.update(cls.instance_attributes().keys()) 267 268 # Record instance attributes provided by this class and its bases, 269 # along with their positions. 270 271 instattr.update(self.instattr) 272 273 for i, name in enumerate(instattr): 274 self.all_instattr[name] = Attr(i, None) 275 276 self.all_instattr_names = self.all_instattr.keys() 277 278 return self.all_instattr 279 280 def all_attribute_names(self): 281 282 """ 283 Return the names of all attributes provided by instances of this class. 284 """ 285 286 self.allattr_names = self.allattr_names or self.all_attributes().keys() 287 return self.allattr_names 288 289 def all_attributes(self): 290 291 """ 292 Return all attributes for an instance, indicating either the class which 293 provides them or that the instance itself provides them. 294 """ 295 296 if self.allattr is None: 297 self.allattr = {} 298 self.allattr.update(self.all_class_attributes()) 299 for i, name in enumerate(self.instance_attribute_names()): 300 if self.allattr.has_key(name): 301 print "Instance attribute %r in %r overrides class attribute." % (name, self) 302 self.allattr[name] = Attr(i, None) 303 return self.allattr 304 305 class Function(NamespaceDict, Naming): 306 307 "An inspected function." 308 309 def __init__(self, name, parent_name, argnames, has_star, has_dstar, global_namespace=None, node=None): 310 NamespaceDict.__init__(self, global_namespace) 311 self.name = name 312 self.parent_name = parent_name 313 self.argnames = argnames 314 self.has_star = has_star 315 self.has_dstar = has_dstar 316 self.node = node 317 318 self.localnames = None # cache for locals 319 self.alllocalnames = None # cache for all_locals 320 321 # Add parameters to the namespace. 322 323 self._add_parameters(argnames) 324 325 # Image generation details. 326 327 self.location = None 328 self.code_location = None 329 330 def _add_parameters(self, argnames): 331 for name in argnames: 332 if isinstance(name, tuple): 333 self._add_parameters(name) 334 else: 335 self[name] = None 336 337 def __repr__(self): 338 return "Function(%r, %r, %r, %r, %r, location=%r)" % ( 339 self.name, self.parent_name, self.argnames, self.has_star, self.has_dstar, self.location 340 ) 341 342 def make_global(self, name): 343 if name not in self.argnames and not self.has_key(name): 344 self.globals.add(name) 345 else: 346 raise InspectError, "Name %r is global and local in %r" % (name, self) 347 348 def parameters(self): 349 350 """ 351 Return a dictionary mapping parameter names to their position in the 352 parameter list. 353 """ 354 355 parameters = {} 356 for i, name in enumerate(self.argnames): 357 parameters[name] = i 358 return parameters 359 360 def locals(self): 361 362 "Return a dictionary mapping names to local details." 363 364 if self.localnames is None: 365 self.localnames = {} 366 start = len(self.argnames) 367 for i, name in enumerate(self.keys()): 368 self.localnames[name] = Attr(start + i, None) 369 return self.localnames 370 371 def all_locals(self): 372 373 "Return a dictionary mapping names to local and parameter details." 374 375 if self.alllocalnames is None: 376 self.alllocalnames = {} 377 self.alllocalnames.update(self.locals()) 378 for i, name in enumerate(self.argnames): 379 self.alllocalnames[name] = Attr(i, None) 380 return self.alllocalnames 381 382 class UnresolvedName(NamespaceDict, Naming): 383 384 "A module, class or function which was mentioned but could not be imported." 385 386 def __init__(self, name, parent_name, global_namespace=None): 387 NamespaceDict.__init__(self, global_namespace) 388 self.name = name 389 self.parent_name = parent_name 390 391 def all_class_attributes(self): 392 return {} 393 394 def instance_attributes(self): 395 return {} 396 397 def __repr__(self): 398 return "UnresolvedName(%r, %r)" % (self.name, self.parent_name) 399 400 class Module(NamespaceDict): 401 402 "An inspected module's core details." 403 404 def __init__(self, name): 405 NamespaceDict.__init__(self, self) 406 self.name = name 407 408 # Module attributes. 409 410 self.modattr = None # cache for module_attributes 411 self.modattr_names = None # from modattr 412 413 # Complete lists of classes and functions. 414 415 self.all_objects = [] 416 417 # Constant records. 418 419 self.constant_values = {} 420 self.constant_list = None # cache for constants 421 422 # Image generation details. 423 424 self.location = None 425 self.code_location = None 426 427 # Original location details. 428 429 self.node = None 430 431 def full_name(self): 432 return self.name 433 434 def __repr__(self): 435 return "Module(%r, location=%r)" % (self.name, self.location) 436 437 # Attribute methods. 438 439 def module_attribute_names(self): 440 441 "Return the module attribute names provided by the module." 442 443 if self.modattr_names is None: 444 self.module_attributes() 445 return self.modattr_names 446 447 def module_attributes(self): 448 449 "Return a dictionary mapping names to module attributes." 450 451 if self.modattr is None: 452 self.modattr = {} 453 self.modattr_names = self.keys() 454 for i, name in enumerate(self.modattr_names): 455 self.modattr[name] = Attr(i, self, self.assignments[name], self[name]) 456 457 return self.modattr 458 459 def constants(self): 460 461 "Return a list of constants." 462 463 if self.constant_list is None: 464 self.constant_list = list(self.constant_values.values()) 465 466 return self.constant_list 467 468 # Program visitors. 469 470 class InspectedModule(ASTVisitor, Module): 471 472 """ 473 An inspected module, providing core details via the Module superclass, but 474 capable of being used as an AST visitor. 475 """ 476 477 def __init__(self, name, importer=None): 478 ASTVisitor.__init__(self) 479 Module.__init__(self, name) 480 self.visitor = self 481 482 self.importer = importer 483 self.loaded = 0 484 485 # Current expression state. 486 487 self.expr = None 488 489 # Namespace state. 490 491 self.in_init = 0 492 self.namespaces = [] 493 self.module = None 494 495 def parse(self, filename): 496 497 "Parse the file having the given 'filename'." 498 499 module = compiler.parseFile(filename) 500 self.process(module) 501 502 def process(self, module): 503 504 "Process the given 'module'." 505 506 self.node = self.module = module 507 processed = self.dispatch(module) 508 if self.has_key("__all__"): 509 all = self["__all__"] 510 if isinstance(all, compiler.ast.List): 511 for n in all.nodes: 512 self[n.value] = self.importer.add_module(self.name + "." + n.value) 513 return processed 514 515 def vacuum(self): 516 517 "Vacuum the module namespace, removing unloaded module references." 518 519 for name, value in self.items(): 520 if isinstance(value, Module) and not value.loaded: 521 del self[name] 522 523 # Complain about globals not initialised at the module level. 524 525 if isinstance(value, Global): 526 print "Warning: global %r in module %r not initialised at the module level." % (name, self.name) 527 528 # Namespace methods. 529 530 def store(self, name, obj): 531 532 "Record attribute or local 'name', storing 'obj'." 533 534 if not self.namespaces: 535 self[name] = obj 536 else: 537 self.namespaces[-1][name] = obj 538 539 # Record all non-local objects. 540 541 if not (self.namespaces and isinstance(self.namespaces[-1], Function)): 542 self.all_objects.append(obj) 543 544 def store_attr(self, name): 545 546 "Record instance attribute 'name' in the current class." 547 548 if self.in_init: 549 550 # Current namespace is the function. 551 # Previous namespace is the class. 552 553 self.namespaces[-2].add_instance_attribute(name) 554 555 def get_parent(self): 556 return (self.namespaces[-1:] or [self])[0] 557 558 # Visitor methods. 559 560 def default(self, node, *args): 561 raise InspectError, node.__class__ 562 563 def dispatch(self, node, *args): 564 return ASTVisitor.dispatch(self, node, *args) 565 566 def NOP(self, node): 567 for n in node.getChildNodes(): 568 self.dispatch(n) 569 return None 570 571 visitAdd = NOP 572 573 visitAnd = NOP 574 575 visitAssert = NOP 576 577 def visitAssign(self, node): 578 self.expr = self.dispatch(node.expr) 579 for n in node.nodes: 580 self.dispatch(n) 581 return None 582 583 def visitAssAttr(self, node): 584 expr = self.dispatch(node.expr) 585 if expr is not None and isinstance(expr, Self): 586 self.store_attr(node.attrname) 587 return None 588 589 def visitAssList(self, node): 590 for n in node.nodes: 591 self.dispatch(n) 592 return None 593 594 def visitAssName(self, node): 595 self.store(node.name, self.expr) 596 return None 597 598 visitAssTuple = visitAssList 599 600 visitAugAssign = NOP 601 602 visitBackquote = NOP 603 604 visitBitand = NOP 605 606 visitBitor = NOP 607 608 visitBitxor = NOP 609 610 visitBreak = NOP 611 612 visitCallFunc = NOP 613 614 def visitClass(self, node): 615 if self.namespaces: 616 print "Class %r in %r is not global: ignored." % (node.name, self) 617 else: 618 cls = Class(node.name, self.get_parent().full_name(), self, node) 619 for base in node.bases: 620 base_ref = self.dispatch(base) 621 if base_ref is None: 622 raise InspectError, "Base class %r for class %r in %r is not found: it may be hidden in some way." % (base, self, cls) 623 cls.add_base(base_ref) 624 625 # Make a back reference from the node for code generation. 626 627 node.cls = cls 628 629 # Make an entry for the class. 630 631 self.store(node.name, cls) 632 633 self.namespaces.append(cls) 634 self.dispatch(node.code) 635 self.namespaces.pop() 636 637 return None 638 639 visitCompare = NOP 640 641 def visitConst(self, node): 642 const = Const(node.value) 643 self.constant_values[node.value] = const 644 return const 645 646 visitContinue = NOP 647 648 visitDecorators = NOP 649 650 visitDict = NOP 651 652 visitDiscard = NOP 653 654 visitDiv = NOP 655 656 visitEllipsis = NOP 657 658 visitExec = NOP 659 660 visitExpression = NOP 661 662 visitFloorDiv = NOP 663 664 visitFor = NOP 665 666 def visitFrom(self, node): 667 if self.importer is None: 668 raise InspectError, "Please use the micropython.Importer class for code which uses the 'from' statement." 669 670 module = self.importer.load(node.modname, 1) 671 672 if module is None: 673 print "Warning:", node.modname, "not imported." 674 675 for name, alias in node.names: 676 if name != "*": 677 if module is not None and module.namespace.has_key(name): 678 attr = module[name] 679 self.store(alias or name, attr) 680 if isinstance(attr, Module) and not attr.loaded: 681 self.importer.load(attr.name) 682 683 # Support the import of names from missing modules. 684 685 else: 686 self.store(alias or name, UnresolvedName(name, node.modname, self)) 687 else: 688 if module is not None: 689 for n in module.namespace.keys(): 690 attr = module[n] 691 self.store(n, attr) 692 if isinstance(attr, Module) and not attr.loaded: 693 self.importer.load(attr.name) 694 695 return None 696 697 def visitFunction(self, node): 698 function = Function( 699 node.name, 700 self.get_parent().full_name(), 701 node.argnames, 702 (node.flags & 4 != 0), 703 (node.flags & 8 != 0), 704 self, 705 node 706 ) 707 708 # Make a back reference from the node for code generation. 709 710 node.function = function 711 712 self.namespaces.append(function) 713 714 # Current namespace is the function. 715 # Previous namespace is the class. 716 717 if node.name == "__init__" and isinstance(self.namespaces[-2], Class): 718 self.in_init = 1 719 720 self.dispatch(node.code) 721 self.in_init = 0 722 self.namespaces.pop() 723 724 self.store(node.name, function) 725 return None 726 727 visitGenExpr = NOP 728 729 visitGenExprFor = NOP 730 731 visitGenExprIf = NOP 732 733 visitGenExprInner = NOP 734 735 def visitGetattr(self, node): 736 expr = self.dispatch(node.expr) 737 if expr is not None: 738 if isinstance(expr, Module): 739 return expr.namespace.get(node.attrname) 740 elif isinstance(expr, UnresolvedName): 741 return UnresolvedName(node.attrname, expr.full_name(), self) 742 return builtins.get(node.attrname) 743 744 def visitGlobal(self, node): 745 if self.namespaces: 746 for name in node.names: 747 self.namespaces[-1].make_global(name) 748 if not self.has_key(name): 749 self[name] = Global() 750 751 def visitIf(self, node): 752 for test, body in node.tests: 753 self.dispatch(body) 754 if node.else_ is not None: 755 self.dispatch(node.else_) 756 return None 757 758 visitIfExp = NOP 759 760 def visitImport(self, node): 761 if self.importer is None: 762 raise InspectError, "Please use the micropython.Importer class for code which uses the 'import' statement." 763 764 for name, alias in node.names: 765 if alias is not None: 766 self.store(alias, self.importer.load(name, 1) or UnresolvedName(None, name, self)) 767 else: 768 self.store(name.split(".")[0], self.importer.load(name) or UnresolvedName(None, name.split(".")[0], self)) 769 770 return None 771 772 visitInvert = NOP 773 774 visitKeyword = NOP 775 776 visitLambda = NOP 777 778 visitLeftShift = NOP 779 780 visitList = NOP 781 782 visitListComp = NOP 783 784 visitListCompFor = NOP 785 786 visitListCompIf = NOP 787 788 visitMod = NOP 789 790 def visitModule(self, node): 791 return self.dispatch(node.node) 792 793 visitMul = NOP 794 795 def visitName(self, node): 796 name = node.name 797 if name == "self": 798 return Self() 799 elif self.has_key(name): 800 return self[name] 801 elif builtins.has_key(name): 802 return builtins[name] 803 else: 804 return None 805 806 visitNot = NOP 807 808 visitOr = NOP 809 810 visitPass = NOP 811 812 visitPower = NOP 813 814 visitPrint = NOP 815 816 visitPrintnl = NOP 817 818 visitRaise = NOP 819 820 visitReturn = NOP 821 822 visitRightShift = NOP 823 824 visitSlice = NOP 825 826 visitSliceobj = NOP 827 828 def visitStmt(self, node): 829 for n in node.nodes: 830 self.dispatch(n) 831 return None 832 833 visitSub = NOP 834 835 visitSubscript = NOP 836 837 def visitTryExcept(self, node): 838 self.dispatch(node.body) 839 for name, var, n in node.handlers: 840 self.dispatch(n) 841 if node.else_ is not None: 842 self.dispatch(node.else_) 843 return None 844 845 visitTryFinally = NOP 846 847 visitTuple = NOP 848 849 visitUnaryAdd = NOP 850 851 visitUnarySub = NOP 852 853 visitWhile = NOP 854 855 visitWith = NOP 856 857 visitYield = NOP 858 859 class Self: 860 861 "A reference to an object within a method." 862 863 pass 864 865 class Global: 866 867 """ 868 A reference to an object assigned to a global from outside the module 869 top-level. 870 """ 871 872 pass 873 874 # Built-in types initialisation. 875 876 class Builtins(Module): 877 878 "The special built-in types module." 879 880 def __init__(self): 881 Module.__init__(self, "__builtins__") 882 883 for key in ['ArithmeticError', 'AssertionError', 'AttributeError', 884 'BaseException', 'DeprecationWarning', 'EOFError', 'Ellipsis', 885 'EnvironmentError', 'Exception', 'False', 'FloatingPointError', 886 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 887 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', 888 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', 889 'None', 'NotImplemented', 'NotImplementedError', 'OSError', 890 'OverflowError', 'PendingDeprecationWarning', 'ReferenceError', 891 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration', 892 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 893 'TabError', 'True', 'TypeError', 'UnboundLocalError', 894 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 895 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', 896 'ValueError', 'Warning', 'ZeroDivisionError', 897 'basestring', 'bool', 'buffer', 'complex', 'dict', 'file', 'float', 898 'frozenset', 'int', 'list', 'long', 'object', 'set', 'slice', 'str', 899 'tuple', 'type', 'unicode', 'xrange']: 900 self[key] = Class(key, self.full_name(), self) 901 902 # NOTE: Incomplete: some functions have more than one parameter. 903 904 for key in ['__import__', 'abs', 'all', 'any', 'callable', 'chr', 905 'classmethod', 'cmp', 'compile', 'delattr', 'dir', 'divmod', 906 'enumerate', 'eval', 'execfile', 'filter', 'getattr', 'globals', 907 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'isinstance', 908 'issubclass', 'iter', 'len', 'locals', 'map', 'max', 'min', 'oct', 909 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce', 910 'reload', 'repr', 'reversed', 'round', 'setattr', 'sorted', 911 'staticmethod', 'sum', 'super', 'unichr', 'vars', 'zip']: 912 self[key] = Function(key, self.full_name(), ['arg'], 0, 0, self) 913 914 builtins = Builtins() 915 916 # vim: tabstop=4 expandtab shiftwidth=4