1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed. Here, the standard path 26 for module searching is employed: 27 28 importer = Importer(sys.path) 29 30 To generate programs, the above importer should be supplied in the 31 initialisation of a program instance, and then various methods are called: 32 33 program = Program(importer) 34 image = program.get_raw_image() 35 36 Such importer and program objects are the most convenient mechanism through 37 which the functionality of the micropython package may be accessed. 38 """ 39 40 from micropython.common import ObjectSet, ProcessingError, TableError, \ 41 TableGenerationError 42 import micropython.ast 43 import micropython.data 44 import micropython.opt 45 import micropython.inspect 46 import micropython.table 47 import os 48 49 try: 50 set 51 except NameError: 52 from sets import Set as set 53 54 class Program: 55 56 "This class supports the generation of a program image." 57 58 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 59 60 def __init__(self, importer, optimisations=None): 61 62 """ 63 Initialise the program representation with an 'importer' which is able 64 to locate and load Python modules. 65 66 The optional 'optimisations' cause certain techniques to be used in 67 reducing program size and improving program efficiency. 68 """ 69 70 self.importer = importer 71 self.optimisations = optimisations or set() 72 73 # Remember the tables once generated. 74 75 self.objtable = None 76 self.paramtable = None 77 78 # Main program information. 79 80 self.code = None 81 self.code_location = None 82 83 def get_importer(self): 84 return self.importer 85 86 # Access to finalised program information. 87 88 def finalise(self): 89 90 "Finalise the program." 91 92 # Need the tables to finalise. 93 94 objtable = self.get_object_table() 95 self.get_parameter_table() 96 97 self.importer.vacuum(objtable) 98 self.importer.finalise() 99 100 # Now remove unneeded things from the tables. 101 102 self.get_object_table(reset=1) 103 self.get_parameter_table(reset=1) 104 105 def get_image(self, with_builtins=0): 106 107 """ 108 Return the program image including built-in objects if 'with_builtins' 109 is specified and set to a true value. 110 """ 111 112 if self.code is not None: 113 return self.code 114 115 # Optimise and regenerate the object table. 116 117 self.finalise() 118 self.code = [] 119 120 # Append constants to the image. 121 122 for const in self.importer.constants(): 123 self.code.append(const) 124 125 last_module = self.importer.modules_ordered[-1] 126 127 for module in self.importer.modules_ordered: 128 suppress_builtins = not with_builtins and module.name == "__builtins__" 129 130 # Position the module in the image and make a translation. 131 132 trans = micropython.ast.Translation(module, self) 133 134 # Add header details. 135 136 self.code.append(module) 137 138 # Append module attributes to the image. 139 140 attributes = module.module_attributes() 141 self.code += module.attributes_as_list() 142 143 # Append classes and functions to the image. 144 145 for obj in module.all_objects: 146 if isinstance(obj, micropython.data.Class): 147 148 # Add header details. 149 150 self.code.append(obj) 151 152 # Append class attributes to the image. 153 154 attributes = obj.class_attributes() 155 self.code += obj.attributes_as_list() 156 157 # Omit built-in function code where requested. 158 159 if suppress_builtins and obj.astnode.doc is None: 160 continue 161 162 # Generate the instantiator/initialiser. 163 # Append the function code to the image. 164 165 code = trans.get_instantiator_code(obj) 166 self.code += code 167 168 # Class-level code is generated separately at the module 169 # level, and the code location is set within the code 170 # generation process for the module. 171 172 elif isinstance(obj, micropython.data.Function): 173 174 # Add header details. 175 176 self.code.append(obj) 177 178 # Append any default values to the image. 179 # Only do this for functions which are not dynamic. 180 181 if not obj.is_dynamic(): 182 self.code += obj.default_attrs 183 184 # Omit built-in function code where requested. 185 186 if suppress_builtins and obj.astnode.doc is None: 187 pass 188 189 # Append the function code to the image. 190 191 else: 192 code = trans.get_code(obj) 193 self.code += code 194 195 # Omit built-in module code where requested. 196 197 if suppress_builtins: 198 pass 199 200 # Append the module top-level code to the image. 201 202 else: 203 code = trans.get_module_code() 204 self.code += code 205 206 return self.code 207 208 def get_raw_image(self, architecture=None, with_builtins=0): 209 210 "Return the raw image representation of the program." 211 212 architecture = architecture or micropython.rsvp 213 214 self.get_image(with_builtins) 215 216 objtable = self.get_object_table() 217 paramtable = self.get_parameter_table() 218 219 # Position the objects. 220 221 pos = 0 222 223 for item in self.code: 224 arch_item = architecture.get_object(item) 225 226 # Get the raw version for the architecture. 227 228 if arch_item is not None: 229 pos = arch_item.set_location(pos, objtable, with_builtins) 230 else: 231 pos += 1 232 233 # Generate the raw code. 234 235 self.raw_code = [] 236 237 for item in self.code: 238 arch_item = architecture.get_object(item) 239 240 # Get the raw version for the architecture. 241 242 if arch_item is not None: 243 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 244 arch_item.finalise_location(with_builtins) 245 else: 246 self.raw_code.append(item) 247 248 # Fix the module locations. 249 250 for module in self.importer.modules_ordered: 251 252 if not with_builtins and module.name == "__builtins__": 253 continue 254 255 module.code_location = module.blocks[0].location 256 257 self.code_location = self.importer.modules["__main__"].code_location 258 return self.raw_code 259 260 def get_object_table(self, reset=0): 261 262 "Return a table with details of attributes for classes and modules." 263 264 if self.objtable is None or reset: 265 266 t = self.objtable = micropython.table.ObjectTable() 267 for module in self.importer.get_modules(): 268 t.add(module.full_name(), module.module_attributes()) 269 270 # Add class and instance attributes for all classes, together 271 # with descendant information. 272 273 for obj in module.all_objects: 274 if isinstance(obj, micropython.data.Class): 275 276 # Prevent ambiguous classes. 277 278 full_name = obj.full_name() 279 name = obj.name 280 281 #if module.has_key(name) and module[name].defines_ambiguous_class(): 282 # raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 283 284 # Define a table entry for the class. 285 286 attributes = {full_name : obj} 287 attributes.update(obj.all_attributes()) 288 attributes.update(obj.all_descendants()) 289 t.add(full_name, attributes) 290 291 return self.objtable 292 293 def get_parameter_table(self, reset=0): 294 295 "Return a table with details of parameters for functions and methods." 296 297 # Need the object table to get at class details. 298 299 if self.paramtable is None or reset: 300 t = self.paramtable = micropython.table.ParameterTable() 301 302 # Visit each module, getting function and method details. 303 304 for module in self.importer.get_modules(): 305 for obj in module.all_objects: 306 if isinstance(obj, micropython.data.Function): 307 t.add(obj.full_name(), obj.parameters()) 308 309 # Classes are callable, too. 310 # Take details of the appropriate __init__ method to make an 311 # entry for an instantiation function for the class. 312 313 elif isinstance(obj, micropython.data.Class): 314 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 315 316 # Filter out all parameter table entries not referenced by keyword 317 # arguments. 318 319 keyword_names = set() 320 321 for module in self.importer.get_modules(): 322 keyword_names.update(module.keyword_names) 323 324 for function_name, parameters in t.table.items(): 325 for name in parameters.keys(): 326 if name in keyword_names: 327 break 328 else: 329 del t.table[function_name] 330 331 return self.paramtable 332 333 class Importer: 334 335 "An import machine, searching for and loading modules." 336 337 predefined_constants = { 338 "None" : None, 339 "True" : True, 340 "False" : False, 341 #"Ellipsis" : Ellipsis, 342 "NotImplemented" : NotImplemented 343 } 344 345 names_always_used = [ 346 "bool", "__call__", "__bool__" 347 ] 348 349 def __init__(self, path=None, verbose=0, optimisations=None): 350 351 """ 352 Initialise the importer with the given search 'path' - a list of 353 directories to search for Python modules. 354 355 The optional 'verbose' parameter causes output concerning the activities 356 of the object to be produced if set to a true value (not the default). 357 358 The optional 'optimisations' cause certain techniques to be used in 359 reducing program size and improving program efficiency. 360 """ 361 362 self.path = path or [os.getcwd()] 363 self.verbose = verbose 364 self.optimisations = optimisations or set() 365 366 self.modules = {} 367 self.modules_ordered = [] 368 self.loading = set() 369 370 # Constant records. 371 372 self.constant_values = {} 373 self.constant_list = None # cache for constants 374 self.init_predefined_constants() 375 376 # Attribute usage. 377 378 self.attributes_used = set() 379 self.name_references = {} 380 self.specific_name_references = {} 381 382 # Attribute coverage calculated during collection. 383 384 self.inferred_name_references = {} 385 386 # Attribute coverage status during collection. 387 388 self.attribute_users_visited = set() 389 self.attributes_to_visit = {} 390 391 # Status information. 392 393 self.vacuumed = 0 394 self.finalised = 0 395 396 def get_modules(self): 397 398 "Return all modules known to the importer." 399 400 return self.modules.values() 401 402 def get_module(self, name): 403 404 "Return the module with the given 'name'." 405 406 return self.modules[name] 407 408 # General maintenance. 409 410 def vacuum(self, objtable): 411 412 "Tidy up the modules." 413 414 if self.vacuumed: 415 return 416 417 # Complete the list of attribute names used in the program. 418 419 self.collect_attributes(objtable) 420 421 for name, module in self.modules.items(): 422 if module.loaded: 423 module.vacuum() 424 else: 425 del self.modules[name] 426 427 self.vacuumed = 1 428 429 def finalise(self): 430 431 "Finalise the program (which should have been vacuumed first)." 432 433 if self.finalised: 434 return 435 436 # Reset any previously compiled information. 437 438 for module in self.get_modules(): 439 module.unfinalise() 440 441 # Prepare module information again. 442 443 for module in self.get_modules(): 444 module.finalise() 445 446 self.finalised = 1 447 448 # Name accounting. 449 450 def use_name(self, name, from_name, value=None): 451 452 """ 453 Register the given 'name' as being used in the program from within an 454 object with the specified 'from_name'. If the optional 'value' is given, 455 note an assignment. 456 """ 457 458 if not self.name_references.has_key(from_name): 459 self.name_references[from_name] = set() 460 461 attrnames = ObjectSet([name]) 462 usage = (attrnames,) 463 self.name_references[from_name].add((None, None, usage)) 464 465 def use_names(self, user, name, usage, from_name): 466 467 """ 468 For the given attribute 'user' (which may be None if no specific user is 469 given), register for the given 'name' the given attribute 'usage' 470 (combinations of attribute names), noting the scope of this usage as 471 being the program object with the specified 'from_name'. 472 """ 473 474 if not self.name_references.has_key(from_name): 475 self.name_references[from_name] = set() 476 477 self.name_references[from_name].add((user, name, usage)) 478 479 def use_specific_name(self, objname, attrname, from_name): 480 481 """ 482 Register the given 'objname' (for an object) whose 'attrname' is being 483 used in the program from within an object with the specified 484 'from_name'. 485 """ 486 487 if not self.specific_name_references.has_key(from_name): 488 self.specific_name_references[from_name] = set() 489 self.specific_name_references[from_name].add((objname, attrname)) 490 491 # Name accounting products. 492 493 def uses_attribute(self, objname, name): 494 495 """ 496 Return whether the attribute of the object with the given 'objname' 497 having the given 'name' is used as an attribute in the program. 498 """ 499 500 return (objname + "." + name) in self.attributes_used 501 502 def use_attribute(self, objname, name): 503 504 """ 505 Indicate that in the object with the given 'objname', the attribute of 506 the given 'name' is used. 507 """ 508 509 self.attributes_used.add(objname + "." + name) 510 511 def use_object(self, objname): 512 513 "Indicate that the object with the given 'objname' is used." 514 515 self.attributes_used.add(objname) 516 517 def collect_attributes(self, objtable): 518 519 "Collect attribute references for the entire program." 520 521 # Include names which may not be explicitly used in programs. 522 # NOTE: Potentially declare these when inspecting. 523 524 for attrname in self.names_always_used: 525 for objname in objtable.all_possible_objects([attrname]): 526 527 # Record attributes of objects for potential visiting. 528 529 self.add_attribute_to_visit(objname, attrname) 530 531 # Visit all modules, since some may employ initialisation code which has 532 # some kind of side-effect. 533 534 for name in self.modules.keys(): 535 self._collect_attributes(name, objtable) 536 537 def add_attribute_to_visit(self, objname, attrname): 538 539 """ 540 Queue an attribute of the object with the given 'objname', having the 541 given 'attrname', to the list for potential visiting if the specified 542 object is actually referenced. 543 """ 544 545 if not self.attributes_to_visit.has_key(objname): 546 self.attributes_to_visit[objname] = set() 547 self.attributes_to_visit[objname].add(attrname) 548 549 def _collect_attributes_from(self, from_name, objname, attrname, objtable): 550 551 """ 552 Record the association between 'from_name' and the attribute of 553 'objname' with the given 'attrname'. Then collect attributes for the 554 referenced attribute using 'objtable'. 555 """ 556 557 if not self.inferred_name_references.has_key(from_name): 558 self.inferred_name_references[from_name] = set() 559 560 self.inferred_name_references[from_name].add((objname, attrname)) 561 self._collect_attributes(objname + "." + attrname, objtable) 562 563 def _collect_attributes(self, from_name, objtable): 564 565 """ 566 Given an object called 'from_name', find all names referenced from such 567 an object according to the register of names, using 'objtable' to infer 568 types. 569 """ 570 571 if from_name in self.attribute_users_visited: 572 return 573 574 self.attribute_users_visited.add(from_name) 575 576 # Get name references and find possible objects which support such 577 # combinations of attribute names. 578 579 for user, name, usage in self.name_references.get(from_name, []): 580 581 # Using all attribute names for a particular name, attempt to get 582 # specific object types. 583 584 all_objtypes = set() 585 586 for attrnames in usage: 587 objtypes = objtable.all_possible_objects_plus_status(attrnames) 588 if not objtypes: 589 print "Warning: usage in %r for %r finds no object supporting all attributes %r" % (from_name, name, attrnames) 590 objtypes = objtable.any_possible_objects_plus_status(attrnames) 591 if not objtypes: 592 print "Warning: usage in %r for %r finds no object supporting any attributes %r" % (from_name, name, attrnames) 593 594 all_objtypes.update(objtypes) 595 596 # Record the object types for generating guards. 597 598 if user is not None: 599 if not hasattr(user, "_attrtypes"): 600 user._attrtypes = {} 601 602 user._attrtypes[name] = all_objtypes 603 604 # For each suggested object type, consider each attribute given by 605 # the names. 606 607 for objname, is_static in all_objtypes: 608 for attrnames in usage: 609 for attrname, attrvalues in attrnames.items(): 610 611 # Test for the presence of an attribute on the suggested 612 # object type. 613 614 try: 615 attr = objtable.access(objname, attrname) 616 except TableError: 617 #print "Warning: object type %r does not support attribute %r" % (objname, attrname) 618 continue 619 620 # Get the real identity of the attribute in order to 621 # properly collect usage from it. 622 623 parent = attr.parent 624 if isinstance(parent, micropython.data.Instance): 625 parentname = objname 626 else: 627 parentname = parent.full_name() 628 629 # Test for assignment. 630 631 if attrvalues: 632 for attrvalue in attrvalues: 633 parent.set(attrname, attrvalue, 0) 634 635 # Visit attributes of objects known to be used. 636 637 if parentname in self.attributes_used: 638 self.use_attribute(parentname, attrname) 639 self._collect_attributes_from(from_name, parentname, attrname, objtable) 640 641 # Record attributes of other objects for potential visiting. 642 643 else: 644 self.add_attribute_to_visit(parentname, attrname) 645 646 # Get specific name references and visit the referenced objects. 647 648 for objname, attrname in self.specific_name_references.get(from_name, []): 649 self.use_attribute(objname, attrname) 650 self._collect_attributes_from(from_name, objname, attrname, objtable) 651 652 # Where the object has an __init__ attribute, assume that it is an 653 # initialiser which is called at some point, and collect attributes used 654 # in this initialiser. 655 656 if "__init__" in objtable.table.get(from_name, []): 657 self.use_attribute(from_name, "__init__") 658 self._collect_attributes_from(from_name, from_name, "__init__", objtable) 659 660 # Visit attributes on this object that were queued in case of the object 661 # being referenced. 662 663 attributes_to_visit = self.attributes_to_visit.get(from_name, []) 664 665 if attributes_to_visit: 666 del self.attributes_to_visit[from_name] 667 668 for attrname in attributes_to_visit: 669 self.use_attribute(from_name, attrname) 670 self._collect_attributes_from(from_name, from_name, attrname, objtable) 671 672 # Constant accounting. 673 674 def init_predefined_constants(self): 675 676 "Ensure the predefined constants." 677 678 for name, value in self.predefined_constants.items(): 679 self.make_constant(value) 680 681 def get_predefined_constant(self, name): 682 683 "Return the predefined constant for the given 'name'." 684 685 return self.make_constant(self.predefined_constants[name]) 686 687 def get_constant(self, value): 688 689 "Return a constant for the given 'value'." 690 691 const = micropython.data.Const(value) 692 return self.constant_values[const] 693 694 def get_constant_type_name(self, value): 695 696 "Return the type name for the given constant 'value'." 697 698 return value.__class__.__name__ 699 700 def make_constant(self, value): 701 702 "Make and return a constant for the given 'value'." 703 704 # Make a constant object and return it. 705 706 const = micropython.data.Const(value) 707 if not self.constant_values.has_key(const): 708 self.constant_values[const] = const 709 return self.constant_values[const] 710 711 def constants(self): 712 713 "Return a list of constants." 714 715 if self.constant_list is None: 716 self.constant_list = list(self.constant_values.values()) 717 718 return self.constant_list 719 720 # Import methods. 721 722 def find_in_path(self, name): 723 724 """ 725 Find the given module 'name' in the search path, returning None where no 726 such module could be found, or a 2-tuple from the 'find' method 727 otherwise. 728 """ 729 730 for d in self.path: 731 m = self.find(d, name) 732 if m: return m 733 return None 734 735 def find(self, d, name): 736 737 """ 738 In the directory 'd', find the given module 'name', where 'name' can 739 either refer to a single file module or to a package. Return None if the 740 'name' cannot be associated with either a file or a package directory, 741 or a 2-tuple from '_find_package' or '_find_module' otherwise. 742 """ 743 744 m = self._find_package(d, name) 745 if m: return m 746 m = self._find_module(d, name) 747 if m: return m 748 return None 749 750 def _find_module(self, d, name): 751 752 """ 753 In the directory 'd', find the given module 'name', returning None where 754 no suitable file exists in the directory, or a 2-tuple consisting of 755 None (indicating that no package directory is involved) and a filename 756 indicating the location of the module. 757 """ 758 759 name_py = name + os.extsep + "py" 760 filename = self._find_file(d, name_py) 761 if filename: 762 return None, filename 763 return None 764 765 def _find_package(self, d, name): 766 767 """ 768 In the directory 'd', find the given package 'name', returning None 769 where no suitable package directory exists, or a 2-tuple consisting of 770 a directory (indicating the location of the package directory itself) 771 and a filename indicating the location of the __init__.py module which 772 declares the package's top-level contents. 773 """ 774 775 filename = self._find_file(d, name) 776 if filename: 777 init_py = "__init__" + os.path.extsep + "py" 778 init_py_filename = self._find_file(filename, init_py) 779 if init_py_filename: 780 return filename, init_py_filename 781 return None 782 783 def _find_file(self, d, filename): 784 785 """ 786 Return the filename obtained when searching the directory 'd' for the 787 given 'filename', or None if no actual file exists for the filename. 788 """ 789 790 filename = os.path.join(d, filename) 791 if os.path.exists(filename): 792 return filename 793 else: 794 return None 795 796 def load(self, name, return_leaf=0): 797 798 """ 799 Load the module or package with the given 'name'. Return an object 800 referencing the loaded module or package, or None if no such module or 801 package exists. 802 """ 803 804 if return_leaf: 805 name_for_return = name 806 else: 807 name_for_return = name.split(".")[0] 808 809 if self.modules.has_key(name) and self.modules[name].loaded: 810 #print "Cached (%s)" % name 811 return self.modules[name_for_return] 812 813 if self.verbose: 814 print "Loading", name 815 816 # Split the name into path components, and try to find the uppermost in 817 # the search path. 818 819 path = name.split(".") 820 m = self.find_in_path(path[0]) 821 if not m: 822 if self.verbose: 823 print "Not found (%s)" % path[0] 824 return None # NOTE: Import error. 825 d, filename = m 826 827 # Either acquire a reference to an already-imported module, or load the 828 # module from a file. 829 830 top = module = self.load_from_file(filename, path[0]) 831 832 # For hierarchical names, traverse each path component... 833 834 if len(path) > 1: 835 if not d: 836 if self.verbose: 837 print "No package (%s)" % filename 838 return None # NOTE: Import error (package not found). 839 else: 840 self.add_submodules(d, module) 841 842 path_so_far = path[:1] 843 for p in path[1:]: 844 path_so_far.append(p) 845 846 # Find the package or module concerned. 847 848 m = self.find(d, p) 849 if not m: 850 if self.verbose: 851 print "Not found (%s)" % p 852 return None # NOTE: Import error. 853 d, filename = m 854 module_name = ".".join(path_so_far) 855 856 # Either reference an imported module or load one from a file. 857 858 submodule = self.load_from_file(filename, module_name) 859 860 if d: 861 self.add_submodules(d, module) 862 863 # Store the submodule within its parent module. 864 865 module.set_module(p, submodule) 866 module = submodule 867 868 # Return either the deepest or the uppermost module. 869 870 if return_leaf: 871 return module 872 else: 873 return top 874 875 def load_from_file(self, name, module_name=None): 876 877 """ 878 Load the module with the given 'name' (which may be a full module path). 879 """ 880 881 if module_name is None: 882 module_name = "__main__" 883 884 module = self.add_module(module_name) 885 if not module.loaded and module not in self.loading: 886 self.loading.add(module) 887 #print "Parsing", name 888 module.parse(name) 889 #print "Done", name 890 self.loading.remove(module) 891 module.loaded = 1 892 893 # Record the module. 894 895 self.use_object(module.full_name()) 896 #print "Loaded", module_name, "with namespace", module.namespace.keys() 897 return module 898 899 def add_module(self, module_name): 900 901 """ 902 Return the module with the given 'module_name', adding a new module 903 object if one does not already exist. 904 """ 905 906 if not self.modules.has_key(module_name): 907 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 908 self.modules_ordered.append(module) 909 else: 910 module = self.modules[module_name] 911 return module 912 913 def add_submodules(self, pathname, module): 914 915 """ 916 Work around insufficient __all__ declarations and examine the directory 917 with the given 'pathname', adding submodules to the given 'module'. 918 """ 919 920 for filename in os.listdir(pathname): 921 submodule, ext = os.path.splitext(filename) 922 if ext not in ("", ".py"): 923 continue 924 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 925 926 # vim: tabstop=4 expandtab shiftwidth=4