1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed. Here, the standard path 26 for module searching is employed: 27 28 importer = Importer(sys.path) 29 30 To generate programs, the above importer should be supplied in the 31 initialisation of a program instance, and then various methods are called: 32 33 program = Program(importer) 34 image = program.get_raw_image() 35 36 Such importer and program objects are the most convenient mechanism through 37 which the functionality of the micropython package may be accessed. 38 """ 39 40 from micropython.common import ProcessingError, TableGenerationError 41 import micropython.ast 42 import micropython.data 43 import micropython.opt 44 import micropython.inspect 45 import micropython.table 46 import os 47 48 try: 49 set 50 except NameError: 51 from sets import Set as set 52 53 class Program: 54 55 "This class supports the generation of a program image." 56 57 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 58 59 def __init__(self, importer, optimisations=None): 60 61 """ 62 Initialise the program representation with an 'importer' which is able 63 to locate and load Python modules. 64 65 The optional 'optimisations' cause certain techniques to be used in 66 reducing program size and improving program efficiency. 67 """ 68 69 self.importer = importer 70 self.optimisations = optimisations or set() 71 72 # Remember the tables once generated. 73 74 self.objtable = None 75 self.paramtable = None 76 77 # Main program information. 78 79 self.code = None 80 self.code_location = None 81 82 def get_importer(self): 83 return self.importer 84 85 # Access to finalised program information. 86 87 def finalise(self): 88 89 "Finalise the program." 90 91 # Need the tables to finalise. 92 93 objtable = self.get_object_table() 94 self.get_parameter_table() 95 96 self.importer.vacuum(objtable) 97 self.importer.finalise() 98 99 # Now remove unneeded things from the tables. 100 101 self.get_object_table(reset=1) 102 self.get_parameter_table(reset=1) 103 104 def get_image(self, with_builtins=0): 105 106 """ 107 Return the program image including built-in objects if 'with_builtins' 108 is specified and set to a true value. 109 """ 110 111 if self.code is not None: 112 return self.code 113 114 # Optimise and regenerate the object table. 115 116 self.finalise() 117 self.code = [] 118 119 # Append constants to the image. 120 121 for const in self.importer.constants(): 122 self.code.append(const) 123 124 last_module = self.importer.modules_ordered[-1] 125 126 for module in self.importer.modules_ordered: 127 suppress_builtins = not with_builtins and module.name == "__builtins__" 128 129 # Position the module in the image and make a translation. 130 131 trans = micropython.ast.Translation(module, self) 132 133 # Add header details. 134 135 self.code.append(module) 136 137 # Append module attributes to the image. 138 139 attributes = module.module_attributes() 140 self.code += module.attributes_as_list() 141 142 # Append classes and functions to the image. 143 144 for obj in module.all_objects: 145 if isinstance(obj, micropython.data.Class): 146 147 # Add header details. 148 149 self.code.append(obj) 150 151 # Append class attributes to the image. 152 153 attributes = obj.class_attributes() 154 self.code += obj.attributes_as_list() 155 156 # Omit built-in function code where requested. 157 158 if suppress_builtins and obj.astnode.doc is None: 159 continue 160 161 # Generate the instantiator/initialiser. 162 # Append the function code to the image. 163 164 code = trans.get_instantiator_code(obj) 165 self.code += code 166 167 # Class-level code is generated separately at the module 168 # level, and the code location is set within the code 169 # generation process for the module. 170 171 elif isinstance(obj, micropython.data.Function): 172 173 # Add header details. 174 175 self.code.append(obj) 176 177 # Append any default values to the image. 178 # Only do this for functions which are not dynamic. 179 180 if not obj.is_dynamic(): 181 self.code += obj.default_attrs 182 183 # Omit built-in function code where requested. 184 185 if suppress_builtins and obj.astnode.doc is None: 186 pass 187 188 # Append the function code to the image. 189 190 else: 191 code = trans.get_code(obj) 192 self.code += code 193 194 # Omit built-in module code where requested. 195 196 if suppress_builtins: 197 pass 198 199 # Append the module top-level code to the image. 200 201 else: 202 code = trans.get_module_code() 203 self.code += code 204 205 return self.code 206 207 def get_raw_image(self, architecture=None, with_builtins=0): 208 209 "Return the raw image representation of the program." 210 211 architecture = architecture or micropython.rsvp 212 213 self.get_image(with_builtins) 214 215 objtable = self.get_object_table() 216 paramtable = self.get_parameter_table() 217 218 # Position the objects. 219 220 pos = 0 221 222 for item in self.code: 223 arch_item = architecture.get_object(item) 224 225 # Get the raw version for the architecture. 226 227 if arch_item is not None: 228 pos = arch_item.set_location(pos, with_builtins) 229 else: 230 pos += 1 231 232 # Generate the raw code. 233 234 self.raw_code = [] 235 236 for item in self.code: 237 arch_item = architecture.get_object(item) 238 239 # Get the raw version for the architecture. 240 241 if arch_item is not None: 242 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 243 arch_item.finalise_location(with_builtins) 244 else: 245 self.raw_code.append(item) 246 247 # Fix the module locations. 248 249 for module in self.importer.modules_ordered: 250 251 if not with_builtins and module.name == "__builtins__": 252 continue 253 254 module.code_location = module.blocks[0].location 255 256 self.code_location = self.importer.modules["__main__"].code_location 257 return self.raw_code 258 259 def get_object_table(self, reset=0): 260 261 "Return a table with details of attributes for classes and modules." 262 263 if self.objtable is None or reset: 264 265 t = self.objtable = micropython.table.ObjectTable() 266 for module in self.importer.get_modules(): 267 t.add(module.full_name(), module.module_attributes()) 268 269 # Add class and instance attributes for all classes, together 270 # with descendant information. 271 272 for obj in module.all_objects: 273 if isinstance(obj, micropython.data.Class): 274 275 # Prevent ambiguous classes. 276 277 full_name = obj.full_name() 278 name = obj.name 279 280 #if module.has_key(name) and module[name].defines_ambiguous_class(): 281 # raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 282 283 # Define a table entry for the class. 284 285 attributes = {full_name : obj} 286 attributes.update(obj.all_attributes()) 287 attributes.update(obj.all_descendants()) 288 t.add(full_name, attributes) 289 290 return self.objtable 291 292 def get_parameter_table(self, reset=0): 293 294 "Return a table with details of parameters for functions and methods." 295 296 # Need the object table to get at class details. 297 298 if self.paramtable is None or reset: 299 t = self.paramtable = micropython.table.ParameterTable() 300 301 # Visit each module, getting function and method details. 302 303 for module in self.importer.get_modules(): 304 for obj in module.all_objects: 305 if isinstance(obj, micropython.data.Function): 306 t.add(obj.full_name(), obj.parameters()) 307 308 # Classes are callable, too. 309 # Take details of the appropriate __init__ method to make an 310 # entry for an instantiation function for the class. 311 312 elif isinstance(obj, micropython.data.Class): 313 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 314 315 # Filter out all parameter table entries not referenced by keyword 316 # arguments. 317 318 keyword_names = set() 319 320 for module in self.importer.get_modules(): 321 keyword_names.update(module.keyword_names) 322 323 for function_name, parameters in t.table.items(): 324 for name in parameters.keys(): 325 if name in keyword_names: 326 break 327 else: 328 del t.table[function_name] 329 330 return self.paramtable 331 332 class Importer: 333 334 "An import machine, searching for and loading modules." 335 336 predefined_constants = { 337 "None" : None, 338 "True" : True, 339 "False" : False, 340 #"Ellipsis" : Ellipsis, 341 "NotImplemented" : NotImplemented 342 } 343 344 names_always_used = [ 345 "bool", "__call__", "__bool__" 346 ] 347 348 def __init__(self, path=None, verbose=0, optimisations=None): 349 350 """ 351 Initialise the importer with the given search 'path' - a list of 352 directories to search for Python modules. 353 354 The optional 'verbose' parameter causes output concerning the activities 355 of the object to be produced if set to a true value (not the default). 356 357 The optional 'optimisations' cause certain techniques to be used in 358 reducing program size and improving program efficiency. 359 """ 360 361 self.path = path or [os.getcwd()] 362 self.verbose = verbose 363 self.optimisations = optimisations or set() 364 365 self.modules = {} 366 self.modules_ordered = [] 367 self.loading = set() 368 369 # Constant records. 370 371 self.constant_values = {} 372 self.constant_list = None # cache for constants 373 self.init_predefined_constants() 374 375 # Attribute usage. 376 377 self.attributes_used = set() 378 self.name_references = {} 379 self.specific_name_references = {} 380 381 # Attribute coverage calculated during collection. 382 383 self.inferred_name_references = {} 384 385 # Attribute coverage status during collection. 386 387 self.attribute_users_visited = set() 388 self.attributes_to_visit = {} 389 390 # Status information. 391 392 self.vacuumed = 0 393 self.finalised = 0 394 395 def get_modules(self): 396 397 "Return all modules known to the importer." 398 399 return self.modules.values() 400 401 def get_module(self, name): 402 403 "Return the module with the given 'name'." 404 405 return self.modules[name] 406 407 # General maintenance. 408 409 def vacuum(self, objtable): 410 411 "Tidy up the modules." 412 413 if self.vacuumed: 414 return 415 416 # Complete the list of attribute names used in the program. 417 418 self.collect_attributes(objtable) 419 420 for name, module in self.modules.items(): 421 if module.loaded: 422 module.vacuum() 423 else: 424 del self.modules[name] 425 426 self.vacuumed = 1 427 428 def finalise(self): 429 430 "Finalise the program (which should have been vacuumed first)." 431 432 if self.finalised: 433 return 434 435 # Reset any previously compiled information. 436 437 for module in self.get_modules(): 438 module.unfinalise() 439 440 # Prepare module information again. 441 442 for module in self.get_modules(): 443 module.finalise() 444 445 self.finalised = 1 446 447 # Name accounting. 448 449 def use_name(self, name, from_name): 450 451 """ 452 Register the given 'name' as being used in the program from within an 453 object with the specified 'from_name'. 454 """ 455 456 if not self.name_references.has_key(from_name): 457 self.name_references[from_name] = set() 458 459 attrnames = (name,) 460 usage = (attrnames,) 461 self.name_references[from_name].add((None, None, usage)) 462 463 def use_names(self, user, name, usage, from_name): 464 465 """ 466 For the given attribute 'user' (which may be None if no specific user is 467 given), register for the given 'name' the given attribute 'usage' 468 (combinations of attribute names), noting the scope of this usage as 469 being the program object with the specified 'from_name'. 470 """ 471 472 if not self.name_references.has_key(from_name): 473 self.name_references[from_name] = set() 474 475 self.name_references[from_name].add((user, name, usage)) 476 477 def use_specific_name(self, objname, attrname, from_name): 478 479 """ 480 Register the given 'objname' (for an object) whose 'attrname' is being 481 used in the program from within an object with the specified 482 'from_name'. 483 """ 484 485 if not self.specific_name_references.has_key(from_name): 486 self.specific_name_references[from_name] = set() 487 self.specific_name_references[from_name].add((objname, attrname)) 488 489 # Name accounting products. 490 491 def uses_attribute(self, objname, name): 492 493 """ 494 Return whether the attribute of the object with the given 'objname' 495 having the given 'name' is used as an attribute in the program. 496 """ 497 498 return (objname + "." + name) in self.attributes_used 499 500 def use_attribute(self, objname, name): 501 502 """ 503 Indicate that in the object with the given 'objname', the attribute of 504 the given 'name' is used. 505 """ 506 507 self.attributes_used.add(objname + "." + name) 508 509 def use_object(self, objname): 510 511 "Indicate that the object with the given 'objname' is used." 512 513 self.attributes_used.add(objname) 514 515 def collect_attributes(self, objtable): 516 517 "Collect attribute references for the entire program." 518 519 # Include names which may not be explicitly used in programs. 520 # NOTE: Potentially declare these when inspecting. 521 522 for attrname in self.names_always_used: 523 for objname in objtable.all_possible_objects([attrname]): 524 525 # Record attributes of objects for potential visiting. 526 527 self.add_attribute_to_visit(objname, attrname) 528 529 # Visit all modules, since some may employ initialisation code which has 530 # some kind of side-effect. 531 532 for name in self.modules.keys(): 533 self._collect_attributes(name, objtable) 534 535 def add_attribute_to_visit(self, objname, attrname): 536 537 """ 538 Queue an attribute of the object with the given 'objname', having the 539 given 'attrname', to the list for potential visiting if the specified 540 object is actually referenced. 541 """ 542 543 if not self.attributes_to_visit.has_key(objname): 544 self.attributes_to_visit[objname] = set() 545 self.attributes_to_visit[objname].add(attrname) 546 547 def _collect_attributes_from(self, from_name, objname, attrname, objtable): 548 549 """ 550 Record the association between 'from_name' and the attribute of 551 'objname' with the given 'attrname'. Then collect attributes for the 552 referenced attribute using 'objtable'. 553 """ 554 555 if not self.inferred_name_references.has_key(from_name): 556 self.inferred_name_references[from_name] = set() 557 558 self.inferred_name_references[from_name].add((objname, attrname)) 559 self._collect_attributes(objname + "." + attrname, objtable) 560 561 def _collect_attributes(self, from_name, objtable): 562 563 """ 564 Given an object called 'from_name', find all names referenced from such 565 an object according to the register of names, using 'objtable' to infer 566 types. 567 """ 568 569 if from_name in self.attribute_users_visited: 570 return 571 572 self.attribute_users_visited.add(from_name) 573 574 # Get name references and find possible objects which support such 575 # combinations of attribute names. 576 577 for user, name, usage in self.name_references.get(from_name, []): 578 579 # Using all attribute names for a particular name, attempt to get 580 # specific object types. 581 582 all_objtypes = set() 583 584 for attrnames in usage: 585 objtypes = objtable.all_possible_objects_plus_status(attrnames) 586 if not objtypes: 587 print "Warning: usage in %r for %r finds no object supporting all attributes %r" % (from_name, name, attrnames) 588 objtypes = objtable.any_possible_objects_plus_status(attrnames) 589 if not objtypes: 590 print "Warning: usage in %r for %r finds no object supporting any attributes %r" % (from_name, name, attrnames) 591 592 all_objtypes.update(objtypes) 593 594 # Record the object types for generating guards. 595 596 if user is not None: 597 if not hasattr(user, "_attrtypes"): 598 user._attrtypes = {} 599 600 user._attrtypes[name] = all_objtypes 601 602 # For each suggested object type, consider each attribute given by 603 # the names. 604 605 for objname, is_static in all_objtypes: 606 for attrnames in usage: 607 for attrname in attrnames: 608 609 # Visit attributes of objects known to be used. 610 611 if objname in self.attributes_used: 612 self.use_attribute(objname, attrname) 613 self._collect_attributes_from(from_name, objname, attrname, objtable) 614 615 # Record attributes of other objects for potential visiting. 616 617 else: 618 self.add_attribute_to_visit(objname, attrname) 619 620 # Get specific name references and visit the referenced objects. 621 622 for objname, attrname in self.specific_name_references.get(from_name, []): 623 self.use_attribute(objname, attrname) 624 self._collect_attributes_from(from_name, objname, attrname, objtable) 625 626 # Where the object has an __init__ attribute, assume that it is an 627 # initialiser which is called at some point, and collect attributes used 628 # in this initialiser. 629 630 if "__init__" in objtable.table.get(from_name, []): 631 self.use_attribute(from_name, "__init__") 632 self._collect_attributes_from(from_name, from_name, "__init__", objtable) 633 634 # Visit attributes on this object that were queued in case of the object 635 # being referenced. 636 637 attributes_to_visit = self.attributes_to_visit.get(from_name, []) 638 639 if attributes_to_visit: 640 del self.attributes_to_visit[from_name] 641 642 for attrname in attributes_to_visit: 643 self.use_attribute(from_name, attrname) 644 self._collect_attributes_from(from_name, from_name, attrname, objtable) 645 646 # Constant accounting. 647 648 def init_predefined_constants(self): 649 650 "Ensure the predefined constants." 651 652 for name, value in self.predefined_constants.items(): 653 self.make_constant(value) 654 655 def get_predefined_constant(self, name): 656 657 "Return the predefined constant for the given 'name'." 658 659 return self.make_constant(self.predefined_constants[name]) 660 661 def get_constant(self, value): 662 663 "Return a constant for the given 'value'." 664 665 const = micropython.data.Const(value) 666 return self.constant_values[const] 667 668 def get_constant_type_name(self, value): 669 return value.__class__.__name__ 670 671 def make_constant(self, value): 672 673 "Make and return a constant for the given 'value'." 674 675 # Ensure the presence of the constant's type. 676 677 name = self.get_constant_type_name(value) 678 if self.modules.has_key("__builtins__"): 679 attr = self.modules["__builtins__"].get(name) 680 681 # Make a constant object and return it. 682 683 const = micropython.data.Const(value) 684 if not self.constant_values.has_key(const): 685 self.constant_values[const] = const 686 return self.constant_values[const] 687 688 def constants(self): 689 690 "Return a list of constants." 691 692 if self.constant_list is None: 693 self.constant_list = list(self.constant_values.values()) 694 695 return self.constant_list 696 697 # Import methods. 698 699 def find_in_path(self, name): 700 701 """ 702 Find the given module 'name' in the search path, returning None where no 703 such module could be found, or a 2-tuple from the 'find' method 704 otherwise. 705 """ 706 707 for d in self.path: 708 m = self.find(d, name) 709 if m: return m 710 return None 711 712 def find(self, d, name): 713 714 """ 715 In the directory 'd', find the given module 'name', where 'name' can 716 either refer to a single file module or to a package. Return None if the 717 'name' cannot be associated with either a file or a package directory, 718 or a 2-tuple from '_find_package' or '_find_module' otherwise. 719 """ 720 721 m = self._find_package(d, name) 722 if m: return m 723 m = self._find_module(d, name) 724 if m: return m 725 return None 726 727 def _find_module(self, d, name): 728 729 """ 730 In the directory 'd', find the given module 'name', returning None where 731 no suitable file exists in the directory, or a 2-tuple consisting of 732 None (indicating that no package directory is involved) and a filename 733 indicating the location of the module. 734 """ 735 736 name_py = name + os.extsep + "py" 737 filename = self._find_file(d, name_py) 738 if filename: 739 return None, filename 740 return None 741 742 def _find_package(self, d, name): 743 744 """ 745 In the directory 'd', find the given package 'name', returning None 746 where no suitable package directory exists, or a 2-tuple consisting of 747 a directory (indicating the location of the package directory itself) 748 and a filename indicating the location of the __init__.py module which 749 declares the package's top-level contents. 750 """ 751 752 filename = self._find_file(d, name) 753 if filename: 754 init_py = "__init__" + os.path.extsep + "py" 755 init_py_filename = self._find_file(filename, init_py) 756 if init_py_filename: 757 return filename, init_py_filename 758 return None 759 760 def _find_file(self, d, filename): 761 762 """ 763 Return the filename obtained when searching the directory 'd' for the 764 given 'filename', or None if no actual file exists for the filename. 765 """ 766 767 filename = os.path.join(d, filename) 768 if os.path.exists(filename): 769 return filename 770 else: 771 return None 772 773 def load(self, name, return_leaf=0): 774 775 """ 776 Load the module or package with the given 'name'. Return an object 777 referencing the loaded module or package, or None if no such module or 778 package exists. 779 """ 780 781 if return_leaf: 782 name_for_return = name 783 else: 784 name_for_return = name.split(".")[0] 785 786 if self.modules.has_key(name) and self.modules[name].loaded: 787 #print "Cached (%s)" % name 788 return self.modules[name_for_return] 789 790 if self.verbose: 791 print "Loading", name 792 793 # Split the name into path components, and try to find the uppermost in 794 # the search path. 795 796 path = name.split(".") 797 m = self.find_in_path(path[0]) 798 if not m: 799 if self.verbose: 800 print "Not found (%s)" % path[0] 801 return None # NOTE: Import error. 802 d, filename = m 803 804 # Either acquire a reference to an already-imported module, or load the 805 # module from a file. 806 807 top = module = self.load_from_file(filename, path[0]) 808 809 # For hierarchical names, traverse each path component... 810 811 if len(path) > 1: 812 if not d: 813 if self.verbose: 814 print "No package (%s)" % filename 815 return None # NOTE: Import error (package not found). 816 else: 817 self.add_submodules(d, module) 818 819 path_so_far = path[:1] 820 for p in path[1:]: 821 path_so_far.append(p) 822 823 # Find the package or module concerned. 824 825 m = self.find(d, p) 826 if not m: 827 if self.verbose: 828 print "Not found (%s)" % p 829 return None # NOTE: Import error. 830 d, filename = m 831 module_name = ".".join(path_so_far) 832 833 # Either reference an imported module or load one from a file. 834 835 submodule = self.load_from_file(filename, module_name) 836 837 if d: 838 self.add_submodules(d, module) 839 840 # Store the submodule within its parent module. 841 842 module.set_module(p, submodule) 843 module = submodule 844 845 # Return either the deepest or the uppermost module. 846 847 if return_leaf: 848 return module 849 else: 850 return top 851 852 def load_from_file(self, name, module_name=None): 853 854 """ 855 Load the module with the given 'name' (which may be a full module path). 856 """ 857 858 if module_name is None: 859 module_name = "__main__" 860 861 module = self.add_module(module_name) 862 if not module.loaded and module not in self.loading: 863 self.loading.add(module) 864 #print "Parsing", name 865 module.parse(name) 866 #print "Done", name 867 self.loading.remove(module) 868 module.loaded = 1 869 870 # Record the module. 871 872 self.use_object(module.full_name()) 873 #print "Loaded", module_name, "with namespace", module.namespace.keys() 874 return module 875 876 def add_module(self, module_name): 877 878 """ 879 Return the module with the given 'module_name', adding a new module 880 object if one does not already exist. 881 """ 882 883 if not self.modules.has_key(module_name): 884 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 885 self.modules_ordered.append(module) 886 else: 887 module = self.modules[module_name] 888 return module 889 890 def add_submodules(self, pathname, module): 891 892 """ 893 Work around insufficient __all__ declarations and examine the directory 894 with the given 'pathname', adding submodules to the given 'module'. 895 """ 896 897 for filename in os.listdir(pathname): 898 submodule, ext = os.path.splitext(filename) 899 if ext not in ("", ".py"): 900 continue 901 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 902 903 # vim: tabstop=4 expandtab shiftwidth=4