1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009, 2010 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed. Here, the standard path 26 for module searching is employed: 27 28 importer = Importer(sys.path) 29 30 To generate programs, the above importer should be supplied in the 31 initialisation of a program instance, and then various methods are called: 32 33 program = Program(importer) 34 image = program.get_raw_image() 35 36 Such importer and program objects are the most convenient mechanism through 37 which the functionality of the micropython package may be accessed. 38 """ 39 40 from micropython.common import ProcessingError, TableGenerationError 41 import micropython.ast 42 import micropython.data 43 import micropython.opt 44 import micropython.inspect 45 import micropython.table 46 import os 47 48 try: 49 set 50 except NameError: 51 from sets import Set as set 52 53 class Program: 54 55 "This class supports the generation of a program image." 56 57 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 58 59 def __init__(self, importer, optimisations=None): 60 61 """ 62 Initialise the program representation with an 'importer' which is able 63 to locate and load Python modules. 64 65 The optional 'optimisations' cause certain techniques to be used in 66 reducing program size and improving program efficiency. 67 """ 68 69 self.importer = importer 70 self.optimisations = optimisations or set() 71 72 # Remember the tables once generated. 73 74 self.objtable = None 75 self.paramtable = None 76 77 # Main program information. 78 79 self.code = None 80 self.code_location = None 81 82 def get_importer(self): 83 return self.importer 84 85 # Access to finalised program information. 86 87 def finalise(self): 88 89 "Finalise the program." 90 91 # Need the tables to finalise. 92 93 objtable = self.get_object_table() 94 self.get_parameter_table() 95 96 self.importer.vacuum(objtable) 97 self.importer.finalise() 98 99 # Now remove unneeded things from the tables. 100 101 self.get_object_table(reset=1) 102 self.get_parameter_table(reset=1) 103 104 def get_image(self, with_builtins=0): 105 106 """ 107 Return the program image including built-in objects if 'with_builtins' 108 is specified and set to a true value. 109 """ 110 111 if self.code is not None: 112 return self.code 113 114 # Optimise and regenerate the object table. 115 116 self.finalise() 117 self.code = [] 118 119 # Append constants to the image. 120 121 for const in self.importer.constants(): 122 self.code.append(const) 123 124 last_module = self.importer.modules_ordered[-1] 125 126 for module in self.importer.modules_ordered: 127 suppress_builtins = not with_builtins and module.name == "__builtins__" 128 129 # Position the module in the image and make a translation. 130 131 trans = micropython.ast.Translation(module, self) 132 133 # Add header details. 134 135 self.code.append(module) 136 137 # Append module attributes to the image. 138 139 attributes = module.module_attributes() 140 self.code += module.attributes_as_list() 141 142 # Append classes and functions to the image. 143 144 for obj in module.all_objects: 145 if isinstance(obj, micropython.inspect.Class): 146 147 # Add header details. 148 149 self.code.append(obj) 150 151 # Append class attributes to the image. 152 153 attributes = obj.class_attributes() 154 self.code += obj.attributes_as_list() 155 156 # Omit built-in function code where requested. 157 158 if suppress_builtins and obj.astnode.doc is None: 159 continue 160 161 # Generate the instantiator/initialiser. 162 # Append the function code to the image. 163 164 code = trans.get_instantiator_code(obj) 165 self.code += code 166 167 # Class-level code is generated separately at the module 168 # level, and the code location is set within the code 169 # generation process for the module. 170 171 elif isinstance(obj, micropython.inspect.Function): 172 173 # Add header details. 174 175 self.code.append(obj) 176 177 # Append any default values to the image. 178 # Only do this for functions which are not dynamic. 179 180 if not obj.is_dynamic(): 181 self.code += obj.default_attrs 182 183 # Omit built-in function code where requested. 184 185 if suppress_builtins and obj.astnode.doc is None: 186 pass 187 188 # Append the function code to the image. 189 190 else: 191 code = trans.get_code(obj) 192 self.code += code 193 194 # Omit built-in module code where requested. 195 196 if suppress_builtins: 197 pass 198 199 # Append the module top-level code to the image. 200 201 else: 202 code = trans.get_module_code() 203 self.code += code 204 205 return self.code 206 207 def get_raw_image(self, architecture=None, with_builtins=0): 208 209 "Return the raw image representation of the program." 210 211 architecture = architecture or micropython.rsvp 212 213 self.get_image(with_builtins) 214 215 objtable = self.get_object_table() 216 paramtable = self.get_parameter_table() 217 218 # Position the objects. 219 220 pos = 0 221 222 for item in self.code: 223 arch_item = architecture.get_object(item) 224 225 # Get the raw version for the architecture. 226 227 if arch_item is not None: 228 pos = arch_item.set_location(pos, with_builtins) 229 else: 230 pos += 1 231 232 # Generate the raw code. 233 234 self.raw_code = [] 235 236 for item in self.code: 237 arch_item = architecture.get_object(item) 238 239 # Get the raw version for the architecture. 240 241 if arch_item is not None: 242 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 243 arch_item.finalise_location(with_builtins) 244 else: 245 self.raw_code.append(item) 246 247 # Fix the module locations. 248 249 for module in self.importer.modules_ordered: 250 251 if not with_builtins and module.name == "__builtins__": 252 continue 253 254 module.code_location = module.blocks[0].location 255 256 self.code_location = self.importer.modules["__main__"].code_location 257 return self.raw_code 258 259 def get_object_table(self, reset=0): 260 261 "Return a table with details of attributes for classes and modules." 262 263 if self.objtable is None or reset: 264 265 t = self.objtable = micropython.table.ObjectTable() 266 for module in self.importer.get_modules(): 267 t.add(module.full_name(), module.module_attributes()) 268 269 # Add class and instance attributes for all classes, together 270 # with descendant information. 271 272 for obj in module.all_objects: 273 if isinstance(obj, micropython.inspect.Class): 274 275 # Prevent ambiguous classes. 276 277 full_name = obj.full_name() 278 name = obj.name 279 280 if module.has_key(name) and module[name].defines_ambiguous_class(): 281 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 282 283 # Define a table entry for the class. 284 285 attributes = {full_name : obj} 286 attributes.update(obj.all_attributes()) 287 attributes.update(obj.all_descendants()) 288 t.add(full_name, attributes) 289 290 return self.objtable 291 292 def get_parameter_table(self, reset=0): 293 294 "Return a table with details of parameters for functions and methods." 295 296 # Need the object table to get at class details. 297 298 if self.paramtable is None or reset: 299 t = self.paramtable = micropython.table.ParameterTable() 300 301 # Visit each module, getting function and method details. 302 303 for module in self.importer.get_modules(): 304 for obj in module.all_objects: 305 if isinstance(obj, micropython.inspect.Function): 306 t.add(obj.full_name(), obj.parameters()) 307 308 # Classes are callable, too. 309 # Take details of the appropriate __init__ method to make an 310 # entry for an instantiation function for the class. 311 312 elif isinstance(obj, micropython.inspect.Class): 313 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 314 315 # Filter out all parameter table entries not referenced by keyword 316 # arguments. 317 318 keyword_names = set() 319 320 for module in self.importer.get_modules(): 321 keyword_names.update(module.keyword_names) 322 323 for function_name, parameters in t.table.items(): 324 for name in parameters.keys(): 325 if name in keyword_names: 326 break 327 else: 328 del t.table[function_name] 329 330 return self.paramtable 331 332 class Importer: 333 334 "An import machine, searching for and loading modules." 335 336 predefined_constants = { 337 "None" : None, 338 "True" : True, 339 "False" : False, 340 #"Ellipsis" : Ellipsis, 341 "NotImplemented" : NotImplemented 342 } 343 344 names_always_used = [ 345 "bool", "__call__", "__bool__" 346 ] 347 348 def __init__(self, path=None, verbose=0, optimisations=None): 349 350 """ 351 Initialise the importer with the given search 'path' - a list of 352 directories to search for Python modules. 353 354 The optional 'verbose' parameter causes output concerning the activities 355 of the object to be produced if set to a true value (not the default). 356 357 The optional 'optimisations' cause certain techniques to be used in 358 reducing program size and improving program efficiency. 359 """ 360 361 self.path = path or [os.getcwd()] 362 self.verbose = verbose 363 self.optimisations = optimisations or set() 364 365 self.modules = {} 366 self.modules_ordered = [] 367 self.loading = set() 368 369 # Constant records. 370 371 self.constant_values = {} 372 self.constant_list = None # cache for constants 373 self.init_predefined_constants() 374 375 # Attribute usage. 376 377 self.attributes_used = set() 378 self.name_references = {} 379 self.specific_name_references = {} 380 self.attribute_users_visited = set() 381 self.attributes_to_visit = {} 382 383 # Status information. 384 385 self.vacuumed = 0 386 self.finalised = 0 387 388 def get_modules(self): 389 390 "Return all modules known to the importer." 391 392 return self.modules.values() 393 394 def get_module(self, name): 395 396 "Return the module with the given 'name'." 397 398 return self.modules[name] 399 400 # General maintenance. 401 402 def vacuum(self, objtable): 403 404 "Tidy up the modules." 405 406 if self.vacuumed: 407 return 408 409 # Complete the list of attribute names used in the program. 410 411 self.collect_attributes(objtable) 412 413 for name, module in self.modules.items(): 414 if module.loaded: 415 module.vacuum() 416 else: 417 del self.modules[name] 418 419 self.vacuumed = 1 420 421 def finalise(self): 422 423 "Finalise the program (which should have been vacuumed first)." 424 425 if self.finalised: 426 return 427 428 # Reset any previously compiled information. 429 430 for module in self.get_modules(): 431 module.unfinalise() 432 433 # Prepare module information again. 434 435 for module in self.get_modules(): 436 module.finalise() 437 438 self.finalised = 1 439 440 # Name accounting. 441 442 def use_name(self, name, from_name): 443 444 """ 445 Register the given 'name' as being used in the program from within an 446 object with the specified 'from_name'. 447 """ 448 449 if not self.name_references.has_key(from_name): 450 self.name_references[from_name] = set() 451 452 attrnames = (name,) 453 self.name_references[from_name].add((None, None, attrnames)) 454 455 def use_names(self, user, name, usage, from_name): 456 457 """ 458 For the given attribute 'user' (which may be None if no specific user is 459 given), register for the given 'name' the given attribute 'usage' 460 (combinations of attribute names), noting the scope of this usage as 461 being the program object with the specified 'from_name'. 462 """ 463 464 if not self.name_references.has_key(from_name): 465 self.name_references[from_name] = set() 466 467 all_attrnames = set() 468 469 for attrnames in usage: 470 all_attrnames.update(attrnames) 471 472 self.name_references[from_name].add((user, name, tuple(all_attrnames))) 473 474 def use_specific_name(self, objname, attrname, from_name): 475 476 """ 477 Register the given 'objname' (for an object) whose 'attrname' is being 478 used in the program from within an object with the specified 479 'from_name'. 480 """ 481 482 if not self.specific_name_references.has_key(from_name): 483 self.specific_name_references[from_name] = set() 484 self.specific_name_references[from_name].add((objname, attrname)) 485 486 # Name accounting products. 487 488 def uses_attribute(self, objname, name): 489 490 """ 491 Return whether the attribute of the object with the given 'objname' 492 having the given 'name' is used as an attribute in the program. 493 """ 494 495 return (objname + "." + name) in self.attributes_used 496 497 def use_attribute(self, objname, name): 498 499 """ 500 Indicate that in the object with the given 'objname', the attribute of 501 the given 'name' is used. 502 """ 503 504 self.attributes_used.add(objname + "." + name) 505 506 def use_object(self, objname): 507 508 "Indicate that the object with the given 'objname' is used." 509 510 self.attributes_used.add(objname) 511 512 def collect_attributes(self, objtable): 513 514 "Collect attribute references for the entire program." 515 516 # Include names which may not be explicitly used in programs. 517 # NOTE: Potentially declare these when inspecting. 518 519 for attrname in self.names_always_used: 520 for objname in objtable.all_possible_objects([attrname]): 521 522 # Record attributes of objects for potential visiting. 523 524 self.add_attribute_to_visit(objname, attrname) 525 526 # Visit all modules, since some may employ initialisation code which has 527 # some kind of side-effect. 528 529 for name in self.modules.keys(): 530 self._collect_attributes(name, objtable) 531 532 def add_attribute_to_visit(self, objname, attrname): 533 534 """ 535 Queue an attribute of the object with the given 'objname', having the 536 given 'attrname', to the list for potential visiting if the specified 537 object is actually referenced. 538 """ 539 540 if not self.attributes_to_visit.has_key(objname): 541 self.attributes_to_visit[objname] = set() 542 self.attributes_to_visit[objname].add(attrname) 543 544 def _collect_attributes(self, from_name, objtable): 545 546 """ 547 Given an object called 'from_name', find all names referenced from such 548 an object according to the register of names. 549 """ 550 551 if from_name in self.attribute_users_visited: 552 return 553 554 self.attribute_users_visited.add(from_name) 555 556 # Get name references and find possible objects which support such 557 # combinations of attribute names. 558 559 for user, name, attrnames in self.name_references.get(from_name, []): 560 561 # Using all attribute names for a particular name, attempt to get 562 # specific object types. 563 564 objtypes = objtable.all_possible_objects_plus_status(attrnames) 565 if not objtypes: 566 print "Warning: usage in %r finds no object supporting all attributes %r" % (from_name, attrnames) 567 objtypes = objtable.any_possible_objects_plus_status(attrnames) 568 if not objtypes: 569 print "Warning: usage in %r finds no object supporting any attributes %r" % (from_name, attrnames) 570 571 # Record the object types for generating guards. 572 573 if user is not None: 574 if not hasattr(user, "_attrtypes"): 575 user._attrtypes = {} 576 577 user._attrtypes[name] = objtypes 578 579 # For each suggested object, consider each attribute given by the 580 # names. 581 582 for objname, is_static in objtypes: 583 for attrname in attrnames: 584 585 # Visit attributes of objects known to be used. 586 587 if objname in self.attributes_used: 588 self.use_attribute(objname, attrname) 589 self._collect_attributes(objname + "." + attrname, objtable) 590 591 # Record attributes of other objects for potential visiting. 592 593 else: 594 self.add_attribute_to_visit(objname, attrname) 595 596 # Get specific name references and visit the referenced objects. 597 598 for objname, attrname in self.specific_name_references.get(from_name, []): 599 self.use_attribute(objname, attrname) 600 self._collect_attributes(objname + "." + attrname, objtable) 601 602 # Where the object has an __init__ attribute, assume that it is an 603 # initialiser which is called at some point, and collect attributes used 604 # in this initialiser. 605 606 if "__init__" in objtable.table.get(from_name, []): 607 self.use_attribute(from_name, "__init__") 608 self._collect_attributes(from_name + ".__init__", objtable) 609 610 # Visit attributes on objects not previously visited. 611 612 attributes_to_visit = self.attributes_to_visit.get(from_name, []) 613 614 if attributes_to_visit: 615 del self.attributes_to_visit[from_name] 616 617 for attrname in attributes_to_visit: 618 self.use_attribute(from_name, attrname) 619 self._collect_attributes(from_name + "." + attrname, objtable) 620 621 # Constant accounting. 622 623 def init_predefined_constants(self): 624 625 "Ensure the predefined constants." 626 627 for name, value in self.predefined_constants.items(): 628 self.make_constant(value) 629 630 def get_predefined_constant(self, name): 631 632 "Return the predefined constant for the given 'name'." 633 634 return self.make_constant(self.predefined_constants[name]) 635 636 def get_constant(self, value): 637 638 "Return a constant for the given 'value'." 639 640 const = micropython.data.Const(value) 641 return self.constant_values[const] 642 643 def get_constant_type_name(self, value): 644 return value.__class__.__name__ 645 646 def make_constant(self, value): 647 648 "Make and return a constant for the given 'value'." 649 650 # Ensure the presence of the constant's type. 651 652 name = self.get_constant_type_name(value) 653 if self.modules.has_key("__builtins__"): 654 attr = self.modules["__builtins__"].get(name) 655 656 # Make a constant object and return it. 657 658 const = micropython.data.Const(value) 659 if not self.constant_values.has_key(const): 660 self.constant_values[const] = const 661 return self.constant_values[const] 662 663 def constants(self): 664 665 "Return a list of constants." 666 667 if self.constant_list is None: 668 self.constant_list = list(self.constant_values.values()) 669 670 return self.constant_list 671 672 # Import methods. 673 674 def find_in_path(self, name): 675 676 """ 677 Find the given module 'name' in the search path, returning None where no 678 such module could be found, or a 2-tuple from the 'find' method 679 otherwise. 680 """ 681 682 for d in self.path: 683 m = self.find(d, name) 684 if m: return m 685 return None 686 687 def find(self, d, name): 688 689 """ 690 In the directory 'd', find the given module 'name', where 'name' can 691 either refer to a single file module or to a package. Return None if the 692 'name' cannot be associated with either a file or a package directory, 693 or a 2-tuple from '_find_package' or '_find_module' otherwise. 694 """ 695 696 m = self._find_package(d, name) 697 if m: return m 698 m = self._find_module(d, name) 699 if m: return m 700 return None 701 702 def _find_module(self, d, name): 703 704 """ 705 In the directory 'd', find the given module 'name', returning None where 706 no suitable file exists in the directory, or a 2-tuple consisting of 707 None (indicating that no package directory is involved) and a filename 708 indicating the location of the module. 709 """ 710 711 name_py = name + os.extsep + "py" 712 filename = self._find_file(d, name_py) 713 if filename: 714 return None, filename 715 return None 716 717 def _find_package(self, d, name): 718 719 """ 720 In the directory 'd', find the given package 'name', returning None 721 where no suitable package directory exists, or a 2-tuple consisting of 722 a directory (indicating the location of the package directory itself) 723 and a filename indicating the location of the __init__.py module which 724 declares the package's top-level contents. 725 """ 726 727 filename = self._find_file(d, name) 728 if filename: 729 init_py = "__init__" + os.path.extsep + "py" 730 init_py_filename = self._find_file(filename, init_py) 731 if init_py_filename: 732 return filename, init_py_filename 733 return None 734 735 def _find_file(self, d, filename): 736 737 """ 738 Return the filename obtained when searching the directory 'd' for the 739 given 'filename', or None if no actual file exists for the filename. 740 """ 741 742 filename = os.path.join(d, filename) 743 if os.path.exists(filename): 744 return filename 745 else: 746 return None 747 748 def load(self, name, return_leaf=0): 749 750 """ 751 Load the module or package with the given 'name'. Return an object 752 referencing the loaded module or package, or None if no such module or 753 package exists. 754 """ 755 756 if return_leaf: 757 name_for_return = name 758 else: 759 name_for_return = name.split(".")[0] 760 761 if self.modules.has_key(name) and self.modules[name].loaded: 762 #print "Cached (%s)" % name 763 return self.modules[name_for_return] 764 765 if self.verbose: 766 print "Loading", name 767 768 # Split the name into path components, and try to find the uppermost in 769 # the search path. 770 771 path = name.split(".") 772 m = self.find_in_path(path[0]) 773 if not m: 774 if self.verbose: 775 print "Not found (%s)" % path[0] 776 return None # NOTE: Import error. 777 d, filename = m 778 779 # Either acquire a reference to an already-imported module, or load the 780 # module from a file. 781 782 top = module = self.load_from_file(filename, path[0]) 783 784 # For hierarchical names, traverse each path component... 785 786 if len(path) > 1: 787 if not d: 788 if self.verbose: 789 print "No package (%s)" % filename 790 return None # NOTE: Import error (package not found). 791 else: 792 self.add_submodules(d, module) 793 794 path_so_far = path[:1] 795 for p in path[1:]: 796 path_so_far.append(p) 797 798 # Find the package or module concerned. 799 800 m = self.find(d, p) 801 if not m: 802 if self.verbose: 803 print "Not found (%s)" % p 804 return None # NOTE: Import error. 805 d, filename = m 806 module_name = ".".join(path_so_far) 807 808 # Either reference an imported module or load one from a file. 809 810 submodule = self.load_from_file(filename, module_name) 811 812 if d: 813 self.add_submodules(d, module) 814 815 # Store the submodule within its parent module. 816 817 module.set_module(p, submodule) 818 module = submodule 819 820 # Return either the deepest or the uppermost module. 821 822 if return_leaf: 823 return module 824 else: 825 return top 826 827 def load_from_file(self, name, module_name=None): 828 829 """ 830 Load the module with the given 'name' (which may be a full module path). 831 """ 832 833 if module_name is None: 834 module_name = "__main__" 835 836 module = self.add_module(module_name) 837 if not module.loaded and module not in self.loading: 838 self.loading.add(module) 839 #print "Parsing", name 840 module.parse(name) 841 #print "Done", name 842 self.loading.remove(module) 843 module.loaded = 1 844 845 # Record the module. 846 847 self.use_object(module.full_name()) 848 #print "Loaded", module_name, "with namespace", module.namespace.keys() 849 return module 850 851 def add_module(self, module_name): 852 853 """ 854 Return the module with the given 'module_name', adding a new module 855 object if one does not already exist. 856 """ 857 858 if not self.modules.has_key(module_name): 859 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 860 self.modules_ordered.append(module) 861 else: 862 module = self.modules[module_name] 863 return module 864 865 def add_submodules(self, pathname, module): 866 867 """ 868 Work around insufficient __all__ declarations and examine the directory 869 with the given 'pathname', adding submodules to the given 'module'. 870 """ 871 872 for filename in os.listdir(pathname): 873 submodule, ext = os.path.splitext(filename) 874 if ext not in ("", ".py"): 875 continue 876 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 877 878 # vim: tabstop=4 expandtab shiftwidth=4