1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed. Here, the standard path 26 for module searching is employed: 27 28 importer = Importer(sys.path) 29 30 To generate programs, the above importer should be supplied in the 31 initialisation of a program instance, and then various methods are called: 32 33 program = Program(importer) 34 image = program.get_raw_image() 35 36 Such importer and program objects are the most convenient mechanism through 37 which the functionality of the micropython package may be accessed. 38 """ 39 40 from micropython.common import ProcessingError, TableGenerationError 41 import micropython.ast 42 import micropython.data 43 import micropython.opt 44 import micropython.inspect 45 import micropython.table 46 import os 47 48 try: 49 set 50 except NameError: 51 from sets import Set as set 52 53 class Program: 54 55 "This class supports the generation of a program image." 56 57 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 58 59 def __init__(self, importer, optimisations=None): 60 61 """ 62 Initialise the program representation with an 'importer' which is able 63 to locate and load Python modules. 64 65 The optional 'optimisations' cause certain techniques to be used in 66 reducing program size and improving program efficiency. 67 """ 68 69 self.importer = importer 70 self.optimisations = optimisations or set() 71 72 # Remember the tables once generated. 73 74 self.objtable = None 75 self.paramtable = None 76 77 # Main program information. 78 79 self.code = None 80 self.code_location = None 81 82 def get_importer(self): 83 return self.importer 84 85 # Access to finalised program information. 86 87 def get_image(self, with_builtins=0): 88 89 """ 90 Return the program image including built-in objects if 'with_builtins' 91 is specified and set to a true value. 92 """ 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 100 # Optimise and regenerate the object table. 101 102 self.importer.vacuum(objtable) 103 objtable = self.get_object_table(reset=1) 104 105 # Finalise the importer. 106 107 self.importer.finalise() 108 109 self.code = [] 110 111 # Append constants to the image. 112 113 for const in self.importer.constants(): 114 self.code.append(const) 115 116 last_module = self.importer.modules_ordered[-1] 117 118 for module in self.importer.modules_ordered: 119 suppress_builtins = not with_builtins and module.name == "__builtins__" 120 121 # Position the module in the image and make a translation. 122 123 trans = micropython.ast.Translation(module, self) 124 125 # Add header details. 126 127 self.code.append(module) 128 129 # Append module attributes to the image. 130 131 attributes = module.module_attributes() 132 self.code += module.attributes_as_list() 133 134 # Append classes and functions to the image. 135 136 for obj in module.all_objects: 137 if isinstance(obj, micropython.inspect.Class): 138 139 # Add header details. 140 141 self.code.append(obj) 142 143 # Append class attributes to the image. 144 145 attributes = obj.class_attributes() 146 self.code += obj.attributes_as_list() 147 148 # Omit built-in function code where requested. 149 150 if suppress_builtins and obj.astnode.doc is None: 151 continue 152 153 # Generate the instantiator/initialiser. 154 # Append the function code to the image. 155 156 instantiator = obj.get_instantiator() 157 code = trans.get_instantiator_code(obj) 158 self.code += code 159 160 # Class-level code is generated separately at the module 161 # level, and the code location is set within the code 162 # generation process for the module. 163 164 elif isinstance(obj, micropython.inspect.Function): 165 166 # Add header details. 167 168 self.code.append(obj) 169 170 # Append any default values to the image. 171 # Only do this for named functions (not lambdas). 172 173 if obj.name is not None: 174 self.code += obj.default_attrs 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and obj.astnode.doc is None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 187 # Omit built-in module code where requested. 188 189 if suppress_builtins: 190 pass 191 192 # Append the module top-level code to the image. 193 194 else: 195 code = trans.get_module_code() 196 self.code += code 197 198 return self.code 199 200 def get_raw_image(self, architecture=None, with_builtins=0): 201 202 "Return the raw image representation of the program." 203 204 architecture = architecture or micropython.rsvp 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 215 for item in self.code: 216 arch_item = architecture.get_object(item) 217 218 # Get the raw version for the architecture. 219 220 if arch_item is not None: 221 pos = arch_item.set_location(pos, with_builtins) 222 else: 223 pos += 1 224 225 # Generate the raw code. 226 227 self.raw_code = [] 228 229 for item in self.code: 230 arch_item = architecture.get_object(item) 231 232 # Get the raw version for the architecture. 233 234 if arch_item is not None: 235 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 236 arch_item.finalise_location(with_builtins) 237 else: 238 self.raw_code.append(item) 239 240 # Fix the module locations. 241 242 for module in self.importer.modules_ordered: 243 244 if not with_builtins and module.name == "__builtins__": 245 continue 246 247 module.code_location = module.blocks[0].location 248 249 self.code_location = self.importer.modules["__main__"].code_location 250 return self.raw_code 251 252 def get_object_table(self, reset=0): 253 254 "Return a table with details of attributes for classes and modules." 255 256 if self.objtable is None or reset: 257 258 t = self.objtable = micropython.table.ObjectTable() 259 for module in self.importer.get_modules(): 260 t.add(module.full_name(), module.module_attributes()) 261 262 # Add class and instance attributes for all classes, together 263 # with descendant information. 264 265 for obj in module.all_objects: 266 if isinstance(obj, micropython.inspect.Class): 267 268 # Prevent ambiguous classes. 269 270 full_name = obj.full_name() 271 name = obj.name 272 273 if module.has_key(name) and module[name].defines_ambiguous_class(): 274 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 275 276 # Define a table entry for the class. 277 278 attributes = {full_name : obj} 279 attributes.update(obj.all_attributes()) 280 attributes.update(obj.all_descendants()) 281 t.add(full_name, attributes) 282 283 return self.objtable 284 285 def get_parameter_table(self): 286 287 "Return a table with details of parameters for functions and methods." 288 289 # Need the object table to get at class details. 290 291 objtable = self.get_object_table() 292 293 if self.paramtable is None: 294 t = self.paramtable = micropython.table.ParameterTable() 295 296 # Visit each module, getting function and method details. 297 298 for module in self.importer.get_modules(): 299 for obj in module.all_objects: 300 if isinstance(obj, micropython.inspect.Function): 301 t.add(obj.full_name(), obj.parameters()) 302 303 # Classes are callable, too. 304 # Take details of the appropriate __init__ method to make an 305 # entry for an instantiation function for the class. 306 307 elif isinstance(obj, micropython.inspect.Class): 308 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 309 310 # Filter out all parameter table entries not referenced by keyword 311 # arguments. 312 313 keyword_names = set() 314 315 for module in self.importer.get_modules(): 316 keyword_names.update(module.keyword_names) 317 318 for function_name, parameters in t.table.items(): 319 for name in parameters.keys(): 320 if name in keyword_names: 321 break 322 else: 323 del t.table[function_name] 324 325 return self.paramtable 326 327 class Importer: 328 329 "An import machine, searching for and loading modules." 330 331 predefined_constants = { 332 "None" : None, 333 "True" : True, 334 "False" : False, 335 #"Ellipsis" : Ellipsis, 336 "NotImplemented" : NotImplemented 337 } 338 339 names_always_used = [ 340 "bool", "__init__", "__call__", "__bool__", "__builtins__", "__main__" 341 ] 342 343 def __init__(self, path=None, verbose=0, optimisations=None): 344 345 """ 346 Initialise the importer with the given search 'path' - a list of 347 directories to search for Python modules. 348 349 The optional 'verbose' parameter causes output concerning the activities 350 of the object to be produced if set to a true value (not the default). 351 352 The optional 'optimisations' cause certain techniques to be used in 353 reducing program size and improving program efficiency. 354 """ 355 356 self.path = path or [os.getcwd()] 357 self.verbose = verbose 358 self.optimisations = optimisations or set() 359 360 self.modules = {} 361 self.modules_ordered = [] 362 self.loading = set() 363 364 # Constant records. 365 366 self.constant_values = {} 367 self.constant_list = None # cache for constants 368 self.init_predefined_constants() 369 370 # Attribute usage. 371 372 self.attributes_used = {} 373 self.name_references = {} 374 self.specific_name_references = {} 375 376 # Status information. 377 378 self.vacuumed = 0 379 self.finalised = 0 380 381 def get_modules(self): 382 383 "Return all modules known to the importer." 384 385 return self.modules.values() 386 387 def get_module(self, name): 388 389 "Return the module with the given 'name'." 390 391 return self.modules[name] 392 393 # General maintenance. 394 395 def vacuum(self, objtable): 396 397 "Tidy up the modules." 398 399 if self.vacuumed: 400 return 401 402 # Complete the list of attribute names used in the program. 403 404 self.collect_attributes(objtable) 405 406 for name, module in self.modules.items(): 407 if module.loaded: 408 module.vacuum() 409 else: 410 del self.modules[name] 411 412 self.vacuumed = 1 413 414 def finalise(self): 415 416 "Finalise the program (which should have been vacuumed first)." 417 418 if self.finalised: 419 return 420 421 for module in self.get_modules(): 422 module.finalise() 423 424 self.finalised = 1 425 426 # Name accounting. 427 428 def use_name(self, name, from_name): 429 430 """ 431 Register the given 'name' as being used in the program from within an 432 object with the specified 'from_name'. 433 """ 434 435 if not self.name_references.has_key(from_name): 436 self.name_references[from_name] = set() 437 self.name_references[from_name].add((name,)) 438 439 def use_names(self, names, from_name): 440 441 """ 442 Register the given 'names' as being used in the program from within an 443 object with the specified 'from_name'. 444 """ 445 446 if not self.name_references.has_key(from_name): 447 self.name_references[from_name] = set() 448 self.name_references[from_name].add(names) 449 450 def use_specific_name(self, objname, attrname, from_name): 451 452 """ 453 Register the given 'objname' (for an object) whose 'attrname' is being 454 used in the program from within an object with the specified 455 'from_name'. 456 """ 457 458 if not self.specific_name_references.has_key(from_name): 459 self.specific_name_references[from_name] = set() 460 self.specific_name_references[from_name].add((objname, attrname)) 461 462 # Name accounting products. 463 464 def uses_attribute(self, obj, name): 465 466 """ 467 Return whether the attribute of object 'obj' having the given 'name' is 468 used as an attribute in the program. 469 """ 470 471 return name in self.attributes_used.get(obj.full_name(), []) 472 473 def use_attribute(self, objname, name): 474 475 """ 476 Indicate that in the object with the given 'objname', the attribute of 477 the given 'name' is used. 478 """ 479 480 if not self.attributes_used.has_key(objname): 481 self.attributes_used[objname] = set() 482 self.attributes_used[objname].add(name) 483 484 def collect_attributes(self, objtable): 485 486 "Collect attribute references for the entire program." 487 488 # Start with the "root" modules, finding referenced objects. 489 490 self._collect_attributes("__builtins__", objtable) 491 self._collect_attributes("__main__", objtable) 492 493 # Include names which may not be explicitly used in programs. 494 # NOTE: Potentially declare these when inspecting. 495 496 for name in self.names_always_used: 497 for objname in objtable.all_possible_objects([name]): 498 self.use_attribute(objname, name) 499 500 def _collect_attributes(self, from_name, objtable): 501 502 """ 503 Given an object called 'from_name', find all names referenced from such 504 an object according to the register of names. 505 """ 506 507 if self.attributes_used.has_key(from_name): 508 return 509 510 # Get name references and find possible objects which support such 511 # combinations of attribute names. 512 513 for names in self.name_references.get(from_name, []): 514 for objname in objtable.all_possible_objects(names): 515 for name in names: 516 self.use_attribute(objname, name) 517 self._collect_attributes(objname + "." + name, objtable) 518 519 # Get specific name references and visit the referenced objects. 520 521 for objname, attrname in self.specific_name_references.get(from_name, []): 522 self.use_attribute(objname, attrname) 523 self._collect_attributes(objname + "." + attrname, objtable) 524 525 # Constant accounting. 526 527 def init_predefined_constants(self): 528 529 "Ensure the predefined constants." 530 531 for name, value in self.predefined_constants.items(): 532 self.make_constant(value) 533 534 def get_predefined_constant(self, name): 535 536 "Return the predefined constant for the given 'name'." 537 538 return self.make_constant(self.predefined_constants[name]) 539 540 def get_constant(self, value): 541 542 "Return a constant for the given 'value'." 543 544 const = micropython.data.Const(value) 545 return self.constant_values[const] 546 547 def get_constant_type_name(self, value): 548 return value.__class__.__name__ 549 550 def make_constant(self, value): 551 552 "Make and return a constant for the given 'value'." 553 554 # Ensure the presence of the constant's type. 555 556 name = self.get_constant_type_name(value) 557 if self.modules.has_key("__builtins__"): 558 attr = self.modules["__builtins__"].get(name) 559 560 # Make a constant object and return it. 561 562 const = micropython.data.Const(value) 563 if not self.constant_values.has_key(const): 564 self.constant_values[const] = const 565 return self.constant_values[const] 566 567 def constants(self): 568 569 "Return a list of constants." 570 571 if self.constant_list is None: 572 self.constant_list = list(self.constant_values.values()) 573 574 return self.constant_list 575 576 # Import methods. 577 578 def find_in_path(self, name): 579 580 """ 581 Find the given module 'name' in the search path, returning None where no 582 such module could be found, or a 2-tuple from the 'find' method 583 otherwise. 584 """ 585 586 for d in self.path: 587 m = self.find(d, name) 588 if m: return m 589 return None 590 591 def find(self, d, name): 592 593 """ 594 In the directory 'd', find the given module 'name', where 'name' can 595 either refer to a single file module or to a package. Return None if the 596 'name' cannot be associated with either a file or a package directory, 597 or a 2-tuple from '_find_package' or '_find_module' otherwise. 598 """ 599 600 m = self._find_package(d, name) 601 if m: return m 602 m = self._find_module(d, name) 603 if m: return m 604 return None 605 606 def _find_module(self, d, name): 607 608 """ 609 In the directory 'd', find the given module 'name', returning None where 610 no suitable file exists in the directory, or a 2-tuple consisting of 611 None (indicating that no package directory is involved) and a filename 612 indicating the location of the module. 613 """ 614 615 name_py = name + os.extsep + "py" 616 filename = self._find_file(d, name_py) 617 if filename: 618 return None, filename 619 return None 620 621 def _find_package(self, d, name): 622 623 """ 624 In the directory 'd', find the given package 'name', returning None 625 where no suitable package directory exists, or a 2-tuple consisting of 626 a directory (indicating the location of the package directory itself) 627 and a filename indicating the location of the __init__.py module which 628 declares the package's top-level contents. 629 """ 630 631 filename = self._find_file(d, name) 632 if filename: 633 init_py = "__init__" + os.path.extsep + "py" 634 init_py_filename = self._find_file(filename, init_py) 635 if init_py_filename: 636 return filename, init_py_filename 637 return None 638 639 def _find_file(self, d, filename): 640 641 """ 642 Return the filename obtained when searching the directory 'd' for the 643 given 'filename', or None if no actual file exists for the filename. 644 """ 645 646 filename = os.path.join(d, filename) 647 if os.path.exists(filename): 648 return filename 649 else: 650 return None 651 652 def load(self, name, return_leaf=0): 653 654 """ 655 Load the module or package with the given 'name'. Return an object 656 referencing the loaded module or package, or None if no such module or 657 package exists. 658 """ 659 660 if self.modules.has_key(name) and self.modules[name].loaded: 661 #print "Cached (%s)" % name 662 return self.modules[name] 663 if self.verbose: 664 print "Loading", name 665 666 # Split the name into path components, and try to find the uppermost in 667 # the search path. 668 669 path = name.split(".") 670 m = self.find_in_path(path[0]) 671 if not m: 672 if self.verbose: 673 print "Not found (%s)" % path[0] 674 return None # NOTE: Import error. 675 d, filename = m 676 677 # Either acquire a reference to an already-imported module, or load the 678 # module from a file. 679 680 top = module = self.load_from_file(filename, path[0]) 681 682 # For hierarchical names, traverse each path component... 683 684 if len(path) > 1: 685 if not d: 686 if self.verbose: 687 print "No package (%s)" % filename 688 return None # NOTE: Import error (package not found). 689 else: 690 self.add_submodules(d, module) 691 692 path_so_far = path[:1] 693 for p in path[1:]: 694 path_so_far.append(p) 695 696 # Find the package or module concerned. 697 698 m = self.find(d, p) 699 if not m: 700 if self.verbose: 701 print "Not found (%s)" % p 702 return None # NOTE: Import error. 703 d, filename = m 704 module_name = ".".join(path_so_far) 705 706 # Either reference an imported module or load one from a file. 707 708 submodule = self.load_from_file(filename, module_name) 709 710 if d: 711 self.add_submodules(d, module) 712 713 # Store the submodule within its parent module. 714 715 module.set_module(p, submodule) 716 module = submodule 717 718 # Return either the deepest or the uppermost module. 719 720 if return_leaf: 721 return module 722 else: 723 return top 724 725 def load_from_file(self, name, module_name=None): 726 727 """ 728 Load the module with the given 'name' (which may be a full module path). 729 """ 730 731 if module_name is None: 732 module_name = "__main__" 733 734 module = self.add_module(module_name) 735 if not module.loaded and module not in self.loading: 736 self.loading.add(module) 737 #print "Parsing", name 738 module.parse(name) 739 #print "Done", name 740 self.loading.remove(module) 741 module.loaded = 1 742 743 # Record the module. 744 745 #print "Loaded", module_name, "with namespace", module.namespace.keys() 746 return module 747 748 def add_module(self, module_name): 749 750 """ 751 Return the module with the given 'module_name', adding a new module 752 object if one does not already exist. 753 """ 754 755 if not self.modules.has_key(module_name): 756 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 757 self.modules_ordered.append(module) 758 else: 759 module = self.modules[module_name] 760 return module 761 762 def add_submodules(self, pathname, module): 763 764 """ 765 Work around insufficient __all__ declarations and examine the directory 766 with the given 'pathname', adding submodules to the given 'module'. 767 """ 768 769 for filename in os.listdir(pathname): 770 submodule, ext = os.path.splitext(filename) 771 if ext not in ("", ".py"): 772 continue 773 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 774 775 # vim: tabstop=4 expandtab shiftwidth=4