1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed. Here, the standard path 26 for module searching is employed: 27 28 importer = Importer(sys.path) 29 30 To generate programs, the above importer should be supplied in the 31 initialisation of a program instance, and then various methods are called: 32 33 program = Program(importer) 34 image = program.get_raw_image() 35 36 Such importer and program objects are the most convenient mechanism through 37 which the functionality of the micropython package may be accessed. 38 """ 39 40 from micropython.common import ProcessingError, TableGenerationError 41 import micropython.ast 42 import micropython.data 43 import micropython.opt 44 import micropython.inspect 45 import micropython.table 46 import os 47 48 try: 49 set 50 except NameError: 51 from sets import Set as set 52 53 class Program: 54 55 "This class supports the generation of a program image." 56 57 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 58 59 def __init__(self, importer, optimisations=None): 60 61 """ 62 Initialise the program representation with an 'importer' which is able 63 to locate and load Python modules. 64 65 The optional 'optimisations' cause certain techniques to be used in 66 reducing program size and improving program efficiency. 67 """ 68 69 self.importer = importer 70 self.optimisations = optimisations or set() 71 72 # Remember the tables once generated. 73 74 self.objtable = None 75 self.paramtable = None 76 77 # Main program information. 78 79 self.code = None 80 self.code_location = None 81 82 def get_importer(self): 83 return self.importer 84 85 # Access to finalised program information. 86 87 def get_image(self, with_builtins=0): 88 89 """ 90 Return the program image including built-in objects if 'with_builtins' 91 is specified and set to a true value. 92 """ 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 100 # Optimise and regenerate the object table. 101 102 self.importer.vacuum(objtable) 103 objtable = self.get_object_table(reset=1) 104 105 # Finalise the importer. 106 107 self.importer.finalise() 108 109 self.code = [] 110 111 # Append constants to the image. 112 113 for const in self.importer.constants(): 114 self.code.append(const) 115 116 last_module = self.importer.modules_ordered[-1] 117 118 for module in self.importer.modules_ordered: 119 suppress_builtins = not with_builtins and module.name == "__builtins__" 120 121 # Position the module in the image and make a translation. 122 123 trans = micropython.ast.Translation(module, self) 124 125 # Add header details. 126 127 self.code.append(module) 128 129 # Append module attributes to the image. 130 131 attributes = module.module_attributes() 132 self.code += module.attributes_as_list() 133 134 # Append classes and functions to the image. 135 136 for obj in module.all_objects: 137 if isinstance(obj, micropython.inspect.Class): 138 139 # Add header details. 140 141 self.code.append(obj) 142 143 # Append class attributes to the image. 144 145 attributes = obj.class_attributes() 146 self.code += obj.attributes_as_list() 147 148 # Omit built-in function code where requested. 149 150 if suppress_builtins and obj.astnode.doc is None: 151 continue 152 153 # Generate the instantiator/initialiser. 154 # Append the function code to the image. 155 156 instantiator = obj.get_instantiator() 157 code = trans.get_instantiator_code(obj) 158 self.code += code 159 160 # Class-level code is generated separately at the module 161 # level, and the code location is set within the code 162 # generation process for the module. 163 164 elif isinstance(obj, micropython.inspect.Function): 165 166 # Add header details. 167 168 self.code.append(obj) 169 170 # Append any default values to the image. 171 # Only do this for named functions (not lambdas). 172 173 if obj.name is not None: 174 self.code += obj.default_attrs 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and obj.astnode.doc is None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 187 # Omit built-in module code where requested. 188 189 if suppress_builtins: 190 pass 191 192 # Append the module top-level code to the image. 193 194 else: 195 code = trans.get_module_code() 196 self.code += code 197 198 return self.code 199 200 def get_raw_image(self, architecture=None, with_builtins=0): 201 202 "Return the raw image representation of the program." 203 204 architecture = architecture or micropython.rsvp 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 215 for item in self.code: 216 arch_item = architecture.get_object(item) 217 218 # Get the raw version for the architecture. 219 220 if arch_item is not None: 221 pos = arch_item.set_location(pos, with_builtins) 222 else: 223 pos += 1 224 225 # Generate the raw code. 226 227 self.raw_code = [] 228 229 for item in self.code: 230 arch_item = architecture.get_object(item) 231 232 # Get the raw version for the architecture. 233 234 if arch_item is not None: 235 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 236 arch_item.finalise_location(with_builtins) 237 else: 238 self.raw_code.append(item) 239 240 # Fix the module locations. 241 242 for module in self.importer.modules_ordered: 243 244 if not with_builtins and module.name == "__builtins__": 245 continue 246 247 module.code_location = module.blocks[0].location 248 249 self.code_location = self.importer.modules["__main__"].code_location 250 return self.raw_code 251 252 def get_object_table(self, reset=0): 253 254 "Return a table with details of attributes for classes and modules." 255 256 if self.objtable is None or reset: 257 258 t = self.objtable = micropython.table.ObjectTable() 259 for module in self.importer.get_modules(): 260 t.add(module.full_name(), module.module_attributes()) 261 262 # Add class and instance attributes for all classes, together 263 # with descendant information. 264 265 for obj in module.all_objects: 266 if isinstance(obj, micropython.inspect.Class): 267 268 # Prevent ambiguous classes. 269 270 full_name = obj.full_name() 271 name = obj.name 272 273 if module.has_key(name) and module[name].defines_ambiguous_class(): 274 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 275 276 # Define a table entry for the class. 277 278 attributes = {full_name : obj} 279 attributes.update(obj.all_attributes()) 280 attributes.update(obj.all_descendants()) 281 t.add(full_name, attributes) 282 283 return self.objtable 284 285 def get_parameter_table(self): 286 287 "Return a table with details of parameters for functions and methods." 288 289 # Need the object table to get at class details. 290 291 objtable = self.get_object_table() 292 293 if self.paramtable is None: 294 t = self.paramtable = micropython.table.ParameterTable() 295 296 # Visit each module, getting function and method details. 297 298 for module in self.importer.get_modules(): 299 for obj in module.all_objects: 300 if isinstance(obj, micropython.inspect.Function): 301 t.add(obj.full_name(), obj.parameters()) 302 303 # Classes are callable, too. 304 # Take details of the appropriate __init__ method to make an 305 # entry for an instantiation function for the class. 306 307 elif isinstance(obj, micropython.inspect.Class): 308 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 309 310 # Filter out all parameter table entries not referenced by keyword 311 # arguments. 312 313 keyword_names = set() 314 315 for module in self.importer.get_modules(): 316 keyword_names.update(module.keyword_names) 317 318 for function_name, parameters in t.table.items(): 319 for name in parameters.keys(): 320 if name in keyword_names: 321 break 322 else: 323 del t.table[function_name] 324 325 return self.paramtable 326 327 class Importer: 328 329 "An import machine, searching for and loading modules." 330 331 predefined_constants = { 332 "None" : None, 333 "True" : True, 334 "False" : False, 335 #"Ellipsis" : Ellipsis, 336 "NotImplemented" : NotImplemented 337 } 338 339 names_always_used = [ 340 "bool", "__init__", "__call__", "__bool__", "__builtins__", "__main__" 341 ] 342 343 def __init__(self, path=None, verbose=0, optimisations=None): 344 345 """ 346 Initialise the importer with the given search 'path' - a list of 347 directories to search for Python modules. 348 349 The optional 'verbose' parameter causes output concerning the activities 350 of the object to be produced if set to a true value (not the default). 351 352 The optional 'optimisations' cause certain techniques to be used in 353 reducing program size and improving program efficiency. 354 """ 355 356 self.path = path or [os.getcwd()] 357 self.verbose = verbose 358 self.optimisations = optimisations or set() 359 360 self.modules = {} 361 self.modules_ordered = [] 362 self.loading = set() 363 364 # Constant records. 365 366 self.constant_values = {} 367 self.constant_list = None # cache for constants 368 self.init_predefined_constants() 369 370 # Attribute usage. 371 372 self.attributes_used = None 373 self.name_references = {} 374 375 # Status information. 376 377 self.vacuumed = 0 378 self.finalised = 0 379 380 def get_modules(self): 381 382 "Return all modules known to the importer." 383 384 return self.modules.values() 385 386 def get_module(self, name): 387 388 "Return the module with the given 'name'." 389 390 return self.modules[name] 391 392 # General maintenance. 393 394 def vacuum(self, objtable): 395 396 "Tidy up the modules." 397 398 if self.vacuumed: 399 return 400 401 # Complete the list of attribute names used in the program. 402 403 self.collect_attributes(objtable) 404 405 for name, module in self.modules.items(): 406 if module.loaded: 407 module.vacuum() 408 else: 409 del self.modules[name] 410 411 self.vacuumed = 1 412 413 def finalise(self): 414 415 "Finalise the program (which should have been vacuumed first)." 416 417 if self.finalised: 418 return 419 420 for module in self.get_modules(): 421 module.finalise() 422 423 self.finalised = 1 424 425 # Name accounting. 426 427 def use_name(self, name, from_name): 428 429 """ 430 Register the given 'name' as being used in the program from within an 431 object with the specified 'from_name'. 432 """ 433 434 if not self.name_references.has_key(from_name): 435 self.name_references[from_name] = set() 436 self.name_references[from_name].add((name,)) 437 438 def use_names(self, names, from_name): 439 440 """ 441 Register the given 'names' as being used in the program from within an 442 object with the specified 'from_name'. 443 """ 444 445 if not self.name_references.has_key(from_name): 446 self.name_references[from_name] = set() 447 self.name_references[from_name].add(names) 448 449 def uses_attribute(self, obj, name): 450 451 """ 452 Return whether the attribute of object 'obj' having the given 'name' is 453 used as an attribute in the program. 454 """ 455 456 return name in self.attributes_used.get(obj.full_name(), []) 457 458 def use_attribute(self, objname, name): 459 460 """ 461 Indicate that in the object with the given 'objname', the attribute of 462 the given 'name' is used. 463 """ 464 465 if not self.attributes_used.has_key(objname): 466 self.attributes_used[objname] = set() 467 self.attributes_used[objname].add(name) 468 469 def collect_attributes(self, objtable): 470 471 "Collect attribute references for the entire program." 472 473 if self.attributes_used is None: 474 475 # Include names which may not be explicitly used in programs. 476 # NOTE: Potentially declare these when inspecting. 477 478 self.attributes_used = {} 479 for name in self.names_always_used: 480 for objname in objtable.all_possible_objects([name]): 481 self.use_attribute(objname, name) 482 483 # Start with the "root" modules, finding referenced objects. 484 485 self._collect_attributes("__builtins__", objtable) 486 self._collect_attributes("__main__", objtable) 487 488 def _collect_attributes(self, from_name, objtable): 489 490 """ 491 Given an object called 'from_name', find all names referenced from such 492 an object according to the register of names. 493 """ 494 495 if self.attributes_used.has_key(from_name): 496 return 497 498 for names in self.name_references.get(from_name, []): 499 for objname in objtable.all_possible_objects(names): 500 for name in names: 501 self.use_attribute(objname, name) 502 self._collect_attributes(objname + "." + name, objtable) 503 504 # Constant accounting. 505 506 def init_predefined_constants(self): 507 508 "Ensure the predefined constants." 509 510 for name, value in self.predefined_constants.items(): 511 self.make_constant(value) 512 513 def get_predefined_constant(self, name): 514 515 "Return the predefined constant for the given 'name'." 516 517 return self.make_constant(self.predefined_constants[name]) 518 519 def get_constant(self, value): 520 521 "Return a constant for the given 'value'." 522 523 const = micropython.data.Const(value) 524 return self.constant_values[const] 525 526 def get_constant_type_name(self, value): 527 return value.__class__.__name__ 528 529 def make_constant(self, value): 530 531 "Make and return a constant for the given 'value'." 532 533 # Ensure the presence of the constant's type. 534 535 name = self.get_constant_type_name(value) 536 if self.modules.has_key("__builtins__"): 537 attr = self.modules["__builtins__"].get(name) 538 539 # Make a constant object and return it. 540 541 const = micropython.data.Const(value) 542 if not self.constant_values.has_key(const): 543 self.constant_values[const] = const 544 return self.constant_values[const] 545 546 def constants(self): 547 548 "Return a list of constants." 549 550 if self.constant_list is None: 551 self.constant_list = list(self.constant_values.values()) 552 553 return self.constant_list 554 555 # Import methods. 556 557 def find_in_path(self, name): 558 559 """ 560 Find the given module 'name' in the search path, returning None where no 561 such module could be found, or a 2-tuple from the 'find' method 562 otherwise. 563 """ 564 565 for d in self.path: 566 m = self.find(d, name) 567 if m: return m 568 return None 569 570 def find(self, d, name): 571 572 """ 573 In the directory 'd', find the given module 'name', where 'name' can 574 either refer to a single file module or to a package. Return None if the 575 'name' cannot be associated with either a file or a package directory, 576 or a 2-tuple from '_find_package' or '_find_module' otherwise. 577 """ 578 579 m = self._find_package(d, name) 580 if m: return m 581 m = self._find_module(d, name) 582 if m: return m 583 return None 584 585 def _find_module(self, d, name): 586 587 """ 588 In the directory 'd', find the given module 'name', returning None where 589 no suitable file exists in the directory, or a 2-tuple consisting of 590 None (indicating that no package directory is involved) and a filename 591 indicating the location of the module. 592 """ 593 594 name_py = name + os.extsep + "py" 595 filename = self._find_file(d, name_py) 596 if filename: 597 return None, filename 598 return None 599 600 def _find_package(self, d, name): 601 602 """ 603 In the directory 'd', find the given package 'name', returning None 604 where no suitable package directory exists, or a 2-tuple consisting of 605 a directory (indicating the location of the package directory itself) 606 and a filename indicating the location of the __init__.py module which 607 declares the package's top-level contents. 608 """ 609 610 filename = self._find_file(d, name) 611 if filename: 612 init_py = "__init__" + os.path.extsep + "py" 613 init_py_filename = self._find_file(filename, init_py) 614 if init_py_filename: 615 return filename, init_py_filename 616 return None 617 618 def _find_file(self, d, filename): 619 620 """ 621 Return the filename obtained when searching the directory 'd' for the 622 given 'filename', or None if no actual file exists for the filename. 623 """ 624 625 filename = os.path.join(d, filename) 626 if os.path.exists(filename): 627 return filename 628 else: 629 return None 630 631 def load(self, name, return_leaf=0): 632 633 """ 634 Load the module or package with the given 'name'. Return an object 635 referencing the loaded module or package, or None if no such module or 636 package exists. 637 """ 638 639 if self.modules.has_key(name) and self.modules[name].loaded: 640 #print "Cached (%s)" % name 641 return self.modules[name] 642 if self.verbose: 643 print "Loading", name 644 645 # Split the name into path components, and try to find the uppermost in 646 # the search path. 647 648 path = name.split(".") 649 m = self.find_in_path(path[0]) 650 if not m: 651 if self.verbose: 652 print "Not found (%s)" % path[0] 653 return None # NOTE: Import error. 654 d, filename = m 655 656 # Either acquire a reference to an already-imported module, or load the 657 # module from a file. 658 659 top = module = self.load_from_file(filename, path[0]) 660 661 # For hierarchical names, traverse each path component... 662 663 if len(path) > 1: 664 if not d: 665 if self.verbose: 666 print "No package (%s)" % filename 667 return None # NOTE: Import error (package not found). 668 else: 669 self.add_submodules(d, module) 670 671 path_so_far = path[:1] 672 for p in path[1:]: 673 path_so_far.append(p) 674 675 # Find the package or module concerned. 676 677 m = self.find(d, p) 678 if not m: 679 if self.verbose: 680 print "Not found (%s)" % p 681 return None # NOTE: Import error. 682 d, filename = m 683 module_name = ".".join(path_so_far) 684 685 # Either reference an imported module or load one from a file. 686 687 submodule = self.load_from_file(filename, module_name) 688 689 if d: 690 self.add_submodules(d, module) 691 692 # Store the submodule within its parent module. 693 694 module.set_module(p, submodule) 695 module = submodule 696 697 # Return either the deepest or the uppermost module. 698 699 if return_leaf: 700 return module 701 else: 702 return top 703 704 def load_from_file(self, name, module_name=None): 705 706 """ 707 Load the module with the given 'name' (which may be a full module path). 708 """ 709 710 if module_name is None: 711 module_name = "__main__" 712 713 module = self.add_module(module_name) 714 if not module.loaded and module not in self.loading: 715 self.loading.add(module) 716 #print "Parsing", name 717 module.parse(name) 718 #print "Done", name 719 self.loading.remove(module) 720 module.loaded = 1 721 722 # Record the module. 723 724 #print "Loaded", module_name, "with namespace", module.namespace.keys() 725 return module 726 727 def add_module(self, module_name): 728 729 """ 730 Return the module with the given 'module_name', adding a new module 731 object if one does not already exist. 732 """ 733 734 if not self.modules.has_key(module_name): 735 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 736 self.modules_ordered.append(module) 737 else: 738 module = self.modules[module_name] 739 return module 740 741 def add_submodules(self, pathname, module): 742 743 """ 744 Work around insufficient __all__ declarations and examine the directory 745 with the given 'pathname', adding submodules to the given 'module'. 746 """ 747 748 for filename in os.listdir(pathname): 749 submodule, ext = os.path.splitext(filename) 750 if ext not in ("", ".py"): 751 continue 752 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 753 754 # vim: tabstop=4 expandtab shiftwidth=4