1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.data 39 import micropython.opt 40 import micropython.inspect 41 import micropython.table 42 import os 43 try: 44 set 45 except NameError: 46 from sets import Set as set 47 48 class Program: 49 50 "This class supports the generation of a program image." 51 52 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 53 54 def __init__(self, importer, optimisations=None): 55 56 """ 57 Initialise the program representation with an 'importer' which is able 58 to locate and load Python modules. 59 60 The optional 'optimisations' cause certain techniques to be used in 61 reducing program size and improving program efficiency. 62 """ 63 64 self.importer = importer 65 self.optimisations = optimisations or set() 66 67 # Remember the tables once generated. 68 69 self.objtable = None 70 self.paramtable = None 71 self.clstable = None 72 73 # Main program information. 74 75 self.code = None 76 self.code_location = None 77 78 def get_importer(self): 79 return self.importer 80 81 # Access to finalised program information. 82 83 def get_image(self, with_builtins=0): 84 85 "Return a dictionary mapping modules to structures." 86 87 if self.code is not None: 88 return self.code 89 90 objtable = self.get_object_table() 91 paramtable = self.get_parameter_table() 92 self.importer.finalise() 93 94 self.code = [] 95 96 # Append constants to the image. 97 98 for pos, const in enumerate(self.importer.constants()): 99 self.code.append(const) 100 101 last_module = self.importer.modules_ordered[-1] 102 103 for module in self.importer.modules_ordered: 104 pos = len(self.code) 105 106 # Position the module in the image and make a translation. 107 108 trans = micropython.ast.Translation(module, self) 109 110 # Add header details. 111 112 self.code.append(module) 113 pos += 1 114 115 # Append module attributes to the image. 116 117 attributes = module.module_attributes() 118 self.code += module.attributes_as_list() 119 pos += len(attributes.keys()) 120 121 # Append classes and functions to the image. 122 123 for obj in module.all_objects: 124 if isinstance(obj, micropython.inspect.Class): 125 126 # Add header details. 127 128 self.code.append(obj) 129 pos += 1 130 131 # Append class attributes to the image. 132 133 attributes = obj.class_attributes() 134 self.code += obj.attributes_as_list() 135 pos += len(attributes.keys()) 136 137 # Generate the instantiator/initialiser. 138 # Append the function code to the image. 139 140 instantiator = obj.get_instantiator() 141 code = trans.get_instantiator_code(obj) 142 self.code += code 143 pos += len(code) 144 145 # Class-level code is generated separately at the module 146 # level, and the code location is set within the code 147 # generation process for the module. 148 149 elif isinstance(obj, micropython.inspect.Function): 150 151 # Add header details. 152 153 self.code.append(obj) 154 pos += 1 155 156 # Append any default values to the image. 157 # Only do this for named functions (not lambdas). 158 159 if obj.name is not None: 160 self.code += obj.default_attrs 161 pos += len(obj.default_attrs) 162 163 # Omit built-in function code where requested. 164 165 if not with_builtins and module.name == "__builtins__": 166 pass 167 168 # Append the function code to the image. 169 170 else: 171 code = trans.get_code(obj) 172 self.code += code 173 pos += len(code) 174 175 # Omit built-in module code where requested. 176 177 if not with_builtins and module.name == "__builtins__": 178 pass 179 180 # Append the module top-level code to the image. 181 182 else: 183 code = trans.get_module_code(final=(module is last_module)) 184 self.code += code 185 pos += len(code) 186 187 return self.code 188 189 def get_raw_image(self, with_builtins=0): 190 191 "Return the raw image representation of the program." 192 193 self.get_image(with_builtins) 194 195 objtable = self.get_object_table() 196 paramtable = self.get_parameter_table() 197 198 # Position the objects. 199 200 pos = 0 201 for item in self.code: 202 203 # Blocks are positioned leaving space for their expansion. 204 205 if isinstance(item, Block): 206 item.location = pos 207 pos += len(item.code) 208 209 # Other multi-location objects. 210 211 elif isinstance(item, ( 212 micropython.data.Class, micropython.data.Const, 213 micropython.data.Function, micropython.data.Module 214 )): 215 216 if isinstance(item, micropython.data.Class): 217 218 # Append a template of an instance for use when 219 # instantiating classes. 220 221 item.instance_template_location = pos 222 pos += 1 223 224 # Record the location of the object. 225 226 item.location = pos 227 pos += 1 228 229 # Code and details are associated with certain objects. 230 231 if isinstance(item, micropython.data.Function): 232 233 # Set the code location only where the code has been 234 # generated. 235 236 if not with_builtins and item.module.name == "__builtins__": 237 item.code_location = item.full_name() 238 239 # Skip any defaults. 240 241 else: 242 item.code_location = pos + len(item.defaults) 243 244 elif isinstance(item, micropython.data.Const): 245 pos += len(self.raw_data(item)) 246 247 else: 248 pos += 1 249 250 # Generate the raw code. 251 252 self.raw_code = [] 253 254 for item in self.code: 255 256 if isinstance(item, micropython.data.Attr): 257 self.raw_code.append(( 258 item.context and item.context.location, 259 item.value and item.value.location # no useful context is provided 260 )) 261 262 elif isinstance(item, Block): 263 self.raw_code += self.raw_block(item) 264 265 # Using classcode, attrcode, codeaddr, codedetails, instance. 266 267 elif isinstance(item, micropython.data.Class): 268 assert item.instance_template_location == len(self.raw_code) 269 270 classcode = objtable.as_list().get_code(item.full_name()) 271 attrcode = objtable.get_index(item.full_name()) 272 273 # Append a template of an instance for use when 274 # instantiating classes. 275 276 call_method = item.get("__call__") 277 call_method_code_location = call_method and call_method.value.code_location 278 279 self.raw_code.append( 280 DataObject( 281 classcode, 282 attrcode, 283 call_method_code_location, 284 ( 285 call_method and len(call_method.value.positional_names), 286 call_method and len(call_method.value.defaults) 287 ), 288 1, 289 item.full_name() 290 ) 291 ) 292 293 assert item.location == len(self.raw_code) 294 295 # NOTE: The instantiator code is the first block of the class. 296 297 instantiator_code_location = item.get_instantiator().blocks[0].location 298 299 # NOTE: Need initialiser details! 300 self.raw_code.append( 301 DataObject( 302 classcode, 303 attrcode, 304 instantiator_code_location, 305 ( 306 len(item.get_instantiator().positional_names), 307 len(item.get_instantiator().defaults) 308 ), 309 0, 310 item.full_name() 311 ) 312 ) 313 314 elif isinstance(item, micropython.data.Const): 315 assert item.location == len(self.raw_code) 316 317 # NOTE: Need class details! 318 self.raw_code.append( 319 DataObject( 320 objtable.as_list().get_code(item.value_type_name()), 321 objtable.get_index(item.value_type_name()), 322 None, 323 None, 324 1, 325 item.value_type_name() 326 ) 327 ) 328 329 self.raw_code += self.raw_data(item) 330 331 elif isinstance(item, micropython.data.Function): 332 assert item.location == len(self.raw_code) 333 334 # NOTE: Need class and parameter details! Should arguably be types.FunctionType. 335 self.raw_code.append( 336 DataObject( 337 objtable.as_list().get_code("__builtins__.function"), 338 objtable.get_index("__builtins__.function"), 339 item.code_location, 340 ( 341 len(item.positional_names), 342 len(item.defaults) 343 ), 344 0, 345 "__builtins__.function" 346 ) 347 ) 348 349 # Check the code location only where the code has been generated. 350 351 assert (not with_builtins and item.module.name == "__builtins__") or \ 352 item.code_location == len(self.raw_code) + len(item.defaults) 353 354 elif isinstance(item, micropython.data.Module): 355 assert item.location == len(self.raw_code) 356 357 self.raw_code.append( 358 DataObject( 359 objtable.as_list().get_code(item.full_name()), 360 None, # module name not used as an attribute 361 None, 362 None, 363 0, 364 item.full_name() 365 ) 366 ) 367 368 else: 369 self.raw_code.append(item) 370 371 # Fix the module locations. 372 373 for module in self.importer.modules_ordered: 374 375 if not with_builtins and module.name == "__builtins__": 376 continue 377 378 module.code_location = module.blocks[0].location 379 380 self.code_location = self.importer.modules["__main__"].code_location 381 return self.raw_code 382 383 def raw_block(self, block): 384 385 "Return the code for the given 'block'." 386 387 assert block.location == len(self.raw_code) 388 for i, item in enumerate(block.code): 389 if hasattr(item, "location"): 390 item.location = location + i 391 return block.code 392 393 def raw_data(self, item): 394 395 "Return the data for the given 'item'." 396 397 datatype = item.value_type_name() 398 399 # NOTE: Start simple and use single entries for most types. 400 401 if datatype in ("__builtins__.tuple", "__builtins__.list"): 402 return [len(item.value)] + list(item.value) 403 else: 404 return [item.value] 405 406 def get_object_table(self): 407 408 "Return a table with details of attributes for classes and modules." 409 410 if self.objtable is None: 411 self.importer.vacuum() 412 413 t = self.objtable = micropython.table.ObjectTable() 414 for module in self.importer.get_modules(): 415 t.add(module.full_name(), module.module_attributes()) 416 417 # Add class and instance attributes for all classes, together 418 # with descendant information. 419 420 for obj in module.all_objects: 421 if isinstance(obj, micropython.inspect.Class): 422 attributes = {obj.full_name() : obj} 423 attributes.update(obj.all_attributes()) 424 attributes.update(obj.all_descendants()) 425 t.add(obj.full_name(), attributes) 426 427 return self.objtable 428 429 def get_parameter_table(self): 430 431 "Return a table with details of parameters for functions and methods." 432 433 # Need the object table to get at class details. 434 435 objtable = self.get_object_table() 436 437 if self.paramtable is None: 438 t = self.paramtable = micropython.table.ParameterTable() 439 440 # Visit each module, getting function and method details. 441 442 for module in self.importer.get_modules(): 443 for obj in module.all_objects: 444 if isinstance(obj, micropython.inspect.Function): 445 t.add(obj.full_name(), obj.parameters()) 446 447 # Classes are callable, too. 448 # Take details of the appropriate __init__ method to make an 449 # entry for an instantiation function for the class. 450 451 elif isinstance(obj, micropython.inspect.Class): 452 t.add(obj.full_name(), obj.get_instantiator().parameters()) 453 454 # Filter out all parameter table entries not referenced by keyword 455 # arguments. 456 457 keyword_names = set() 458 459 for module in self.importer.get_modules(): 460 keyword_names.update(module.keyword_names) 461 462 for function_name, parameters in t.table.items(): 463 for name in parameters.keys(): 464 if name in keyword_names: 465 break 466 else: 467 del t.table[function_name] 468 469 return self.paramtable 470 471 class Importer: 472 473 "An import machine, searching for and loading modules." 474 475 predefined_constants = { 476 "None" : None, 477 "True" : True, 478 "False" : False, 479 "Ellipsis" : Ellipsis, 480 "NotImplemented" : NotImplemented 481 } 482 483 def __init__(self, path=None, verbose=0, optimisations=None): 484 485 """ 486 Initialise the importer with the given search 'path' - a list of 487 directories to search for Python modules. 488 489 The optional 'verbose' parameter causes output concerning the activities 490 of the object to be produced if set to a true value (not the default). 491 492 The optional 'optimisations' cause certain techniques to be used in 493 reducing program size and improving program efficiency. 494 """ 495 496 self.path = path or [os.getcwd()] 497 self.verbose = verbose 498 self.optimisations = optimisations or set() 499 500 self.modules = {} 501 self.modules_ordered = [] 502 self.loading = set() 503 504 # Constant records. 505 506 self.constant_values = {} 507 self.constant_list = None # cache for constants 508 self.init_predefined_constants() 509 510 # Name records (used to track actual use of names). 511 # Include names which may not be explicitly used in programs. 512 513 self.names_used = set(["__init__", "__call__"]) 514 515 # Status information. 516 517 self.vacuumed = 0 518 self.finalised = 0 519 520 def get_modules(self): 521 522 "Return all modules known to the importer." 523 524 return self.modules.values() 525 526 # General maintenance. 527 528 def vacuum(self): 529 530 "Tidy up the modules." 531 532 if self.vacuumed: 533 return 534 535 for name, module in self.modules.items(): 536 if module.loaded: 537 module.vacuum() 538 else: 539 del self.modules[name] 540 541 self.vacuumed = 1 542 543 def finalise(self): 544 545 "Finalise the program." 546 547 if self.finalised: 548 return 549 550 self.vacuum() 551 552 for module in self.get_modules(): 553 module.finalise() 554 555 self.finalised = 1 556 557 # Name accounting. 558 559 def use_name(self, name): 560 561 "Register the given 'name' as being used in the program." 562 563 self.names_used.add(name) 564 565 # Constant accounting. 566 567 def init_predefined_constants(self): 568 569 "Ensure the predefined constants." 570 571 for name, value in self.predefined_constants.items(): 572 self.make_constant(value) 573 574 def get_predefined_constant(self, name): 575 576 "Return the predefined constant for the given 'name'." 577 578 return self.make_constant(self.predefined_constants[name]) 579 580 def get_constant(self, value): 581 582 "Return a constant for the given 'value'." 583 584 const = micropython.data.Const(value) 585 return self.constant_values[const] 586 587 def make_constant(self, value): 588 589 "Make and return a constant for the given 'value'." 590 591 const = micropython.data.Const(value) 592 if not self.constant_values.has_key(const): 593 self.constant_values[const] = const 594 return self.constant_values[const] 595 596 def constants(self): 597 598 "Return a list of constants." 599 600 if self.constant_list is None: 601 self.constant_list = list(self.constant_values.values()) 602 603 return self.constant_list 604 605 # Import methods. 606 607 def find_in_path(self, name): 608 609 """ 610 Find the given module 'name' in the search path, returning None where no 611 such module could be found, or a 2-tuple from the 'find' method 612 otherwise. 613 """ 614 615 for d in self.path: 616 m = self.find(d, name) 617 if m: return m 618 return None 619 620 def find(self, d, name): 621 622 """ 623 In the directory 'd', find the given module 'name', where 'name' can 624 either refer to a single file module or to a package. Return None if the 625 'name' cannot be associated with either a file or a package directory, 626 or a 2-tuple from '_find_package' or '_find_module' otherwise. 627 """ 628 629 m = self._find_package(d, name) 630 if m: return m 631 m = self._find_module(d, name) 632 if m: return m 633 return None 634 635 def _find_module(self, d, name): 636 637 """ 638 In the directory 'd', find the given module 'name', returning None where 639 no suitable file exists in the directory, or a 2-tuple consisting of 640 None (indicating that no package directory is involved) and a filename 641 indicating the location of the module. 642 """ 643 644 name_py = name + os.extsep + "py" 645 filename = self._find_file(d, name_py) 646 if filename: 647 return None, filename 648 return None 649 650 def _find_package(self, d, name): 651 652 """ 653 In the directory 'd', find the given package 'name', returning None 654 where no suitable package directory exists, or a 2-tuple consisting of 655 a directory (indicating the location of the package directory itself) 656 and a filename indicating the location of the __init__.py module which 657 declares the package's top-level contents. 658 """ 659 660 filename = self._find_file(d, name) 661 if filename: 662 init_py = "__init__" + os.path.extsep + "py" 663 init_py_filename = self._find_file(filename, init_py) 664 if init_py_filename: 665 return filename, init_py_filename 666 return None 667 668 def _find_file(self, d, filename): 669 670 """ 671 Return the filename obtained when searching the directory 'd' for the 672 given 'filename', or None if no actual file exists for the filename. 673 """ 674 675 filename = os.path.join(d, filename) 676 if os.path.exists(filename): 677 return filename 678 else: 679 return None 680 681 def load(self, name, return_leaf=0): 682 683 """ 684 Load the module or package with the given 'name'. Return an object 685 referencing the loaded module or package, or None if no such module or 686 package exists. 687 """ 688 689 if self.modules.has_key(name) and self.modules[name].loaded: 690 #print "Cached (%s)" % name 691 return self.modules[name] 692 if self.verbose: 693 print "Loading", name 694 695 # Split the name into path components, and try to find the uppermost in 696 # the search path. 697 698 path = name.split(".") 699 m = self.find_in_path(path[0]) 700 if not m: 701 if self.verbose: 702 print "Not found (%s)" % path[0] 703 return None # NOTE: Import error. 704 d, filename = m 705 706 # Either acquire a reference to an already-imported module, or load the 707 # module from a file. 708 709 top = module = self.load_from_file(filename, path[0]) 710 711 # For hierarchical names, traverse each path component... 712 713 if len(path) > 1: 714 if not d: 715 if self.verbose: 716 print "No package (%s)" % filename 717 return None # NOTE: Import error (package not found). 718 else: 719 self.add_submodules(d, module) 720 721 path_so_far = path[:1] 722 for p in path[1:]: 723 path_so_far.append(p) 724 725 # Find the package or module concerned. 726 727 m = self.find(d, p) 728 if not m: 729 if self.verbose: 730 print "Not found (%s)" % p 731 return None # NOTE: Import error. 732 d, filename = m 733 module_name = ".".join(path_so_far) 734 735 # Either reference an imported module or load one from a file. 736 737 submodule = self.load_from_file(filename, module_name) 738 739 if d: 740 self.add_submodules(d, module) 741 742 # Store the submodule within its parent module. 743 744 module.set_module(p, submodule) 745 module = submodule 746 747 # Return either the deepest or the uppermost module. 748 749 if return_leaf: 750 return module 751 else: 752 return top 753 754 def load_from_file(self, name, module_name=None): 755 756 """ 757 Load the module with the given 'name' (which may be a full module path). 758 """ 759 760 if module_name is None: 761 module_name = "__main__" 762 763 module = self.add_module(module_name) 764 if not module.loaded and module not in self.loading: 765 self.loading.add(module) 766 #print "Parsing", name 767 module.parse(name) 768 #print "Done", name 769 self.loading.remove(module) 770 module.loaded = 1 771 772 # Record the module. 773 774 #print "Loaded", module_name, "with namespace", module.namespace.keys() 775 return module 776 777 def add_module(self, module_name): 778 779 """ 780 Return the module with the given 'module_name', adding a new module 781 object if one does not already exist. 782 """ 783 784 if not self.modules.has_key(module_name): 785 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 786 self.modules_ordered.append(module) 787 else: 788 module = self.modules[module_name] 789 return module 790 791 def add_submodules(self, pathname, module): 792 793 """ 794 Work around insufficient __all__ declarations and examine the directory 795 with the given 'pathname', adding submodules to the given 'module'. 796 """ 797 798 for filename in os.listdir(pathname): 799 submodule, ext = os.path.splitext(filename) 800 if ext not in ("", ".py"): 801 continue 802 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 803 804 # vim: tabstop=4 expandtab shiftwidth=4