1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.data 39 import micropython.opt 40 import micropython.inspect 41 import micropython.table 42 import os 43 try: 44 set 45 except NameError: 46 from sets import Set as set 47 48 class Program: 49 50 "This class supports the generation of a program image." 51 52 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 53 54 def __init__(self, importer, optimisations=None): 55 56 """ 57 Initialise the program representation with an 'importer' which is able 58 to locate and load Python modules. 59 60 The optional 'optimisations' cause certain techniques to be used in 61 reducing program size and improving program efficiency. 62 """ 63 64 self.importer = importer 65 self.optimisations = optimisations or set() 66 67 # Remember the tables once generated. 68 69 self.objtable = None 70 self.paramtable = None 71 self.clstable = None 72 73 # Main program information. 74 75 self.code = None 76 self.code_location = None 77 78 def get_importer(self): 79 return self.importer 80 81 # Access to finalised program information. 82 83 def get_image(self, with_builtins=0): 84 85 "Return a dictionary mapping modules to structures." 86 87 if self.code is not None: 88 return self.code 89 90 objtable = self.get_object_table() 91 paramtable = self.get_parameter_table() 92 self.importer.finalise() 93 94 self.code = [] 95 96 # Append constants to the image. 97 98 for pos, const in enumerate(self.importer.constants()): 99 self.code.append(const) 100 101 last_module = self.importer.modules_ordered[-1] 102 103 for module in self.importer.modules_ordered: 104 pos = len(self.code) 105 106 # Position the module in the image and make a translation. 107 108 trans = micropython.ast.Translation(module, self) 109 110 # Add header details. 111 112 self.code.append(module) 113 pos += 1 114 115 # Append module attributes to the image. 116 117 attributes = module.module_attributes() 118 self.code += module.attributes_as_list() 119 pos += len(attributes.keys()) 120 121 # Append classes and functions to the image. 122 123 for obj in module.all_objects: 124 if isinstance(obj, micropython.inspect.Class): 125 126 # Add header details. 127 128 self.code.append(obj) 129 pos += 1 130 131 # Append class attributes to the image. 132 133 attributes = obj.class_attributes() 134 self.code += obj.attributes_as_list() 135 pos += len(attributes.keys()) 136 137 # Generate the instantiator/initialiser. 138 # Append the function code to the image. 139 140 instantiator = obj.get_instantiator() 141 code = trans.get_instantiator_code(obj) 142 self.code += code 143 pos += len(code) 144 145 # Class-level code is generated separately at the module 146 # level, and the code location is set within the code 147 # generation process for the module. 148 149 elif isinstance(obj, micropython.inspect.Function): 150 151 # Add header details. 152 153 self.code.append(obj) 154 pos += 1 155 156 # Append any default values to the image. 157 # Only do this for named functions (not lambdas). 158 159 if obj.name is not None: 160 self.code += obj.default_attrs 161 pos += len(obj.default_attrs) 162 163 # Omit built-in function code where requested. 164 165 if not with_builtins and module.name == "__builtins__": 166 pass 167 168 # Append the function code to the image. 169 170 else: 171 code = trans.get_code(obj) 172 self.code += code 173 pos += len(code) 174 175 # Omit built-in module code where requested. 176 177 if not with_builtins and module.name == "__builtins__": 178 pass 179 180 # Append the module top-level code to the image. 181 182 else: 183 code = trans.get_module_code(final=(module is last_module)) 184 self.code += code 185 pos += len(code) 186 187 return self.code 188 189 def get_raw_image(self, with_builtins=0): 190 191 "Return the raw image representation of the program." 192 193 self.get_image(with_builtins) 194 195 objtable = self.get_object_table() 196 paramtable = self.get_parameter_table() 197 198 # Position the objects. 199 200 pos = 0 201 for item in self.code: 202 203 # Blocks are positioned leaving space for their expansion. 204 205 if isinstance(item, Block): 206 item.location = pos 207 pos += len(item.code) 208 209 # Other multi-location objects. 210 211 elif isinstance(item, (micropython.data.Class, micropython.data.Const, 212 micropython.data.Function, micropython.data.Module)): 213 214 item.location = pos 215 pos += 1 216 217 # Code and details are associated with certain objects. 218 219 if isinstance(item, micropython.data.Function): 220 221 # Set the code location only where the code has been 222 # generated. 223 224 if not with_builtins and item.module.name == "__builtins__": 225 item.code_location = item.full_name() 226 227 # Skip any defaults. 228 229 else: 230 item.code_location = pos + len(item.defaults) 231 232 elif isinstance(item, micropython.data.Const): 233 pos += len(self.raw_data(item)) 234 235 else: 236 pos += 1 237 238 # Generate the raw code. 239 240 self.raw_code = [] 241 242 for item in self.code: 243 244 if isinstance(item, micropython.data.Attr): 245 self.raw_code.append(( 246 item.context and item.context.location, 247 item.value and item.value.location # no useful context is provided 248 )) 249 250 elif isinstance(item, Block): 251 self.raw_code += self.raw_block(item) 252 253 # Using classcode, attrcode, codeaddr, codedetails, instance. 254 255 elif isinstance(item, micropython.data.Class): 256 assert item.location == len(self.raw_code) 257 258 # NOTE: The instantiator code is the first block of the class. 259 260 instantiator_code_location = item.get_instantiator().blocks[0].location 261 262 # NOTE: Need initialiser details! 263 self.raw_code.append( 264 DataObject( 265 objtable.as_list().get_code(item.full_name()), 266 objtable.get_index(item.full_name()), 267 instantiator_code_location, 268 ( 269 len(item.get_instantiator().positional_names), 270 len(item.get_instantiator().defaults) 271 ), 272 0, 273 item.full_name() 274 ) 275 ) 276 277 elif isinstance(item, micropython.data.Const): 278 assert item.location == len(self.raw_code) 279 280 # NOTE: Need class details! 281 self.raw_code.append( 282 DataObject( 283 objtable.as_list().get_code(item.value_type_name()), 284 objtable.get_index(item.value_type_name()), 285 None, 286 None, 287 1, 288 item.value_type_name() 289 ) 290 ) 291 292 self.raw_code += self.raw_data(item) 293 294 elif isinstance(item, micropython.data.Function): 295 assert item.location == len(self.raw_code) 296 297 # NOTE: Need class and parameter details! Should arguably be types.FunctionType. 298 self.raw_code.append( 299 DataObject( 300 objtable.as_list().get_code("__builtins__.function"), 301 objtable.get_index("__builtins__.function"), 302 item.code_location, 303 ( 304 len(item.positional_names), 305 len(item.defaults) 306 ), 307 0, 308 "__builtins__.function" 309 ) 310 ) 311 312 # Check the code location only where the code has been generated. 313 314 assert (not with_builtins and item.module.name == "__builtins__") or \ 315 item.code_location == len(self.raw_code) + len(item.defaults) 316 317 elif isinstance(item, micropython.data.Module): 318 assert item.location == len(self.raw_code) 319 320 self.raw_code.append( 321 DataObject( 322 objtable.as_list().get_code(item.full_name()), 323 None, # module name not used as an attribute 324 None, 325 None, 326 0, 327 item.full_name() 328 ) 329 ) 330 331 else: 332 self.raw_code.append(item) 333 334 # Fix the module locations. 335 336 for module in self.importer.modules_ordered: 337 338 if not with_builtins and module.name == "__builtins__": 339 continue 340 341 module.code_location = module.blocks[0].location 342 343 self.code_location = self.importer.modules["__main__"].code_location 344 return self.raw_code 345 346 def raw_block(self, block): 347 348 "Return the code for the given 'block'." 349 350 assert block.location == len(self.raw_code) 351 for i, item in enumerate(block.code): 352 if hasattr(item, "location"): 353 item.location = location + i 354 return block.code 355 356 def raw_data(self, item): 357 358 "Return the data for the given 'item'." 359 360 datatype = item.value_type_name() 361 362 # NOTE: Start simple and use single entries for most types. 363 364 if datatype in ("__builtins__.tuple", "__builtins__.list"): 365 return [len(item.value)] + list(item.value) 366 else: 367 return [item.value] 368 369 def get_object_table(self): 370 371 "Return a table with details of attributes for classes and modules." 372 373 if self.objtable is None: 374 self.importer.vacuum() 375 376 t = self.objtable = micropython.table.ObjectTable() 377 for module in self.importer.get_modules(): 378 t.add(module.full_name(), module.module_attributes()) 379 380 # Add class and instance attributes for all classes, together 381 # with descendant information. 382 383 for obj in module.all_objects: 384 if isinstance(obj, micropython.inspect.Class): 385 attributes = {obj.full_name() : obj} 386 attributes.update(obj.all_attributes()) 387 attributes.update(obj.all_descendants()) 388 t.add(obj.full_name(), attributes) 389 390 return self.objtable 391 392 def get_parameter_table(self): 393 394 "Return a table with details of parameters for functions and methods." 395 396 # Need the object table to get at class details. 397 398 objtable = self.get_object_table() 399 400 if self.paramtable is None: 401 t = self.paramtable = micropython.table.ParameterTable() 402 403 # Visit each module, getting function and method details. 404 405 for module in self.importer.get_modules(): 406 for obj in module.all_objects: 407 if isinstance(obj, micropython.inspect.Function): 408 t.add(obj.full_name(), obj.parameters()) 409 410 # Classes are callable, too. 411 # Take details of the appropriate __init__ method to make an 412 # entry for an instantiation function for the class. 413 414 elif isinstance(obj, micropython.inspect.Class): 415 t.add(obj.full_name(), obj.get_instantiator().parameters()) 416 417 # Filter out all parameter table entries not referenced by keyword 418 # arguments. 419 420 keyword_names = set() 421 422 for module in self.importer.get_modules(): 423 keyword_names.update(module.keyword_names) 424 425 for function_name, parameters in t.table.items(): 426 for name in parameters.keys(): 427 if name in keyword_names: 428 break 429 else: 430 del t.table[function_name] 431 432 return self.paramtable 433 434 class Importer: 435 436 "An import machine, searching for and loading modules." 437 438 predefined_constants = { 439 "None" : None, 440 "True" : True, 441 "False" : False, 442 "Ellipsis" : Ellipsis, 443 "NotImplemented" : NotImplemented 444 } 445 446 def __init__(self, path=None, verbose=0, optimisations=None): 447 448 """ 449 Initialise the importer with the given search 'path' - a list of 450 directories to search for Python modules. 451 452 The optional 'verbose' parameter causes output concerning the activities 453 of the object to be produced if set to a true value (not the default). 454 455 The optional 'optimisations' cause certain techniques to be used in 456 reducing program size and improving program efficiency. 457 """ 458 459 self.path = path or [os.getcwd()] 460 self.verbose = verbose 461 self.optimisations = optimisations or set() 462 463 self.modules = {} 464 self.modules_ordered = [] 465 self.loading = set() 466 467 # Constant records. 468 469 self.constant_values = {} 470 self.constant_list = None # cache for constants 471 self.init_predefined_constants() 472 473 # Name records (used to track actual use of names). 474 # Include names which may not be explicitly used in programs. 475 476 self.names_used = set(["__init__", "__call__"]) 477 478 # Status information. 479 480 self.vacuumed = 0 481 self.finalised = 0 482 483 def get_modules(self): 484 485 "Return all modules known to the importer." 486 487 return self.modules.values() 488 489 # General maintenance. 490 491 def vacuum(self): 492 493 "Tidy up the modules." 494 495 if self.vacuumed: 496 return 497 498 for name, module in self.modules.items(): 499 if module.loaded: 500 module.vacuum() 501 else: 502 del self.modules[name] 503 504 self.vacuumed = 1 505 506 def finalise(self): 507 508 "Finalise the program." 509 510 if self.finalised: 511 return 512 513 self.vacuum() 514 515 for module in self.get_modules(): 516 module.finalise() 517 518 self.finalised = 1 519 520 # Name accounting. 521 522 def use_name(self, name): 523 524 "Register the given 'name' as being used in the program." 525 526 self.names_used.add(name) 527 528 # Constant accounting. 529 530 def init_predefined_constants(self): 531 532 "Ensure the predefined constants." 533 534 for name, value in self.predefined_constants.items(): 535 self.make_constant(value) 536 537 def get_predefined_constant(self, name): 538 539 "Return the predefined constant for the given 'name'." 540 541 return self.make_constant(self.predefined_constants[name]) 542 543 def get_constant(self, value): 544 545 "Return a constant for the given 'value'." 546 547 const = micropython.data.Const(value) 548 return self.constant_values[const] 549 550 def make_constant(self, value): 551 552 "Make and return a constant for the given 'value'." 553 554 const = micropython.data.Const(value) 555 if not self.constant_values.has_key(const): 556 self.constant_values[const] = const 557 return self.constant_values[const] 558 559 def constants(self): 560 561 "Return a list of constants." 562 563 if self.constant_list is None: 564 self.constant_list = list(self.constant_values.values()) 565 566 return self.constant_list 567 568 # Import methods. 569 570 def find_in_path(self, name): 571 572 """ 573 Find the given module 'name' in the search path, returning None where no 574 such module could be found, or a 2-tuple from the 'find' method 575 otherwise. 576 """ 577 578 for d in self.path: 579 m = self.find(d, name) 580 if m: return m 581 return None 582 583 def find(self, d, name): 584 585 """ 586 In the directory 'd', find the given module 'name', where 'name' can 587 either refer to a single file module or to a package. Return None if the 588 'name' cannot be associated with either a file or a package directory, 589 or a 2-tuple from '_find_package' or '_find_module' otherwise. 590 """ 591 592 m = self._find_package(d, name) 593 if m: return m 594 m = self._find_module(d, name) 595 if m: return m 596 return None 597 598 def _find_module(self, d, name): 599 600 """ 601 In the directory 'd', find the given module 'name', returning None where 602 no suitable file exists in the directory, or a 2-tuple consisting of 603 None (indicating that no package directory is involved) and a filename 604 indicating the location of the module. 605 """ 606 607 name_py = name + os.extsep + "py" 608 filename = self._find_file(d, name_py) 609 if filename: 610 return None, filename 611 return None 612 613 def _find_package(self, d, name): 614 615 """ 616 In the directory 'd', find the given package 'name', returning None 617 where no suitable package directory exists, or a 2-tuple consisting of 618 a directory (indicating the location of the package directory itself) 619 and a filename indicating the location of the __init__.py module which 620 declares the package's top-level contents. 621 """ 622 623 filename = self._find_file(d, name) 624 if filename: 625 init_py = "__init__" + os.path.extsep + "py" 626 init_py_filename = self._find_file(filename, init_py) 627 if init_py_filename: 628 return filename, init_py_filename 629 return None 630 631 def _find_file(self, d, filename): 632 633 """ 634 Return the filename obtained when searching the directory 'd' for the 635 given 'filename', or None if no actual file exists for the filename. 636 """ 637 638 filename = os.path.join(d, filename) 639 if os.path.exists(filename): 640 return filename 641 else: 642 return None 643 644 def load(self, name, return_leaf=0): 645 646 """ 647 Load the module or package with the given 'name'. Return an object 648 referencing the loaded module or package, or None if no such module or 649 package exists. 650 """ 651 652 if self.modules.has_key(name) and self.modules[name].loaded: 653 #print "Cached (%s)" % name 654 return self.modules[name] 655 if self.verbose: 656 print "Loading", name 657 658 # Split the name into path components, and try to find the uppermost in 659 # the search path. 660 661 path = name.split(".") 662 m = self.find_in_path(path[0]) 663 if not m: 664 if self.verbose: 665 print "Not found (%s)" % path[0] 666 return None # NOTE: Import error. 667 d, filename = m 668 669 # Either acquire a reference to an already-imported module, or load the 670 # module from a file. 671 672 top = module = self.load_from_file(filename, path[0]) 673 674 # For hierarchical names, traverse each path component... 675 676 if len(path) > 1: 677 if not d: 678 if self.verbose: 679 print "No package (%s)" % filename 680 return None # NOTE: Import error (package not found). 681 else: 682 self.add_submodules(d, module) 683 684 path_so_far = path[:1] 685 for p in path[1:]: 686 path_so_far.append(p) 687 688 # Find the package or module concerned. 689 690 m = self.find(d, p) 691 if not m: 692 if self.verbose: 693 print "Not found (%s)" % p 694 return None # NOTE: Import error. 695 d, filename = m 696 module_name = ".".join(path_so_far) 697 698 # Either reference an imported module or load one from a file. 699 700 submodule = self.load_from_file(filename, module_name) 701 702 if d: 703 self.add_submodules(d, module) 704 705 # Store the submodule within its parent module. 706 707 module.set_module(p, submodule) 708 module = submodule 709 710 # Return either the deepest or the uppermost module. 711 712 if return_leaf: 713 return module 714 else: 715 return top 716 717 def load_from_file(self, name, module_name=None): 718 719 """ 720 Load the module with the given 'name' (which may be a full module path). 721 """ 722 723 if module_name is None: 724 module_name = "__main__" 725 726 module = self.add_module(module_name) 727 if not module.loaded and module not in self.loading: 728 self.loading.add(module) 729 #print "Parsing", name 730 module.parse(name) 731 #print "Done", name 732 self.loading.remove(module) 733 module.loaded = 1 734 735 # Record the module. 736 737 #print "Loaded", module_name, "with namespace", module.namespace.keys() 738 return module 739 740 def add_module(self, module_name): 741 742 """ 743 Return the module with the given 'module_name', adding a new module 744 object if one does not already exist. 745 """ 746 747 if not self.modules.has_key(module_name): 748 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 749 self.modules_ordered.append(module) 750 else: 751 module = self.modules[module_name] 752 return module 753 754 def add_submodules(self, pathname, module): 755 756 """ 757 Work around insufficient __all__ declarations and examine the directory 758 with the given 'pathname', adding submodules to the given 'module'. 759 """ 760 761 for filename in os.listdir(pathname): 762 submodule, ext = os.path.splitext(filename) 763 if ext not in ("", ".py"): 764 continue 765 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 766 767 # vim: tabstop=4 expandtab shiftwidth=4