1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 51 try: 52 set 53 except NameError: 54 from sets import Set as set 55 56 class Program: 57 58 "This class supports the generation of a program image." 59 60 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 61 62 def __init__(self, importer, optimisations=None): 63 64 """ 65 Initialise the program representation with an 'importer' which is able 66 to locate and load Python modules. 67 68 The optional 'optimisations' cause certain techniques to be used in 69 reducing program size and improving program efficiency. 70 """ 71 72 self.importer = importer 73 self.optimisations = optimisations or set() 74 75 # Remember the tables once generated. 76 77 self.objtable = None 78 self.paramtable = None 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def get_importer(self): 86 return self.importer 87 88 # Access to finalised program information. 89 90 def get_image(self, with_builtins=0): 91 92 "Return a dictionary mapping modules to structures." 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 self.importer.finalise() 100 101 self.code = [] 102 103 # Append constants to the image. 104 105 for pos, const in enumerate(self.importer.constants()): 106 self.code.append(const) 107 108 last_module = self.importer.modules_ordered[-1] 109 110 for module in self.importer.modules_ordered: 111 pos = len(self.code) 112 suppress_builtins = not with_builtins and module.name == "__builtins__" 113 114 # Position the module in the image and make a translation. 115 116 trans = micropython.ast.Translation(module, self) 117 118 # Add header details. 119 120 self.code.append(module) 121 pos += 1 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 pos += len(attributes.keys()) 128 129 # Append classes and functions to the image. 130 131 for obj in module.all_objects: 132 if isinstance(obj, micropython.inspect.Class): 133 134 # Add header details. 135 136 self.code.append(obj) 137 pos += 1 138 139 # Append class attributes to the image. 140 141 attributes = obj.class_attributes() 142 self.code += obj.attributes_as_list() 143 pos += len(attributes.keys()) 144 145 # Omit built-in function code where requested. 146 147 if suppress_builtins: 148 continue 149 150 # Generate the instantiator/initialiser. 151 # Append the function code to the image. 152 153 instantiator = obj.get_instantiator() 154 code = trans.get_instantiator_code(obj) 155 self.code += code 156 pos += len(code) 157 158 # Class-level code is generated separately at the module 159 # level, and the code location is set within the code 160 # generation process for the module. 161 162 elif isinstance(obj, micropython.inspect.Function): 163 164 # Add header details. 165 166 self.code.append(obj) 167 pos += 1 168 169 # Append any default values to the image. 170 # Only do this for named functions (not lambdas). 171 172 if obj.name is not None: 173 self.code += obj.default_attrs 174 pos += len(obj.default_attrs) 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 pos += len(code) 187 188 # Omit built-in module code where requested. 189 190 if suppress_builtins: 191 pass 192 193 # Append the module top-level code to the image. 194 195 else: 196 code = trans.get_module_code(final=(module is last_module)) 197 self.code += code 198 pos += len(code) 199 200 return self.code 201 202 def get_raw_image(self, with_builtins=0): 203 204 "Return the raw image representation of the program." 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 current_function = None 215 216 for item in self.code: 217 218 # Blocks are positioned leaving space for their expansion. 219 220 if isinstance(item, Block): 221 item.location = pos 222 pos += len(item.code) 223 224 # Set code body information on functions, assuming that the 225 # first block is for argument checks. 226 227 if current_function is not None: 228 current_function.code_body_location = pos 229 230 current_function = None 231 232 # Other multi-location objects. 233 234 elif isinstance(item, ( 235 micropython.data.Class, micropython.data.Const, 236 micropython.data.Function, micropython.data.Module 237 )): 238 239 if isinstance(item, micropython.data.Class): 240 241 # Append a template of an instance for use when 242 # instantiating classes. 243 244 item.instance_template_location = pos 245 pos += 1 246 247 # Record the location of the object. 248 249 item.location = pos 250 pos += 1 251 252 # Code and details are associated with certain objects. 253 254 if isinstance(item, micropython.data.Function): 255 256 # Set the code location only where the code has been 257 # generated. 258 259 if not with_builtins and item.module.name == "__builtins__": 260 item.code_location = item.full_name() 261 262 # Skip any defaults. 263 264 else: 265 item.code_location = pos + len(item.defaults) 266 267 current_function = item 268 269 elif isinstance(item, micropython.data.Const): 270 pos += len(item.raw_data()) 271 current_function = None 272 273 else: 274 current_function = None 275 276 else: 277 pos += 1 278 279 # Generate the raw code. 280 281 self.raw_code = [] 282 283 for item in self.code: 284 285 if isinstance(item, micropython.data.Attr): 286 self.raw_code += item.as_raw(objtable, paramtable) 287 288 elif isinstance(item, Block): 289 assert item.location == len(self.raw_code) 290 self.raw_code += item.as_raw(objtable, paramtable) 291 292 # Using classcode, attrcode, codeaddr, codedetails, instance. 293 294 elif isinstance(item, micropython.data.Class): 295 assert item.instance_template_location == len(self.raw_code) 296 self.raw_code += item.as_raw(objtable, paramtable, with_builtins or item.module.name != "__builtins__") 297 assert item.location == len(self.raw_code) - 1 298 299 elif isinstance(item, micropython.data.Const): 300 assert item.location == len(self.raw_code) 301 self.raw_code += item.as_raw(objtable, paramtable) 302 303 elif isinstance(item, micropython.data.Function): 304 assert item.location == len(self.raw_code) 305 self.raw_code += item.as_raw(objtable, paramtable) 306 307 # Check the code location only where the code has been generated. 308 309 assert (not with_builtins and item.module.name == "__builtins__") or \ 310 item.code_location == len(self.raw_code) + len(item.defaults) 311 312 elif isinstance(item, micropython.data.Module): 313 assert item.location == len(self.raw_code) 314 self.raw_code += item.as_raw(objtable, paramtable) 315 316 else: 317 self.raw_code.append(item) 318 319 # Fix the module locations. 320 321 for module in self.importer.modules_ordered: 322 323 if not with_builtins and module.name == "__builtins__": 324 continue 325 326 module.code_location = module.blocks[0].location 327 328 self.code_location = self.importer.modules["__main__"].code_location 329 return self.raw_code 330 331 def get_object_table(self): 332 333 "Return a table with details of attributes for classes and modules." 334 335 if self.objtable is None: 336 self.importer.vacuum() 337 338 t = self.objtable = micropython.table.ObjectTable() 339 for module in self.importer.get_modules(): 340 t.add(module.full_name(), module.module_attributes()) 341 342 # Add class and instance attributes for all classes, together 343 # with descendant information. 344 345 for obj in module.all_objects: 346 if isinstance(obj, micropython.inspect.Class): 347 attributes = {obj.full_name() : obj} 348 attributes.update(obj.all_attributes()) 349 attributes.update(obj.all_descendants()) 350 t.add(obj.full_name(), attributes) 351 352 return self.objtable 353 354 def get_parameter_table(self): 355 356 "Return a table with details of parameters for functions and methods." 357 358 # Need the object table to get at class details. 359 360 objtable = self.get_object_table() 361 362 if self.paramtable is None: 363 t = self.paramtable = micropython.table.ParameterTable() 364 365 # Visit each module, getting function and method details. 366 367 for module in self.importer.get_modules(): 368 for obj in module.all_objects: 369 if isinstance(obj, micropython.inspect.Function): 370 t.add(obj.full_name(), obj.parameters()) 371 372 # Classes are callable, too. 373 # Take details of the appropriate __init__ method to make an 374 # entry for an instantiation function for the class. 375 376 elif isinstance(obj, micropython.inspect.Class): 377 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 378 379 # Filter out all parameter table entries not referenced by keyword 380 # arguments. 381 382 keyword_names = set() 383 384 for module in self.importer.get_modules(): 385 keyword_names.update(module.keyword_names) 386 387 for function_name, parameters in t.table.items(): 388 for name in parameters.keys(): 389 if name in keyword_names: 390 break 391 else: 392 del t.table[function_name] 393 394 return self.paramtable 395 396 class Importer: 397 398 "An import machine, searching for and loading modules." 399 400 predefined_constants = { 401 "None" : None, 402 "True" : True, 403 "False" : False, 404 #"Ellipsis" : Ellipsis, 405 "NotImplemented" : NotImplemented 406 } 407 408 def __init__(self, path=None, verbose=0, optimisations=None): 409 410 """ 411 Initialise the importer with the given search 'path' - a list of 412 directories to search for Python modules. 413 414 The optional 'verbose' parameter causes output concerning the activities 415 of the object to be produced if set to a true value (not the default). 416 417 The optional 'optimisations' cause certain techniques to be used in 418 reducing program size and improving program efficiency. 419 """ 420 421 self.path = path or [os.getcwd()] 422 self.verbose = verbose 423 self.optimisations = optimisations or set() 424 425 self.modules = {} 426 self.modules_ordered = [] 427 self.loading = set() 428 429 # Constant records. 430 431 self.constant_values = {} 432 self.constant_list = None # cache for constants 433 self.init_predefined_constants() 434 435 # Name records (used to track actual use of names). 436 # Include names which may not be explicitly used in programs. 437 # NOTE: Potentially declare these when inspecting. 438 439 self.names_used = set(["__init__", "__call__", "__bool__"]) 440 441 # Status information. 442 443 self.vacuumed = 0 444 self.finalised = 0 445 446 def get_modules(self): 447 448 "Return all modules known to the importer." 449 450 return self.modules.values() 451 452 def get_module(self, name): 453 454 "Return the module with the given 'name'." 455 456 return self.modules[name] 457 458 # General maintenance. 459 460 def vacuum(self): 461 462 "Tidy up the modules." 463 464 if self.vacuumed: 465 return 466 467 for name, module in self.modules.items(): 468 if module.loaded: 469 module.vacuum() 470 else: 471 del self.modules[name] 472 473 self.vacuumed = 1 474 475 def finalise(self): 476 477 "Finalise the program." 478 479 if self.finalised: 480 return 481 482 self.vacuum() 483 484 for module in self.get_modules(): 485 module.finalise() 486 487 self.finalised = 1 488 489 # Name accounting. 490 491 def use_name(self, name): 492 493 "Register the given 'name' as being used in the program." 494 495 self.names_used.add(name) 496 497 # Constant accounting. 498 499 def init_predefined_constants(self): 500 501 "Ensure the predefined constants." 502 503 for name, value in self.predefined_constants.items(): 504 self.make_constant(value) 505 506 def get_predefined_constant(self, name): 507 508 "Return the predefined constant for the given 'name'." 509 510 return self.make_constant(self.predefined_constants[name]) 511 512 def get_constant(self, value): 513 514 "Return a constant for the given 'value'." 515 516 const = micropython.data.Const(value) 517 return self.constant_values[const] 518 519 def make_constant(self, value): 520 521 "Make and return a constant for the given 'value'." 522 523 # Ensure the presence of the constant's type. 524 525 name = value.__class__.__name__ 526 if self.modules.has_key("__builtins__"): 527 attr = self.modules["__builtins__"].get(name) 528 if attr is not None: 529 attr.set_referenced() 530 531 # Make a constant object and return it. 532 533 const = micropython.data.Const(value) 534 if not self.constant_values.has_key(const): 535 self.constant_values[const] = const 536 return self.constant_values[const] 537 538 def constants(self): 539 540 "Return a list of constants." 541 542 if self.constant_list is None: 543 self.constant_list = list(self.constant_values.values()) 544 545 return self.constant_list 546 547 # Import methods. 548 549 def find_in_path(self, name): 550 551 """ 552 Find the given module 'name' in the search path, returning None where no 553 such module could be found, or a 2-tuple from the 'find' method 554 otherwise. 555 """ 556 557 for d in self.path: 558 m = self.find(d, name) 559 if m: return m 560 return None 561 562 def find(self, d, name): 563 564 """ 565 In the directory 'd', find the given module 'name', where 'name' can 566 either refer to a single file module or to a package. Return None if the 567 'name' cannot be associated with either a file or a package directory, 568 or a 2-tuple from '_find_package' or '_find_module' otherwise. 569 """ 570 571 m = self._find_package(d, name) 572 if m: return m 573 m = self._find_module(d, name) 574 if m: return m 575 return None 576 577 def _find_module(self, d, name): 578 579 """ 580 In the directory 'd', find the given module 'name', returning None where 581 no suitable file exists in the directory, or a 2-tuple consisting of 582 None (indicating that no package directory is involved) and a filename 583 indicating the location of the module. 584 """ 585 586 name_py = name + os.extsep + "py" 587 filename = self._find_file(d, name_py) 588 if filename: 589 return None, filename 590 return None 591 592 def _find_package(self, d, name): 593 594 """ 595 In the directory 'd', find the given package 'name', returning None 596 where no suitable package directory exists, or a 2-tuple consisting of 597 a directory (indicating the location of the package directory itself) 598 and a filename indicating the location of the __init__.py module which 599 declares the package's top-level contents. 600 """ 601 602 filename = self._find_file(d, name) 603 if filename: 604 init_py = "__init__" + os.path.extsep + "py" 605 init_py_filename = self._find_file(filename, init_py) 606 if init_py_filename: 607 return filename, init_py_filename 608 return None 609 610 def _find_file(self, d, filename): 611 612 """ 613 Return the filename obtained when searching the directory 'd' for the 614 given 'filename', or None if no actual file exists for the filename. 615 """ 616 617 filename = os.path.join(d, filename) 618 if os.path.exists(filename): 619 return filename 620 else: 621 return None 622 623 def load(self, name, return_leaf=0): 624 625 """ 626 Load the module or package with the given 'name'. Return an object 627 referencing the loaded module or package, or None if no such module or 628 package exists. 629 """ 630 631 if self.modules.has_key(name) and self.modules[name].loaded: 632 #print "Cached (%s)" % name 633 return self.modules[name] 634 if self.verbose: 635 print "Loading", name 636 637 # Split the name into path components, and try to find the uppermost in 638 # the search path. 639 640 path = name.split(".") 641 m = self.find_in_path(path[0]) 642 if not m: 643 if self.verbose: 644 print "Not found (%s)" % path[0] 645 return None # NOTE: Import error. 646 d, filename = m 647 648 # Either acquire a reference to an already-imported module, or load the 649 # module from a file. 650 651 top = module = self.load_from_file(filename, path[0]) 652 653 # For hierarchical names, traverse each path component... 654 655 if len(path) > 1: 656 if not d: 657 if self.verbose: 658 print "No package (%s)" % filename 659 return None # NOTE: Import error (package not found). 660 else: 661 self.add_submodules(d, module) 662 663 path_so_far = path[:1] 664 for p in path[1:]: 665 path_so_far.append(p) 666 667 # Find the package or module concerned. 668 669 m = self.find(d, p) 670 if not m: 671 if self.verbose: 672 print "Not found (%s)" % p 673 return None # NOTE: Import error. 674 d, filename = m 675 module_name = ".".join(path_so_far) 676 677 # Either reference an imported module or load one from a file. 678 679 submodule = self.load_from_file(filename, module_name) 680 681 if d: 682 self.add_submodules(d, module) 683 684 # Store the submodule within its parent module. 685 686 module.set_module(p, submodule) 687 module = submodule 688 689 # Return either the deepest or the uppermost module. 690 691 if return_leaf: 692 return module 693 else: 694 return top 695 696 def load_from_file(self, name, module_name=None): 697 698 """ 699 Load the module with the given 'name' (which may be a full module path). 700 """ 701 702 if module_name is None: 703 module_name = "__main__" 704 705 module = self.add_module(module_name) 706 if not module.loaded and module not in self.loading: 707 self.loading.add(module) 708 #print "Parsing", name 709 module.parse(name) 710 #print "Done", name 711 self.loading.remove(module) 712 module.loaded = 1 713 714 # Record the module. 715 716 #print "Loaded", module_name, "with namespace", module.namespace.keys() 717 return module 718 719 def add_module(self, module_name): 720 721 """ 722 Return the module with the given 'module_name', adding a new module 723 object if one does not already exist. 724 """ 725 726 if not self.modules.has_key(module_name): 727 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 728 self.modules_ordered.append(module) 729 else: 730 module = self.modules[module_name] 731 return module 732 733 def add_submodules(self, pathname, module): 734 735 """ 736 Work around insufficient __all__ declarations and examine the directory 737 with the given 'pathname', adding submodules to the given 'module'. 738 """ 739 740 for filename in os.listdir(pathname): 741 submodule, ext = os.path.splitext(filename) 742 if ext not in ("", ".py"): 743 continue 744 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 745 746 # Convenience functions. 747 748 def program(filename, path, requested_optimisations, verbose=0): 749 750 """ 751 Return the program object for the program specified by the given 'filename', 752 module search 'path' and 'requested_optimisations'. 753 """ 754 755 i = micropython.Importer(path, verbose, requested_optimisations) 756 p = micropython.Program(i, requested_optimisations) 757 758 i.load_from_file("lib/builtins.py", "__builtins__") 759 760 if filename is not None: 761 i.load_from_file(filename) 762 763 return p 764 765 # vim: tabstop=4 expandtab shiftwidth=4