1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 51 try: 52 set 53 except NameError: 54 from sets import Set as set 55 56 class Program: 57 58 "This class supports the generation of a program image." 59 60 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 61 62 def __init__(self, importer, optimisations=None): 63 64 """ 65 Initialise the program representation with an 'importer' which is able 66 to locate and load Python modules. 67 68 The optional 'optimisations' cause certain techniques to be used in 69 reducing program size and improving program efficiency. 70 """ 71 72 self.importer = importer 73 self.optimisations = optimisations or set() 74 75 # Remember the tables once generated. 76 77 self.objtable = None 78 self.paramtable = None 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def get_importer(self): 86 return self.importer 87 88 # Access to finalised program information. 89 90 def get_image(self, with_builtins=0): 91 92 "Return a dictionary mapping modules to structures." 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 self.importer.finalise() 100 101 self.code = [] 102 103 # Append constants to the image. 104 105 for pos, const in enumerate(self.importer.constants()): 106 self.code.append(const) 107 108 last_module = self.importer.modules_ordered[-1] 109 110 for module in self.importer.modules_ordered: 111 pos = len(self.code) 112 suppress_builtins = not with_builtins and module.name == "__builtins__" 113 114 # Position the module in the image and make a translation. 115 116 trans = micropython.ast.Translation(module, self) 117 118 # Add header details. 119 120 self.code.append(module) 121 pos += 1 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 pos += len(attributes.keys()) 128 129 # Append classes and functions to the image. 130 131 for obj in module.all_objects: 132 if isinstance(obj, micropython.inspect.Class): 133 134 # Add header details. 135 136 self.code.append(obj) 137 pos += 1 138 139 # Append class attributes to the image. 140 141 attributes = obj.class_attributes() 142 self.code += obj.attributes_as_list() 143 pos += len(attributes.keys()) 144 145 # Omit built-in function code where requested. 146 147 if suppress_builtins and not obj.astnode.doc is not None: 148 continue 149 150 # Generate the instantiator/initialiser. 151 # Append the function code to the image. 152 153 instantiator = obj.get_instantiator() 154 code = trans.get_instantiator_code(obj) 155 self.code += code 156 pos += len(code) 157 158 # Class-level code is generated separately at the module 159 # level, and the code location is set within the code 160 # generation process for the module. 161 162 elif isinstance(obj, micropython.inspect.Function): 163 164 # Add header details. 165 166 self.code.append(obj) 167 pos += 1 168 169 # Append any default values to the image. 170 # Only do this for named functions (not lambdas). 171 172 if obj.name is not None: 173 self.code += obj.default_attrs 174 pos += len(obj.default_attrs) 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and not obj.astnode.doc is not None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 pos += len(code) 187 188 # Omit built-in module code where requested. 189 190 if suppress_builtins: 191 pass 192 193 # Append the module top-level code to the image. 194 195 else: 196 code = trans.get_module_code(final=(module is last_module)) 197 self.code += code 198 pos += len(code) 199 200 return self.code 201 202 def get_raw_image(self, with_builtins=0): 203 204 "Return the raw image representation of the program." 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 current_function = None 215 216 for item in self.code: 217 218 # Blocks are positioned leaving space for their expansion. 219 220 if isinstance(item, Block): 221 item.location = pos 222 pos += len(item.code) 223 224 # Set code body information on functions, assuming that the 225 # first block is for argument checks. 226 227 if current_function is not None: 228 current_function.code_body_location = pos 229 230 current_function = None 231 232 # Other multi-location objects. 233 234 elif isinstance(item, ( 235 micropython.data.Class, micropython.data.Const, 236 micropython.data.Function, micropython.data.Module 237 )): 238 239 if isinstance(item, micropython.data.Class): 240 241 # Append a template of an instance for use when 242 # instantiating classes. 243 244 item.instance_template_location = pos 245 pos += 1 246 247 # Record the location of the object. 248 249 item.location = pos 250 pos += 1 251 252 # Code and details are associated with certain objects. 253 254 if isinstance(item, micropython.data.Function): 255 256 # Set the code location only where the code has been 257 # generated. 258 259 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 260 item.code_location = item.full_name() 261 262 # Skip any defaults for named functions. 263 264 elif item.name is not None: 265 item.code_location = pos + len(item.defaults) 266 267 # Skip any defaults for lambda functions. 268 269 else: 270 item.code_location = pos 271 272 current_function = item 273 274 elif isinstance(item, micropython.data.Const): 275 pos += len(item.raw_data()) 276 current_function = None 277 278 else: 279 current_function = None 280 281 else: 282 pos += 1 283 284 # Generate the raw code. 285 286 self.raw_code = [] 287 288 for item in self.code: 289 290 if isinstance(item, micropython.data.Attr): 291 self.raw_code += item.as_raw(objtable, paramtable) 292 293 elif isinstance(item, Block): 294 assert item.location == len(self.raw_code) 295 self.raw_code += item.as_raw(objtable, paramtable) 296 297 # Using classcode, attrcode, codeaddr, codedetails, instance. 298 299 elif isinstance(item, micropython.data.Class): 300 assert item.instance_template_location == len(self.raw_code) 301 self.raw_code += item.as_raw(objtable, paramtable, 302 with_builtins or item.module.name != "__builtins__" or item.astnode.doc is not None) 303 assert item.location == len(self.raw_code) - 1 304 305 elif isinstance(item, micropython.data.Const): 306 assert item.location == len(self.raw_code) 307 self.raw_code += item.as_raw(objtable, paramtable) 308 309 elif isinstance(item, micropython.data.Function): 310 assert item.location == len(self.raw_code) 311 self.raw_code += item.as_raw(objtable, paramtable) 312 313 # Check the code location only where the code has been generated. 314 315 assert (not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None) or \ 316 item.name is not None and item.code_location == len(self.raw_code) + len(item.defaults) or \ 317 item.name is None and item.code_location == len(self.raw_code) 318 319 elif isinstance(item, micropython.data.Module): 320 assert item.location == len(self.raw_code) 321 self.raw_code += item.as_raw(objtable, paramtable) 322 323 else: 324 self.raw_code.append(item) 325 326 # Fix the module locations. 327 328 for module in self.importer.modules_ordered: 329 330 if not with_builtins and module.name == "__builtins__": 331 continue 332 333 module.code_location = module.blocks[0].location 334 335 self.code_location = self.importer.modules["__main__"].code_location 336 return self.raw_code 337 338 def get_object_table(self): 339 340 "Return a table with details of attributes for classes and modules." 341 342 if self.objtable is None: 343 self.importer.vacuum() 344 345 t = self.objtable = micropython.table.ObjectTable() 346 for module in self.importer.get_modules(): 347 t.add(module.full_name(), module.module_attributes()) 348 349 # Add class and instance attributes for all classes, together 350 # with descendant information. 351 352 for obj in module.all_objects: 353 if isinstance(obj, micropython.inspect.Class): 354 attributes = {obj.full_name() : obj} 355 attributes.update(obj.all_attributes()) 356 attributes.update(obj.all_descendants()) 357 t.add(obj.full_name(), attributes) 358 359 return self.objtable 360 361 def get_parameter_table(self): 362 363 "Return a table with details of parameters for functions and methods." 364 365 # Need the object table to get at class details. 366 367 objtable = self.get_object_table() 368 369 if self.paramtable is None: 370 t = self.paramtable = micropython.table.ParameterTable() 371 372 # Visit each module, getting function and method details. 373 374 for module in self.importer.get_modules(): 375 for obj in module.all_objects: 376 if isinstance(obj, micropython.inspect.Function): 377 t.add(obj.full_name(), obj.parameters()) 378 379 # Classes are callable, too. 380 # Take details of the appropriate __init__ method to make an 381 # entry for an instantiation function for the class. 382 383 elif isinstance(obj, micropython.inspect.Class): 384 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 385 386 # Filter out all parameter table entries not referenced by keyword 387 # arguments. 388 389 keyword_names = set() 390 391 for module in self.importer.get_modules(): 392 keyword_names.update(module.keyword_names) 393 394 for function_name, parameters in t.table.items(): 395 for name in parameters.keys(): 396 if name in keyword_names: 397 break 398 else: 399 del t.table[function_name] 400 401 return self.paramtable 402 403 class Importer: 404 405 "An import machine, searching for and loading modules." 406 407 predefined_constants = { 408 "None" : None, 409 "True" : True, 410 "False" : False, 411 #"Ellipsis" : Ellipsis, 412 "NotImplemented" : NotImplemented 413 } 414 415 def __init__(self, path=None, verbose=0, optimisations=None): 416 417 """ 418 Initialise the importer with the given search 'path' - a list of 419 directories to search for Python modules. 420 421 The optional 'verbose' parameter causes output concerning the activities 422 of the object to be produced if set to a true value (not the default). 423 424 The optional 'optimisations' cause certain techniques to be used in 425 reducing program size and improving program efficiency. 426 """ 427 428 self.path = path or [os.getcwd()] 429 self.verbose = verbose 430 self.optimisations = optimisations or set() 431 432 self.modules = {} 433 self.modules_ordered = [] 434 self.loading = set() 435 436 # Constant records. 437 438 self.constant_values = {} 439 self.constant_list = None # cache for constants 440 self.init_predefined_constants() 441 442 # Name records (used to track actual use of names). 443 # Include names which may not be explicitly used in programs. 444 # NOTE: Potentially declare these when inspecting. 445 446 self.names_used = set(["__init__", "__call__", "__bool__"]) 447 448 # Status information. 449 450 self.vacuumed = 0 451 self.finalised = 0 452 453 def get_modules(self): 454 455 "Return all modules known to the importer." 456 457 return self.modules.values() 458 459 def get_module(self, name): 460 461 "Return the module with the given 'name'." 462 463 return self.modules[name] 464 465 # General maintenance. 466 467 def vacuum(self): 468 469 "Tidy up the modules." 470 471 if self.vacuumed: 472 return 473 474 for name, module in self.modules.items(): 475 if module.loaded: 476 module.vacuum() 477 else: 478 del self.modules[name] 479 480 self.vacuumed = 1 481 482 def finalise(self): 483 484 "Finalise the program." 485 486 if self.finalised: 487 return 488 489 self.vacuum() 490 491 for module in self.get_modules(): 492 module.finalise() 493 494 self.finalised = 1 495 496 # Name accounting. 497 498 def use_name(self, name): 499 500 "Register the given 'name' as being used in the program." 501 502 self.names_used.add(name) 503 504 # Constant accounting. 505 506 def init_predefined_constants(self): 507 508 "Ensure the predefined constants." 509 510 for name, value in self.predefined_constants.items(): 511 self.make_constant(value) 512 513 def get_predefined_constant(self, name): 514 515 "Return the predefined constant for the given 'name'." 516 517 return self.make_constant(self.predefined_constants[name]) 518 519 def get_constant(self, value): 520 521 "Return a constant for the given 'value'." 522 523 const = micropython.data.Const(value) 524 return self.constant_values[const] 525 526 def make_constant(self, value): 527 528 "Make and return a constant for the given 'value'." 529 530 # Ensure the presence of the constant's type. 531 532 name = value.__class__.__name__ 533 if self.modules.has_key("__builtins__"): 534 attr = self.modules["__builtins__"].get(name) 535 if attr is not None: 536 attr.set_referenced() 537 538 # Make a constant object and return it. 539 540 const = micropython.data.Const(value) 541 if not self.constant_values.has_key(const): 542 self.constant_values[const] = const 543 return self.constant_values[const] 544 545 def constants(self): 546 547 "Return a list of constants." 548 549 if self.constant_list is None: 550 self.constant_list = list(self.constant_values.values()) 551 552 return self.constant_list 553 554 # Import methods. 555 556 def find_in_path(self, name): 557 558 """ 559 Find the given module 'name' in the search path, returning None where no 560 such module could be found, or a 2-tuple from the 'find' method 561 otherwise. 562 """ 563 564 for d in self.path: 565 m = self.find(d, name) 566 if m: return m 567 return None 568 569 def find(self, d, name): 570 571 """ 572 In the directory 'd', find the given module 'name', where 'name' can 573 either refer to a single file module or to a package. Return None if the 574 'name' cannot be associated with either a file or a package directory, 575 or a 2-tuple from '_find_package' or '_find_module' otherwise. 576 """ 577 578 m = self._find_package(d, name) 579 if m: return m 580 m = self._find_module(d, name) 581 if m: return m 582 return None 583 584 def _find_module(self, d, name): 585 586 """ 587 In the directory 'd', find the given module 'name', returning None where 588 no suitable file exists in the directory, or a 2-tuple consisting of 589 None (indicating that no package directory is involved) and a filename 590 indicating the location of the module. 591 """ 592 593 name_py = name + os.extsep + "py" 594 filename = self._find_file(d, name_py) 595 if filename: 596 return None, filename 597 return None 598 599 def _find_package(self, d, name): 600 601 """ 602 In the directory 'd', find the given package 'name', returning None 603 where no suitable package directory exists, or a 2-tuple consisting of 604 a directory (indicating the location of the package directory itself) 605 and a filename indicating the location of the __init__.py module which 606 declares the package's top-level contents. 607 """ 608 609 filename = self._find_file(d, name) 610 if filename: 611 init_py = "__init__" + os.path.extsep + "py" 612 init_py_filename = self._find_file(filename, init_py) 613 if init_py_filename: 614 return filename, init_py_filename 615 return None 616 617 def _find_file(self, d, filename): 618 619 """ 620 Return the filename obtained when searching the directory 'd' for the 621 given 'filename', or None if no actual file exists for the filename. 622 """ 623 624 filename = os.path.join(d, filename) 625 if os.path.exists(filename): 626 return filename 627 else: 628 return None 629 630 def load(self, name, return_leaf=0): 631 632 """ 633 Load the module or package with the given 'name'. Return an object 634 referencing the loaded module or package, or None if no such module or 635 package exists. 636 """ 637 638 if self.modules.has_key(name) and self.modules[name].loaded: 639 #print "Cached (%s)" % name 640 return self.modules[name] 641 if self.verbose: 642 print "Loading", name 643 644 # Split the name into path components, and try to find the uppermost in 645 # the search path. 646 647 path = name.split(".") 648 m = self.find_in_path(path[0]) 649 if not m: 650 if self.verbose: 651 print "Not found (%s)" % path[0] 652 return None # NOTE: Import error. 653 d, filename = m 654 655 # Either acquire a reference to an already-imported module, or load the 656 # module from a file. 657 658 top = module = self.load_from_file(filename, path[0]) 659 660 # For hierarchical names, traverse each path component... 661 662 if len(path) > 1: 663 if not d: 664 if self.verbose: 665 print "No package (%s)" % filename 666 return None # NOTE: Import error (package not found). 667 else: 668 self.add_submodules(d, module) 669 670 path_so_far = path[:1] 671 for p in path[1:]: 672 path_so_far.append(p) 673 674 # Find the package or module concerned. 675 676 m = self.find(d, p) 677 if not m: 678 if self.verbose: 679 print "Not found (%s)" % p 680 return None # NOTE: Import error. 681 d, filename = m 682 module_name = ".".join(path_so_far) 683 684 # Either reference an imported module or load one from a file. 685 686 submodule = self.load_from_file(filename, module_name) 687 688 if d: 689 self.add_submodules(d, module) 690 691 # Store the submodule within its parent module. 692 693 module.set_module(p, submodule) 694 module = submodule 695 696 # Return either the deepest or the uppermost module. 697 698 if return_leaf: 699 return module 700 else: 701 return top 702 703 def load_from_file(self, name, module_name=None): 704 705 """ 706 Load the module with the given 'name' (which may be a full module path). 707 """ 708 709 if module_name is None: 710 module_name = "__main__" 711 712 module = self.add_module(module_name) 713 if not module.loaded and module not in self.loading: 714 self.loading.add(module) 715 #print "Parsing", name 716 module.parse(name) 717 #print "Done", name 718 self.loading.remove(module) 719 module.loaded = 1 720 721 # Record the module. 722 723 #print "Loaded", module_name, "with namespace", module.namespace.keys() 724 return module 725 726 def add_module(self, module_name): 727 728 """ 729 Return the module with the given 'module_name', adding a new module 730 object if one does not already exist. 731 """ 732 733 if not self.modules.has_key(module_name): 734 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 735 self.modules_ordered.append(module) 736 else: 737 module = self.modules[module_name] 738 return module 739 740 def add_submodules(self, pathname, module): 741 742 """ 743 Work around insufficient __all__ declarations and examine the directory 744 with the given 'pathname', adding submodules to the given 'module'. 745 """ 746 747 for filename in os.listdir(pathname): 748 submodule, ext = os.path.splitext(filename) 749 if ext not in ("", ".py"): 750 continue 751 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 752 753 # vim: tabstop=4 expandtab shiftwidth=4