1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 51 try: 52 set 53 except NameError: 54 from sets import Set as set 55 56 class Program: 57 58 "This class supports the generation of a program image." 59 60 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 61 62 def __init__(self, importer, optimisations=None): 63 64 """ 65 Initialise the program representation with an 'importer' which is able 66 to locate and load Python modules. 67 68 The optional 'optimisations' cause certain techniques to be used in 69 reducing program size and improving program efficiency. 70 """ 71 72 self.importer = importer 73 self.optimisations = optimisations or set() 74 75 # Remember the tables once generated. 76 77 self.objtable = None 78 self.paramtable = None 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def get_importer(self): 86 return self.importer 87 88 # Access to finalised program information. 89 90 def get_image(self, with_builtins=0): 91 92 "Return a dictionary mapping modules to structures." 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 self.importer.finalise() 100 101 self.code = [] 102 103 # Append constants to the image. 104 105 for pos, const in enumerate(self.importer.constants()): 106 self.code.append(const) 107 108 last_module = self.importer.modules_ordered[-1] 109 110 for module in self.importer.modules_ordered: 111 pos = len(self.code) 112 suppress_builtins = not with_builtins and module.name == "__builtins__" 113 114 # Position the module in the image and make a translation. 115 116 trans = micropython.ast.Translation(module, self) 117 118 # Add header details. 119 120 self.code.append(module) 121 pos += 1 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 pos += len(attributes.keys()) 128 129 # Append classes and functions to the image. 130 131 for obj in module.all_objects: 132 if isinstance(obj, micropython.inspect.Class): 133 134 # Add header details. 135 136 self.code.append(obj) 137 pos += 1 138 139 # Append class attributes to the image. 140 141 attributes = obj.class_attributes() 142 self.code += obj.attributes_as_list() 143 pos += len(attributes.keys()) 144 145 # Omit built-in function code where requested. 146 147 if suppress_builtins and not obj.astnode.doc is not None: 148 continue 149 150 # Generate the instantiator/initialiser. 151 # Append the function code to the image. 152 153 instantiator = obj.get_instantiator() 154 code = trans.get_instantiator_code(obj) 155 self.code += code 156 pos += len(code) 157 158 # Class-level code is generated separately at the module 159 # level, and the code location is set within the code 160 # generation process for the module. 161 162 elif isinstance(obj, micropython.inspect.Function): 163 164 # Add header details. 165 166 self.code.append(obj) 167 pos += 1 168 169 # Append any default values to the image. 170 # Only do this for named functions (not lambdas). 171 172 if obj.name is not None: 173 self.code += obj.default_attrs 174 pos += len(obj.default_attrs) 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and not obj.astnode.doc is not None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 pos += len(code) 187 188 # Omit built-in module code where requested. 189 190 if suppress_builtins: 191 pass 192 193 # Append the module top-level code to the image. 194 195 else: 196 code = trans.get_module_code(final=(module is last_module)) 197 self.code += code 198 pos += len(code) 199 200 return self.code 201 202 def get_raw_image(self, with_builtins=0): 203 204 "Return the raw image representation of the program." 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 215 for item in self.code: 216 217 # Blocks are positioned leaving space for their expansion. 218 219 if isinstance(item, Block): 220 item.location = pos 221 pos += len(item.code) 222 223 # Other multi-location objects. 224 225 elif isinstance(item, ( 226 micropython.data.Class, micropython.data.Const, 227 micropython.data.Function, micropython.data.Module 228 )): 229 230 if isinstance(item, micropython.data.Class): 231 232 # Append a template of an instance for use when 233 # instantiating classes. 234 235 item.instance_template_location = pos 236 pos += 1 237 238 # Record the location of the object. 239 240 item.location = pos 241 pos += 1 242 243 # Code and details are associated with certain objects. 244 245 if isinstance(item, micropython.data.Function): 246 247 # Set the code location only where the code has been 248 # generated. 249 250 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 251 item.code_location = item.full_name() 252 253 # Skip any defaults for named functions. 254 255 elif item.name is not None: 256 item.code_location = pos + len(item.defaults) 257 258 # Skip any defaults for lambda functions. 259 260 else: 261 item.code_location = pos 262 263 elif isinstance(item, micropython.data.Const): 264 pos += len(item.raw_data()) 265 266 else: 267 pos += 1 268 269 # Generate the raw code. 270 271 self.raw_code = [] 272 273 for item in self.code: 274 275 if isinstance(item, micropython.data.Attr): 276 self.raw_code += item.as_raw(objtable, paramtable) 277 278 elif isinstance(item, Block): 279 assert item.location == len(self.raw_code) 280 self.raw_code += item.as_raw(objtable, paramtable) 281 282 # Using classcode, attrcode, codeaddr, codedetails, instance. 283 284 elif isinstance(item, micropython.data.Class): 285 assert item.instance_template_location == len(self.raw_code) 286 self.raw_code += item.as_raw(objtable, paramtable, 287 with_builtins or item.module.name != "__builtins__" or item.astnode.doc is not None) 288 assert item.location == len(self.raw_code) - 1 289 290 elif isinstance(item, micropython.data.Const): 291 assert item.location == len(self.raw_code) 292 self.raw_code += item.as_raw(objtable, paramtable) 293 294 elif isinstance(item, micropython.data.Function): 295 assert item.location == len(self.raw_code) 296 self.raw_code += item.as_raw(objtable, paramtable) 297 298 # Check the code location only where the code has been generated. 299 300 assert (not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None) or \ 301 item.name is not None and item.code_location == len(self.raw_code) + len(item.defaults) or \ 302 item.name is None and item.code_location == len(self.raw_code) 303 304 elif isinstance(item, micropython.data.Module): 305 assert item.location == len(self.raw_code) 306 self.raw_code += item.as_raw(objtable, paramtable) 307 308 else: 309 self.raw_code.append(item) 310 311 # Set the code body location for items now that the code blocks have 312 # been positioned. 313 314 if isinstance(item, (micropython.data.Class, micropython.data.Function)): 315 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 316 item.code_body_location = item.full_name() 317 else: 318 item.code_body_location = item.get_body_block().location 319 320 # Fix the module locations. 321 322 for module in self.importer.modules_ordered: 323 324 if not with_builtins and module.name == "__builtins__": 325 continue 326 327 module.code_location = module.blocks[0].location 328 329 self.code_location = self.importer.modules["__main__"].code_location 330 return self.raw_code 331 332 def get_object_table(self): 333 334 "Return a table with details of attributes for classes and modules." 335 336 if self.objtable is None: 337 self.importer.vacuum() 338 339 t = self.objtable = micropython.table.ObjectTable() 340 for module in self.importer.get_modules(): 341 t.add(module.full_name(), module.module_attributes()) 342 343 # Add class and instance attributes for all classes, together 344 # with descendant information. 345 346 for obj in module.all_objects: 347 if isinstance(obj, micropython.inspect.Class): 348 attributes = {obj.full_name() : obj} 349 attributes.update(obj.all_attributes()) 350 attributes.update(obj.all_descendants()) 351 t.add(obj.full_name(), attributes) 352 353 return self.objtable 354 355 def get_parameter_table(self): 356 357 "Return a table with details of parameters for functions and methods." 358 359 # Need the object table to get at class details. 360 361 objtable = self.get_object_table() 362 363 if self.paramtable is None: 364 t = self.paramtable = micropython.table.ParameterTable() 365 366 # Visit each module, getting function and method details. 367 368 for module in self.importer.get_modules(): 369 for obj in module.all_objects: 370 if isinstance(obj, micropython.inspect.Function): 371 t.add(obj.full_name(), obj.parameters()) 372 373 # Classes are callable, too. 374 # Take details of the appropriate __init__ method to make an 375 # entry for an instantiation function for the class. 376 377 elif isinstance(obj, micropython.inspect.Class): 378 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 379 380 # Filter out all parameter table entries not referenced by keyword 381 # arguments. 382 383 keyword_names = set() 384 385 for module in self.importer.get_modules(): 386 keyword_names.update(module.keyword_names) 387 388 for function_name, parameters in t.table.items(): 389 for name in parameters.keys(): 390 if name in keyword_names: 391 break 392 else: 393 del t.table[function_name] 394 395 return self.paramtable 396 397 class Importer: 398 399 "An import machine, searching for and loading modules." 400 401 predefined_constants = { 402 "None" : None, 403 "True" : True, 404 "False" : False, 405 #"Ellipsis" : Ellipsis, 406 "NotImplemented" : NotImplemented 407 } 408 409 def __init__(self, path=None, verbose=0, optimisations=None): 410 411 """ 412 Initialise the importer with the given search 'path' - a list of 413 directories to search for Python modules. 414 415 The optional 'verbose' parameter causes output concerning the activities 416 of the object to be produced if set to a true value (not the default). 417 418 The optional 'optimisations' cause certain techniques to be used in 419 reducing program size and improving program efficiency. 420 """ 421 422 self.path = path or [os.getcwd()] 423 self.verbose = verbose 424 self.optimisations = optimisations or set() 425 426 self.modules = {} 427 self.modules_ordered = [] 428 self.loading = set() 429 430 # Constant records. 431 432 self.constant_values = {} 433 self.constant_list = None # cache for constants 434 self.init_predefined_constants() 435 436 # Name records (used to track actual use of names). 437 # Include names which may not be explicitly used in programs. 438 # NOTE: Potentially declare these when inspecting. 439 440 self.names_used = set(["__init__", "__call__", "__bool__"]) 441 442 # Status information. 443 444 self.vacuumed = 0 445 self.finalised = 0 446 447 def get_modules(self): 448 449 "Return all modules known to the importer." 450 451 return self.modules.values() 452 453 def get_module(self, name): 454 455 "Return the module with the given 'name'." 456 457 return self.modules[name] 458 459 # General maintenance. 460 461 def vacuum(self): 462 463 "Tidy up the modules." 464 465 if self.vacuumed: 466 return 467 468 for name, module in self.modules.items(): 469 if module.loaded: 470 module.vacuum() 471 else: 472 del self.modules[name] 473 474 self.vacuumed = 1 475 476 def finalise(self): 477 478 "Finalise the program." 479 480 if self.finalised: 481 return 482 483 self.vacuum() 484 485 for module in self.get_modules(): 486 module.finalise() 487 488 self.finalised = 1 489 490 # Name accounting. 491 492 def use_name(self, name): 493 494 "Register the given 'name' as being used in the program." 495 496 self.names_used.add(name) 497 498 # Constant accounting. 499 500 def init_predefined_constants(self): 501 502 "Ensure the predefined constants." 503 504 for name, value in self.predefined_constants.items(): 505 self.make_constant(value) 506 507 def get_predefined_constant(self, name): 508 509 "Return the predefined constant for the given 'name'." 510 511 return self.make_constant(self.predefined_constants[name]) 512 513 def get_constant(self, value): 514 515 "Return a constant for the given 'value'." 516 517 const = micropython.data.Const(value) 518 return self.constant_values[const] 519 520 def make_constant(self, value): 521 522 "Make and return a constant for the given 'value'." 523 524 # Ensure the presence of the constant's type. 525 526 name = value.__class__.__name__ 527 if self.modules.has_key("__builtins__"): 528 attr = self.modules["__builtins__"].get(name) 529 if attr is not None: 530 attr.set_referenced() 531 532 # Make a constant object and return it. 533 534 const = micropython.data.Const(value) 535 if not self.constant_values.has_key(const): 536 self.constant_values[const] = const 537 return self.constant_values[const] 538 539 def constants(self): 540 541 "Return a list of constants." 542 543 if self.constant_list is None: 544 self.constant_list = list(self.constant_values.values()) 545 546 return self.constant_list 547 548 # Import methods. 549 550 def find_in_path(self, name): 551 552 """ 553 Find the given module 'name' in the search path, returning None where no 554 such module could be found, or a 2-tuple from the 'find' method 555 otherwise. 556 """ 557 558 for d in self.path: 559 m = self.find(d, name) 560 if m: return m 561 return None 562 563 def find(self, d, name): 564 565 """ 566 In the directory 'd', find the given module 'name', where 'name' can 567 either refer to a single file module or to a package. Return None if the 568 'name' cannot be associated with either a file or a package directory, 569 or a 2-tuple from '_find_package' or '_find_module' otherwise. 570 """ 571 572 m = self._find_package(d, name) 573 if m: return m 574 m = self._find_module(d, name) 575 if m: return m 576 return None 577 578 def _find_module(self, d, name): 579 580 """ 581 In the directory 'd', find the given module 'name', returning None where 582 no suitable file exists in the directory, or a 2-tuple consisting of 583 None (indicating that no package directory is involved) and a filename 584 indicating the location of the module. 585 """ 586 587 name_py = name + os.extsep + "py" 588 filename = self._find_file(d, name_py) 589 if filename: 590 return None, filename 591 return None 592 593 def _find_package(self, d, name): 594 595 """ 596 In the directory 'd', find the given package 'name', returning None 597 where no suitable package directory exists, or a 2-tuple consisting of 598 a directory (indicating the location of the package directory itself) 599 and a filename indicating the location of the __init__.py module which 600 declares the package's top-level contents. 601 """ 602 603 filename = self._find_file(d, name) 604 if filename: 605 init_py = "__init__" + os.path.extsep + "py" 606 init_py_filename = self._find_file(filename, init_py) 607 if init_py_filename: 608 return filename, init_py_filename 609 return None 610 611 def _find_file(self, d, filename): 612 613 """ 614 Return the filename obtained when searching the directory 'd' for the 615 given 'filename', or None if no actual file exists for the filename. 616 """ 617 618 filename = os.path.join(d, filename) 619 if os.path.exists(filename): 620 return filename 621 else: 622 return None 623 624 def load(self, name, return_leaf=0): 625 626 """ 627 Load the module or package with the given 'name'. Return an object 628 referencing the loaded module or package, or None if no such module or 629 package exists. 630 """ 631 632 if self.modules.has_key(name) and self.modules[name].loaded: 633 #print "Cached (%s)" % name 634 return self.modules[name] 635 if self.verbose: 636 print "Loading", name 637 638 # Split the name into path components, and try to find the uppermost in 639 # the search path. 640 641 path = name.split(".") 642 m = self.find_in_path(path[0]) 643 if not m: 644 if self.verbose: 645 print "Not found (%s)" % path[0] 646 return None # NOTE: Import error. 647 d, filename = m 648 649 # Either acquire a reference to an already-imported module, or load the 650 # module from a file. 651 652 top = module = self.load_from_file(filename, path[0]) 653 654 # For hierarchical names, traverse each path component... 655 656 if len(path) > 1: 657 if not d: 658 if self.verbose: 659 print "No package (%s)" % filename 660 return None # NOTE: Import error (package not found). 661 else: 662 self.add_submodules(d, module) 663 664 path_so_far = path[:1] 665 for p in path[1:]: 666 path_so_far.append(p) 667 668 # Find the package or module concerned. 669 670 m = self.find(d, p) 671 if not m: 672 if self.verbose: 673 print "Not found (%s)" % p 674 return None # NOTE: Import error. 675 d, filename = m 676 module_name = ".".join(path_so_far) 677 678 # Either reference an imported module or load one from a file. 679 680 submodule = self.load_from_file(filename, module_name) 681 682 if d: 683 self.add_submodules(d, module) 684 685 # Store the submodule within its parent module. 686 687 module.set_module(p, submodule) 688 module = submodule 689 690 # Return either the deepest or the uppermost module. 691 692 if return_leaf: 693 return module 694 else: 695 return top 696 697 def load_from_file(self, name, module_name=None): 698 699 """ 700 Load the module with the given 'name' (which may be a full module path). 701 """ 702 703 if module_name is None: 704 module_name = "__main__" 705 706 module = self.add_module(module_name) 707 if not module.loaded and module not in self.loading: 708 self.loading.add(module) 709 #print "Parsing", name 710 module.parse(name) 711 #print "Done", name 712 self.loading.remove(module) 713 module.loaded = 1 714 715 # Record the module. 716 717 #print "Loaded", module_name, "with namespace", module.namespace.keys() 718 return module 719 720 def add_module(self, module_name): 721 722 """ 723 Return the module with the given 'module_name', adding a new module 724 object if one does not already exist. 725 """ 726 727 if not self.modules.has_key(module_name): 728 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 729 self.modules_ordered.append(module) 730 else: 731 module = self.modules[module_name] 732 return module 733 734 def add_submodules(self, pathname, module): 735 736 """ 737 Work around insufficient __all__ declarations and examine the directory 738 with the given 'pathname', adding submodules to the given 'module'. 739 """ 740 741 for filename in os.listdir(pathname): 742 submodule, ext = os.path.splitext(filename) 743 if ext not in ("", ".py"): 744 continue 745 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 746 747 # vim: tabstop=4 expandtab shiftwidth=4