1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 51 try: 52 set 53 except NameError: 54 from sets import Set as set 55 56 class Program: 57 58 "This class supports the generation of a program image." 59 60 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 61 62 def __init__(self, importer, optimisations=None): 63 64 """ 65 Initialise the program representation with an 'importer' which is able 66 to locate and load Python modules. 67 68 The optional 'optimisations' cause certain techniques to be used in 69 reducing program size and improving program efficiency. 70 """ 71 72 self.importer = importer 73 self.optimisations = optimisations or set() 74 75 # Remember the tables once generated. 76 77 self.objtable = None 78 self.paramtable = None 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def get_importer(self): 86 return self.importer 87 88 # Access to finalised program information. 89 90 def get_image(self, with_builtins=0): 91 92 "Return a dictionary mapping modules to structures." 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 self.importer.finalise() 100 101 self.code = [] 102 103 # Append constants to the image. 104 105 for pos, const in enumerate(self.importer.constants()): 106 self.code.append(const) 107 108 last_module = self.importer.modules_ordered[-1] 109 110 for module in self.importer.modules_ordered: 111 pos = len(self.code) 112 suppress_builtins = not with_builtins and module.name == "__builtins__" 113 114 # Position the module in the image and make a translation. 115 116 trans = micropython.ast.Translation(module, self) 117 118 # Add header details. 119 120 self.code.append(module) 121 pos += 1 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 pos += len(attributes.keys()) 128 129 # Append classes and functions to the image. 130 131 for obj in module.all_objects: 132 if isinstance(obj, micropython.inspect.Class): 133 134 # Add header details. 135 136 self.code.append(obj) 137 pos += 1 138 139 # Append class attributes to the image. 140 141 attributes = obj.class_attributes() 142 self.code += obj.attributes_as_list() 143 pos += len(attributes.keys()) 144 145 # Omit built-in function code where requested. 146 147 if suppress_builtins and not obj.astnode.doc is not None: 148 continue 149 150 # Generate the instantiator/initialiser. 151 # Append the function code to the image. 152 153 instantiator = obj.get_instantiator() 154 code = trans.get_instantiator_code(obj) 155 self.code += code 156 pos += len(code) 157 158 # Class-level code is generated separately at the module 159 # level, and the code location is set within the code 160 # generation process for the module. 161 162 elif isinstance(obj, micropython.inspect.Function): 163 164 # Add header details. 165 166 self.code.append(obj) 167 pos += 1 168 169 # Append any default values to the image. 170 # Only do this for named functions (not lambdas). 171 172 if obj.name is not None: 173 self.code += obj.default_attrs 174 pos += len(obj.default_attrs) 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and not obj.astnode.doc is not None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 pos += len(code) 187 188 # Omit built-in module code where requested. 189 190 if suppress_builtins: 191 pass 192 193 # Append the module top-level code to the image. 194 195 else: 196 code = trans.get_module_code() 197 self.code += code 198 pos += len(code) 199 200 return self.code 201 202 def get_raw_image(self, with_builtins=0): 203 204 "Return the raw image representation of the program." 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 215 for item in self.code: 216 217 # Blocks are positioned leaving space for their expansion. 218 219 if isinstance(item, Block): 220 item.location = pos 221 pos += len(item.code) 222 223 # Other multi-location objects. 224 225 elif isinstance(item, ( 226 micropython.data.Class, micropython.data.Const, 227 micropython.data.Function, micropython.data.Module 228 )): 229 230 if isinstance(item, micropython.data.Class): 231 232 # Append a template of an instance for use when 233 # instantiating classes. 234 235 item.instance_template_location = pos 236 pos += 1 237 238 # Record the location of the object. 239 240 item.location = pos 241 pos += 1 242 243 # Code and details are associated with certain objects. 244 245 if isinstance(item, micropython.data.Function): 246 247 # Set the code location only where the code has been 248 # generated. 249 250 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 251 item.code_location = item.full_name() 252 253 # Skip any defaults for named functions. 254 255 elif item.name is not None: 256 item.code_location = pos + len(item.defaults) 257 258 # Skip any defaults for lambda functions. 259 260 else: 261 item.code_location = pos 262 263 elif isinstance(item, micropython.data.Const): 264 pos += len(item.raw_data()) 265 266 else: 267 pos += 1 268 269 # Generate the raw code. 270 271 self.raw_code = [] 272 273 for item in self.code: 274 275 if isinstance(item, micropython.data.Attr): 276 self.raw_code += item.as_raw(objtable, paramtable) 277 278 elif isinstance(item, Block): 279 assert item.location == len(self.raw_code) 280 self.raw_code += item.as_raw(objtable, paramtable) 281 282 elif isinstance(item, micropython.data.Class): 283 assert item.instance_template_location == len(self.raw_code) 284 self.raw_code += item.as_raw(objtable, paramtable, 285 with_builtins or item.module.name != "__builtins__" or item.astnode.doc is not None) 286 assert item.location == len(self.raw_code) - 1 287 288 elif isinstance(item, micropython.data.Const): 289 assert item.location == len(self.raw_code) 290 self.raw_code += item.as_raw(objtable, paramtable) 291 292 elif isinstance(item, micropython.data.Function): 293 assert item.location == len(self.raw_code) 294 self.raw_code += item.as_raw(objtable, paramtable) 295 296 # Check the code location only where the code has been generated. 297 298 assert (not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None) or \ 299 item.name is not None and item.code_location == len(self.raw_code) + len(item.defaults) or \ 300 item.name is None and item.code_location == len(self.raw_code) 301 302 elif isinstance(item, micropython.data.Module): 303 assert item.location == len(self.raw_code) 304 self.raw_code += item.as_raw(objtable, paramtable) 305 306 else: 307 self.raw_code.append(item) 308 309 # Set the code body location for items now that the code blocks have 310 # been positioned. 311 312 if isinstance(item, (micropython.data.Class, micropython.data.Function)): 313 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 314 item.code_body_location = item.full_name() 315 else: 316 item.code_body_location = item.get_body_block().location 317 318 # Fix the module locations. 319 320 for module in self.importer.modules_ordered: 321 322 if not with_builtins and module.name == "__builtins__": 323 continue 324 325 module.code_location = module.blocks[0].location 326 327 self.code_location = self.importer.modules["__main__"].code_location 328 return self.raw_code 329 330 def get_object_table(self): 331 332 "Return a table with details of attributes for classes and modules." 333 334 if self.objtable is None: 335 self.importer.vacuum() 336 337 t = self.objtable = micropython.table.ObjectTable() 338 for module in self.importer.get_modules(): 339 t.add(module.full_name(), module.module_attributes()) 340 341 # Add class and instance attributes for all classes, together 342 # with descendant information. 343 344 for obj in module.all_objects: 345 if isinstance(obj, micropython.inspect.Class): 346 347 # Prevent ambiguous classes. 348 349 full_name = obj.full_name() 350 name = obj.name 351 352 if module.has_key(name) and module[name].defines_ambiguous_class(): 353 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 354 355 # Define a table entry for the class. 356 357 attributes = {full_name : obj} 358 attributes.update(obj.all_attributes()) 359 attributes.update(obj.all_descendants()) 360 t.add(full_name, attributes) 361 362 return self.objtable 363 364 def get_parameter_table(self): 365 366 "Return a table with details of parameters for functions and methods." 367 368 # Need the object table to get at class details. 369 370 objtable = self.get_object_table() 371 372 if self.paramtable is None: 373 t = self.paramtable = micropython.table.ParameterTable() 374 375 # Visit each module, getting function and method details. 376 377 for module in self.importer.get_modules(): 378 for obj in module.all_objects: 379 if isinstance(obj, micropython.inspect.Function): 380 t.add(obj.full_name(), obj.parameters()) 381 382 # Classes are callable, too. 383 # Take details of the appropriate __init__ method to make an 384 # entry for an instantiation function for the class. 385 386 elif isinstance(obj, micropython.inspect.Class): 387 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 388 389 # Filter out all parameter table entries not referenced by keyword 390 # arguments. 391 392 keyword_names = set() 393 394 for module in self.importer.get_modules(): 395 keyword_names.update(module.keyword_names) 396 397 for function_name, parameters in t.table.items(): 398 for name in parameters.keys(): 399 if name in keyword_names: 400 break 401 else: 402 del t.table[function_name] 403 404 return self.paramtable 405 406 class Importer: 407 408 "An import machine, searching for and loading modules." 409 410 predefined_constants = { 411 "None" : None, 412 "True" : True, 413 "False" : False, 414 #"Ellipsis" : Ellipsis, 415 "NotImplemented" : NotImplemented 416 } 417 418 def __init__(self, path=None, verbose=0, optimisations=None): 419 420 """ 421 Initialise the importer with the given search 'path' - a list of 422 directories to search for Python modules. 423 424 The optional 'verbose' parameter causes output concerning the activities 425 of the object to be produced if set to a true value (not the default). 426 427 The optional 'optimisations' cause certain techniques to be used in 428 reducing program size and improving program efficiency. 429 """ 430 431 self.path = path or [os.getcwd()] 432 self.verbose = verbose 433 self.optimisations = optimisations or set() 434 435 self.modules = {} 436 self.modules_ordered = [] 437 self.loading = set() 438 439 # Constant records. 440 441 self.constant_values = {} 442 self.constant_list = None # cache for constants 443 self.init_predefined_constants() 444 445 # Name records (used to track actual use of names). 446 # Include names which may not be explicitly used in programs. 447 # NOTE: Potentially declare these when inspecting. 448 449 self.names_used = None 450 self.name_references = {} 451 452 # Status information. 453 454 self.vacuumed = 0 455 self.finalised = 0 456 457 def get_modules(self): 458 459 "Return all modules known to the importer." 460 461 return self.modules.values() 462 463 def get_module(self, name): 464 465 "Return the module with the given 'name'." 466 467 return self.modules[name] 468 469 # General maintenance. 470 471 def vacuum(self): 472 473 "Tidy up the modules." 474 475 if self.vacuumed: 476 return 477 478 for name, module in self.modules.items(): 479 if module.loaded: 480 module.vacuum() 481 else: 482 del self.modules[name] 483 484 self.vacuumed = 1 485 486 def finalise(self): 487 488 "Finalise the program." 489 490 if self.finalised: 491 return 492 493 self.vacuum() 494 495 for module in self.get_modules(): 496 module.finalise() 497 498 self.finalised = 1 499 500 # Name accounting. 501 502 def use_name(self, name, from_name): 503 504 """ 505 Register the given 'name' as being used in the program from within an 506 object with the specified 'from_name'. 507 """ 508 509 if not self.name_references.has_key(from_name): 510 self.name_references[from_name] = set() 511 self.name_references[from_name].add(name) 512 513 def uses_name(self, name): 514 515 "Return whether the given 'name' is used." 516 517 if self.names_used is None: 518 self.names_used = set(["__init__", "__call__", "__bool__"]) 519 self._collect_names("__main__") 520 521 return name in self.names_used 522 523 def _collect_names(self, from_name): 524 for name in self.name_references.get(from_name, []): 525 if name not in self.names_used: 526 self.names_used.add(name) 527 self._collect_names(name) 528 529 # Constant accounting. 530 531 def init_predefined_constants(self): 532 533 "Ensure the predefined constants." 534 535 for name, value in self.predefined_constants.items(): 536 self.make_constant(value) 537 538 def get_predefined_constant(self, name): 539 540 "Return the predefined constant for the given 'name'." 541 542 return self.make_constant(self.predefined_constants[name]) 543 544 def get_constant(self, value): 545 546 "Return a constant for the given 'value'." 547 548 const = micropython.data.Const(value) 549 return self.constant_values[const] 550 551 def make_constant(self, value): 552 553 "Make and return a constant for the given 'value'." 554 555 # Ensure the presence of the constant's type. 556 557 name = value.__class__.__name__ 558 if self.modules.has_key("__builtins__"): 559 attr = self.modules["__builtins__"].get(name) 560 if attr is not None: 561 attr.set_referenced() 562 563 # Make a constant object and return it. 564 565 const = micropython.data.Const(value) 566 if not self.constant_values.has_key(const): 567 self.constant_values[const] = const 568 return self.constant_values[const] 569 570 def constants(self): 571 572 "Return a list of constants." 573 574 if self.constant_list is None: 575 self.constant_list = list(self.constant_values.values()) 576 577 return self.constant_list 578 579 # Import methods. 580 581 def find_in_path(self, name): 582 583 """ 584 Find the given module 'name' in the search path, returning None where no 585 such module could be found, or a 2-tuple from the 'find' method 586 otherwise. 587 """ 588 589 for d in self.path: 590 m = self.find(d, name) 591 if m: return m 592 return None 593 594 def find(self, d, name): 595 596 """ 597 In the directory 'd', find the given module 'name', where 'name' can 598 either refer to a single file module or to a package. Return None if the 599 'name' cannot be associated with either a file or a package directory, 600 or a 2-tuple from '_find_package' or '_find_module' otherwise. 601 """ 602 603 m = self._find_package(d, name) 604 if m: return m 605 m = self._find_module(d, name) 606 if m: return m 607 return None 608 609 def _find_module(self, d, name): 610 611 """ 612 In the directory 'd', find the given module 'name', returning None where 613 no suitable file exists in the directory, or a 2-tuple consisting of 614 None (indicating that no package directory is involved) and a filename 615 indicating the location of the module. 616 """ 617 618 name_py = name + os.extsep + "py" 619 filename = self._find_file(d, name_py) 620 if filename: 621 return None, filename 622 return None 623 624 def _find_package(self, d, name): 625 626 """ 627 In the directory 'd', find the given package 'name', returning None 628 where no suitable package directory exists, or a 2-tuple consisting of 629 a directory (indicating the location of the package directory itself) 630 and a filename indicating the location of the __init__.py module which 631 declares the package's top-level contents. 632 """ 633 634 filename = self._find_file(d, name) 635 if filename: 636 init_py = "__init__" + os.path.extsep + "py" 637 init_py_filename = self._find_file(filename, init_py) 638 if init_py_filename: 639 return filename, init_py_filename 640 return None 641 642 def _find_file(self, d, filename): 643 644 """ 645 Return the filename obtained when searching the directory 'd' for the 646 given 'filename', or None if no actual file exists for the filename. 647 """ 648 649 filename = os.path.join(d, filename) 650 if os.path.exists(filename): 651 return filename 652 else: 653 return None 654 655 def load(self, name, return_leaf=0): 656 657 """ 658 Load the module or package with the given 'name'. Return an object 659 referencing the loaded module or package, or None if no such module or 660 package exists. 661 """ 662 663 if self.modules.has_key(name) and self.modules[name].loaded: 664 #print "Cached (%s)" % name 665 return self.modules[name] 666 if self.verbose: 667 print "Loading", name 668 669 # Split the name into path components, and try to find the uppermost in 670 # the search path. 671 672 path = name.split(".") 673 m = self.find_in_path(path[0]) 674 if not m: 675 if self.verbose: 676 print "Not found (%s)" % path[0] 677 return None # NOTE: Import error. 678 d, filename = m 679 680 # Either acquire a reference to an already-imported module, or load the 681 # module from a file. 682 683 top = module = self.load_from_file(filename, path[0]) 684 685 # For hierarchical names, traverse each path component... 686 687 if len(path) > 1: 688 if not d: 689 if self.verbose: 690 print "No package (%s)" % filename 691 return None # NOTE: Import error (package not found). 692 else: 693 self.add_submodules(d, module) 694 695 path_so_far = path[:1] 696 for p in path[1:]: 697 path_so_far.append(p) 698 699 # Find the package or module concerned. 700 701 m = self.find(d, p) 702 if not m: 703 if self.verbose: 704 print "Not found (%s)" % p 705 return None # NOTE: Import error. 706 d, filename = m 707 module_name = ".".join(path_so_far) 708 709 # Either reference an imported module or load one from a file. 710 711 submodule = self.load_from_file(filename, module_name) 712 713 if d: 714 self.add_submodules(d, module) 715 716 # Store the submodule within its parent module. 717 718 module.set_module(p, submodule) 719 module = submodule 720 721 # Return either the deepest or the uppermost module. 722 723 if return_leaf: 724 return module 725 else: 726 return top 727 728 def load_from_file(self, name, module_name=None): 729 730 """ 731 Load the module with the given 'name' (which may be a full module path). 732 """ 733 734 if module_name is None: 735 module_name = "__main__" 736 737 module = self.add_module(module_name) 738 if not module.loaded and module not in self.loading: 739 self.loading.add(module) 740 #print "Parsing", name 741 module.parse(name) 742 #print "Done", name 743 self.loading.remove(module) 744 module.loaded = 1 745 746 # Record the module. 747 748 #print "Loaded", module_name, "with namespace", module.namespace.keys() 749 return module 750 751 def add_module(self, module_name): 752 753 """ 754 Return the module with the given 'module_name', adding a new module 755 object if one does not already exist. 756 """ 757 758 if not self.modules.has_key(module_name): 759 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 760 self.modules_ordered.append(module) 761 else: 762 module = self.modules[module_name] 763 return module 764 765 def add_submodules(self, pathname, module): 766 767 """ 768 Work around insufficient __all__ declarations and examine the directory 769 with the given 'pathname', adding submodules to the given 'module'. 770 """ 771 772 for filename in os.listdir(pathname): 773 submodule, ext = os.path.splitext(filename) 774 if ext not in ("", ".py"): 775 continue 776 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 777 778 # vim: tabstop=4 expandtab shiftwidth=4