1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 51 try: 52 set 53 except NameError: 54 from sets import Set as set 55 56 class Program: 57 58 "This class supports the generation of a program image." 59 60 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 61 62 def __init__(self, importer, optimisations=None): 63 64 """ 65 Initialise the program representation with an 'importer' which is able 66 to locate and load Python modules. 67 68 The optional 'optimisations' cause certain techniques to be used in 69 reducing program size and improving program efficiency. 70 """ 71 72 self.importer = importer 73 self.optimisations = optimisations or set() 74 75 # Remember the tables once generated. 76 77 self.objtable = None 78 self.paramtable = None 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def get_importer(self): 86 return self.importer 87 88 # Access to finalised program information. 89 90 def get_image(self, with_builtins=0): 91 92 "Return a dictionary mapping modules to structures." 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 self.importer.finalise() 100 101 self.code = [] 102 103 # Append constants to the image. 104 105 for pos, const in enumerate(self.importer.constants()): 106 self.code.append(const) 107 108 last_module = self.importer.modules_ordered[-1] 109 110 for module in self.importer.modules_ordered: 111 pos = len(self.code) 112 suppress_builtins = not with_builtins and module.name == "__builtins__" 113 114 # Position the module in the image and make a translation. 115 116 trans = micropython.ast.Translation(module, self) 117 118 # Add header details. 119 120 self.code.append(module) 121 pos += 1 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 pos += len(attributes.keys()) 128 129 # Append classes and functions to the image. 130 131 for obj in module.all_objects: 132 if isinstance(obj, micropython.inspect.Class): 133 134 # Add header details. 135 136 self.code.append(obj) 137 pos += 1 138 139 # Append class attributes to the image. 140 141 attributes = obj.class_attributes() 142 self.code += obj.attributes_as_list() 143 pos += len(attributes.keys()) 144 145 # Omit built-in function code where requested. 146 147 if suppress_builtins and not obj.astnode.doc is not None: 148 continue 149 150 # Generate the instantiator/initialiser. 151 # Append the function code to the image. 152 153 instantiator = obj.get_instantiator() 154 code = trans.get_instantiator_code(obj) 155 self.code += code 156 pos += len(code) 157 158 # Class-level code is generated separately at the module 159 # level, and the code location is set within the code 160 # generation process for the module. 161 162 elif isinstance(obj, micropython.inspect.Function): 163 164 # Add header details. 165 166 self.code.append(obj) 167 pos += 1 168 169 # Append any default values to the image. 170 # Only do this for named functions (not lambdas). 171 172 if obj.name is not None: 173 self.code += obj.default_attrs 174 pos += len(obj.default_attrs) 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and not obj.astnode.doc is not None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 pos += len(code) 187 188 # Omit built-in module code where requested. 189 190 if suppress_builtins: 191 pass 192 193 # Append the module top-level code to the image. 194 195 else: 196 code = trans.get_module_code() 197 self.code += code 198 pos += len(code) 199 200 return self.code 201 202 def get_raw_image(self, with_builtins=0): 203 204 "Return the raw image representation of the program." 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 215 for item in self.code: 216 217 # Blocks are positioned leaving space for their expansion. 218 219 if isinstance(item, Block): 220 item.location = pos 221 pos += len(item.code) 222 223 # Other multi-location objects. 224 225 elif isinstance(item, ( 226 micropython.data.Class, micropython.data.Const, 227 micropython.data.Function, micropython.data.Module 228 )): 229 230 if isinstance(item, micropython.data.Class): 231 232 # Append a template of an instance for use when 233 # instantiating classes. 234 235 item.instance_template_location = pos 236 pos += 1 237 238 # Record the location of the object. 239 240 item.location = pos 241 pos += 1 242 243 # Code and details are associated with certain objects. 244 245 if isinstance(item, micropython.data.Function): 246 247 # Set the code location only where the code has been 248 # generated. 249 250 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 251 item.code_location = item.full_name() 252 253 # Skip any defaults for named functions. 254 255 elif item.name is not None: 256 item.code_location = pos + len(item.defaults) 257 258 # Skip any defaults for lambda functions. 259 260 else: 261 item.code_location = pos 262 263 elif isinstance(item, micropython.data.Const): 264 pos += len(item.raw_data()) 265 266 else: 267 pos += 1 268 269 # Generate the raw code. 270 271 self.raw_code = [] 272 273 for item in self.code: 274 275 if isinstance(item, micropython.data.Attr): 276 self.raw_code += item.as_raw(objtable, paramtable) 277 278 elif isinstance(item, Block): 279 assert item.location == len(self.raw_code) 280 self.raw_code += item.as_raw(objtable, paramtable) 281 282 elif isinstance(item, micropython.data.Class): 283 assert item.instance_template_location == len(self.raw_code) 284 self.raw_code += item.as_raw(objtable, paramtable, 285 with_builtins or item.module.name != "__builtins__" or item.astnode.doc is not None) 286 assert item.location == len(self.raw_code) - 1 287 288 elif isinstance(item, micropython.data.Const): 289 assert item.location == len(self.raw_code) 290 self.raw_code += item.as_raw(objtable, paramtable) 291 292 elif isinstance(item, micropython.data.Function): 293 assert item.location == len(self.raw_code) 294 self.raw_code += item.as_raw(objtable, paramtable) 295 296 # Check the code location only where the code has been generated. 297 298 assert (not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None) or \ 299 item.name is not None and item.code_location == len(self.raw_code) + len(item.defaults) or \ 300 item.name is None and item.code_location == len(self.raw_code) 301 302 elif isinstance(item, micropython.data.Module): 303 assert item.location == len(self.raw_code) 304 self.raw_code += item.as_raw(objtable, paramtable) 305 306 else: 307 self.raw_code.append(item) 308 309 # Set the code body location for items now that the code blocks have 310 # been positioned. 311 312 if isinstance(item, (micropython.data.Class, micropython.data.Function)): 313 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 314 item.code_body_location = item.full_name() 315 else: 316 item.code_body_location = item.get_body_block().location 317 318 # Fix the module locations. 319 320 for module in self.importer.modules_ordered: 321 322 if not with_builtins and module.name == "__builtins__": 323 continue 324 325 module.code_location = module.blocks[0].location 326 327 self.code_location = self.importer.modules["__main__"].code_location 328 return self.raw_code 329 330 def get_object_table(self): 331 332 "Return a table with details of attributes for classes and modules." 333 334 if self.objtable is None: 335 self.importer.vacuum() 336 337 t = self.objtable = micropython.table.ObjectTable() 338 for module in self.importer.get_modules(): 339 t.add(module.full_name(), module.module_attributes()) 340 341 # Add class and instance attributes for all classes, together 342 # with descendant information. 343 344 for obj in module.all_objects: 345 if isinstance(obj, micropython.inspect.Class): 346 347 # Prevent ambiguous classes. 348 349 full_name = obj.full_name() 350 name = obj.name 351 352 if module.has_key(name) and module[name].defines_ambiguous_class(): 353 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 354 355 # Define a table entry for the class. 356 357 attributes = {full_name : obj} 358 attributes.update(obj.all_attributes()) 359 attributes.update(obj.all_descendants()) 360 t.add(full_name, attributes) 361 362 return self.objtable 363 364 def get_parameter_table(self): 365 366 "Return a table with details of parameters for functions and methods." 367 368 # Need the object table to get at class details. 369 370 objtable = self.get_object_table() 371 372 if self.paramtable is None: 373 t = self.paramtable = micropython.table.ParameterTable() 374 375 # Visit each module, getting function and method details. 376 377 for module in self.importer.get_modules(): 378 for obj in module.all_objects: 379 if isinstance(obj, micropython.inspect.Function): 380 t.add(obj.full_name(), obj.parameters()) 381 382 # Classes are callable, too. 383 # Take details of the appropriate __init__ method to make an 384 # entry for an instantiation function for the class. 385 386 elif isinstance(obj, micropython.inspect.Class): 387 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 388 389 # Filter out all parameter table entries not referenced by keyword 390 # arguments. 391 392 keyword_names = set() 393 394 for module in self.importer.get_modules(): 395 keyword_names.update(module.keyword_names) 396 397 for function_name, parameters in t.table.items(): 398 for name in parameters.keys(): 399 if name in keyword_names: 400 break 401 else: 402 del t.table[function_name] 403 404 return self.paramtable 405 406 class Importer: 407 408 "An import machine, searching for and loading modules." 409 410 predefined_constants = { 411 "None" : None, 412 "True" : True, 413 "False" : False, 414 #"Ellipsis" : Ellipsis, 415 "NotImplemented" : NotImplemented 416 } 417 418 def __init__(self, path=None, verbose=0, optimisations=None): 419 420 """ 421 Initialise the importer with the given search 'path' - a list of 422 directories to search for Python modules. 423 424 The optional 'verbose' parameter causes output concerning the activities 425 of the object to be produced if set to a true value (not the default). 426 427 The optional 'optimisations' cause certain techniques to be used in 428 reducing program size and improving program efficiency. 429 """ 430 431 self.path = path or [os.getcwd()] 432 self.verbose = verbose 433 self.optimisations = optimisations or set() 434 435 self.modules = {} 436 self.modules_ordered = [] 437 self.loading = set() 438 439 # Constant records. 440 441 self.constant_values = {} 442 self.constant_list = None # cache for constants 443 self.init_predefined_constants() 444 445 # Name records (used to track actual use of names). 446 # Include names which may not be explicitly used in programs. 447 # NOTE: Potentially declare these when inspecting. 448 449 self.names_used = None 450 self.name_references = {} 451 452 # Status information. 453 454 self.vacuumed = 0 455 self.finalised = 0 456 457 def get_modules(self): 458 459 "Return all modules known to the importer." 460 461 return self.modules.values() 462 463 def get_module(self, name): 464 465 "Return the module with the given 'name'." 466 467 return self.modules[name] 468 469 # General maintenance. 470 471 def vacuum(self): 472 473 "Tidy up the modules." 474 475 if self.vacuumed: 476 return 477 478 for name, module in self.modules.items(): 479 if module.loaded: 480 module.vacuum() 481 else: 482 del self.modules[name] 483 484 self.vacuumed = 1 485 486 def finalise(self): 487 488 "Finalise the program." 489 490 if self.finalised: 491 return 492 493 self.vacuum() 494 495 for module in self.get_modules(): 496 module.finalise() 497 498 self.finalised = 1 499 500 # Name accounting. 501 502 def use_name(self, name, from_name): 503 504 """ 505 Register the given 'name' as being used in the program from within an 506 object with the specified 'from_name'. 507 """ 508 509 if not self.name_references.has_key(from_name): 510 self.name_references[from_name] = set() 511 self.name_references[from_name].add(name) 512 513 def uses_name(self, name): 514 515 "Return whether the given 'name' is used." 516 517 if self.names_used is None: 518 self.names_used = set(["bool", "__init__", "__call__", "__bool__", "__builtins__", "__main__"]) 519 self._collect_names("__builtins__") 520 self._collect_names("__main__") 521 522 return name in self.names_used 523 524 def _collect_names(self, from_name): 525 for name in self.name_references.get(from_name, []): 526 if name not in self.names_used: 527 self.names_used.add(name) 528 self._collect_names(name) 529 530 # Constant accounting. 531 532 def init_predefined_constants(self): 533 534 "Ensure the predefined constants." 535 536 for name, value in self.predefined_constants.items(): 537 self.make_constant(value) 538 539 def get_predefined_constant(self, name): 540 541 "Return the predefined constant for the given 'name'." 542 543 return self.make_constant(self.predefined_constants[name]) 544 545 def get_constant(self, value): 546 547 "Return a constant for the given 'value'." 548 549 const = micropython.data.Const(value) 550 return self.constant_values[const] 551 552 def get_constant_type_name(self, value): 553 return value.__class__.__name__ 554 555 def make_constant(self, value): 556 557 "Make and return a constant for the given 'value'." 558 559 # Ensure the presence of the constant's type. 560 561 name = self.get_constant_type_name(value) 562 if self.modules.has_key("__builtins__"): 563 attr = self.modules["__builtins__"].get(name) 564 565 # Make a constant object and return it. 566 567 const = micropython.data.Const(value) 568 if not self.constant_values.has_key(const): 569 self.constant_values[const] = const 570 return self.constant_values[const] 571 572 def constants(self): 573 574 "Return a list of constants." 575 576 if self.constant_list is None: 577 self.constant_list = list(self.constant_values.values()) 578 579 return self.constant_list 580 581 # Import methods. 582 583 def find_in_path(self, name): 584 585 """ 586 Find the given module 'name' in the search path, returning None where no 587 such module could be found, or a 2-tuple from the 'find' method 588 otherwise. 589 """ 590 591 for d in self.path: 592 m = self.find(d, name) 593 if m: return m 594 return None 595 596 def find(self, d, name): 597 598 """ 599 In the directory 'd', find the given module 'name', where 'name' can 600 either refer to a single file module or to a package. Return None if the 601 'name' cannot be associated with either a file or a package directory, 602 or a 2-tuple from '_find_package' or '_find_module' otherwise. 603 """ 604 605 m = self._find_package(d, name) 606 if m: return m 607 m = self._find_module(d, name) 608 if m: return m 609 return None 610 611 def _find_module(self, d, name): 612 613 """ 614 In the directory 'd', find the given module 'name', returning None where 615 no suitable file exists in the directory, or a 2-tuple consisting of 616 None (indicating that no package directory is involved) and a filename 617 indicating the location of the module. 618 """ 619 620 name_py = name + os.extsep + "py" 621 filename = self._find_file(d, name_py) 622 if filename: 623 return None, filename 624 return None 625 626 def _find_package(self, d, name): 627 628 """ 629 In the directory 'd', find the given package 'name', returning None 630 where no suitable package directory exists, or a 2-tuple consisting of 631 a directory (indicating the location of the package directory itself) 632 and a filename indicating the location of the __init__.py module which 633 declares the package's top-level contents. 634 """ 635 636 filename = self._find_file(d, name) 637 if filename: 638 init_py = "__init__" + os.path.extsep + "py" 639 init_py_filename = self._find_file(filename, init_py) 640 if init_py_filename: 641 return filename, init_py_filename 642 return None 643 644 def _find_file(self, d, filename): 645 646 """ 647 Return the filename obtained when searching the directory 'd' for the 648 given 'filename', or None if no actual file exists for the filename. 649 """ 650 651 filename = os.path.join(d, filename) 652 if os.path.exists(filename): 653 return filename 654 else: 655 return None 656 657 def load(self, name, return_leaf=0): 658 659 """ 660 Load the module or package with the given 'name'. Return an object 661 referencing the loaded module or package, or None if no such module or 662 package exists. 663 """ 664 665 if self.modules.has_key(name) and self.modules[name].loaded: 666 #print "Cached (%s)" % name 667 return self.modules[name] 668 if self.verbose: 669 print "Loading", name 670 671 # Split the name into path components, and try to find the uppermost in 672 # the search path. 673 674 path = name.split(".") 675 m = self.find_in_path(path[0]) 676 if not m: 677 if self.verbose: 678 print "Not found (%s)" % path[0] 679 return None # NOTE: Import error. 680 d, filename = m 681 682 # Either acquire a reference to an already-imported module, or load the 683 # module from a file. 684 685 top = module = self.load_from_file(filename, path[0]) 686 687 # For hierarchical names, traverse each path component... 688 689 if len(path) > 1: 690 if not d: 691 if self.verbose: 692 print "No package (%s)" % filename 693 return None # NOTE: Import error (package not found). 694 else: 695 self.add_submodules(d, module) 696 697 path_so_far = path[:1] 698 for p in path[1:]: 699 path_so_far.append(p) 700 701 # Find the package or module concerned. 702 703 m = self.find(d, p) 704 if not m: 705 if self.verbose: 706 print "Not found (%s)" % p 707 return None # NOTE: Import error. 708 d, filename = m 709 module_name = ".".join(path_so_far) 710 711 # Either reference an imported module or load one from a file. 712 713 submodule = self.load_from_file(filename, module_name) 714 715 if d: 716 self.add_submodules(d, module) 717 718 # Store the submodule within its parent module. 719 720 module.set_module(p, submodule) 721 module = submodule 722 723 # Return either the deepest or the uppermost module. 724 725 if return_leaf: 726 return module 727 else: 728 return top 729 730 def load_from_file(self, name, module_name=None): 731 732 """ 733 Load the module with the given 'name' (which may be a full module path). 734 """ 735 736 if module_name is None: 737 module_name = "__main__" 738 739 module = self.add_module(module_name) 740 if not module.loaded and module not in self.loading: 741 self.loading.add(module) 742 #print "Parsing", name 743 module.parse(name) 744 #print "Done", name 745 self.loading.remove(module) 746 module.loaded = 1 747 748 # Record the module. 749 750 #print "Loaded", module_name, "with namespace", module.namespace.keys() 751 return module 752 753 def add_module(self, module_name): 754 755 """ 756 Return the module with the given 'module_name', adding a new module 757 object if one does not already exist. 758 """ 759 760 if not self.modules.has_key(module_name): 761 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 762 self.modules_ordered.append(module) 763 else: 764 module = self.modules[module_name] 765 return module 766 767 def add_submodules(self, pathname, module): 768 769 """ 770 Work around insufficient __all__ declarations and examine the directory 771 with the given 'pathname', adding submodules to the given 'module'. 772 """ 773 774 for filename in os.listdir(pathname): 775 submodule, ext = os.path.splitext(filename) 776 if ext not in ("", ".py"): 777 continue 778 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 779 780 # vim: tabstop=4 expandtab shiftwidth=4