1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 51 try: 52 set 53 except NameError: 54 from sets import Set as set 55 56 class Program: 57 58 "This class supports the generation of a program image." 59 60 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 61 62 def __init__(self, importer, optimisations=None): 63 64 """ 65 Initialise the program representation with an 'importer' which is able 66 to locate and load Python modules. 67 68 The optional 'optimisations' cause certain techniques to be used in 69 reducing program size and improving program efficiency. 70 """ 71 72 self.importer = importer 73 self.optimisations = optimisations or set() 74 75 # Remember the tables once generated. 76 77 self.objtable = None 78 self.paramtable = None 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def get_importer(self): 86 return self.importer 87 88 # Access to finalised program information. 89 90 def get_image(self, with_builtins=0): 91 92 "Return a dictionary mapping modules to structures." 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 self.importer.finalise() 100 101 self.code = [] 102 103 # Append constants to the image. 104 105 for pos, const in enumerate(self.importer.constants()): 106 self.code.append(const) 107 108 last_module = self.importer.modules_ordered[-1] 109 110 for module in self.importer.modules_ordered: 111 pos = len(self.code) 112 suppress_builtins = not with_builtins and module.name == "__builtins__" 113 114 # Position the module in the image and make a translation. 115 116 trans = micropython.ast.Translation(module, self) 117 118 # Add header details. 119 120 self.code.append(module) 121 pos += 1 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 pos += len(attributes.keys()) 128 129 # Append classes and functions to the image. 130 131 for obj in module.all_objects: 132 if isinstance(obj, micropython.inspect.Class): 133 134 # Add header details. 135 136 self.code.append(obj) 137 pos += 1 138 139 # Append class attributes to the image. 140 141 attributes = obj.class_attributes() 142 self.code += obj.attributes_as_list() 143 pos += len(attributes.keys()) 144 145 # Omit built-in function code where requested. 146 147 if suppress_builtins and not obj.astnode.doc is not None: 148 continue 149 150 # Generate the instantiator/initialiser. 151 # Append the function code to the image. 152 153 instantiator = obj.get_instantiator() 154 code = trans.get_instantiator_code(obj) 155 self.code += code 156 pos += len(code) 157 158 # Class-level code is generated separately at the module 159 # level, and the code location is set within the code 160 # generation process for the module. 161 162 elif isinstance(obj, micropython.inspect.Function): 163 164 # Add header details. 165 166 self.code.append(obj) 167 pos += 1 168 169 # Append any default values to the image. 170 # Only do this for named functions (not lambdas). 171 172 if obj.name is not None: 173 self.code += obj.default_attrs 174 pos += len(obj.default_attrs) 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and not obj.astnode.doc is not None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 pos += len(code) 187 188 # Omit built-in module code where requested. 189 190 if suppress_builtins: 191 pass 192 193 # Append the module top-level code to the image. 194 195 else: 196 code = trans.get_module_code() 197 self.code += code 198 pos += len(code) 199 200 return self.code 201 202 def get_raw_image(self, with_builtins=0): 203 204 "Return the raw image representation of the program." 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 215 for item in self.code: 216 217 # Blocks are positioned leaving space for their expansion. 218 219 if isinstance(item, Block): 220 item.location = pos 221 pos += len(item.code) 222 223 # Other multi-location objects. 224 225 elif isinstance(item, ( 226 micropython.data.Class, micropython.data.Const, 227 micropython.data.Function, micropython.data.Module 228 )): 229 230 if isinstance(item, micropython.data.Class): 231 232 # Append a template of an instance for use when 233 # instantiating classes. 234 235 item.instance_template_location = pos 236 pos += 1 237 238 # Record the location of the object. 239 240 item.location = pos 241 pos += 1 242 243 # Code and details are associated with certain objects. 244 245 if isinstance(item, micropython.data.Function): 246 247 # Set the code location only where the code has been 248 # generated. 249 250 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 251 item.code_location = item.full_name() 252 253 # Skip any defaults for named functions. 254 255 elif item.name is not None: 256 item.code_location = pos + len(item.defaults) 257 258 # Skip any defaults for lambda functions. 259 260 else: 261 item.code_location = pos 262 263 elif isinstance(item, micropython.data.Const): 264 pos += len(item.raw_data()) 265 266 else: 267 pos += 1 268 269 # Generate the raw code. 270 271 self.raw_code = [] 272 273 for item in self.code: 274 275 if isinstance(item, micropython.data.Attr): 276 self.raw_code += item.as_raw(objtable, paramtable) 277 278 elif isinstance(item, Block): 279 assert item.location == len(self.raw_code) 280 self.raw_code += item.as_raw(objtable, paramtable) 281 282 elif isinstance(item, micropython.data.Class): 283 assert item.instance_template_location == len(self.raw_code) 284 self.raw_code += item.as_raw(objtable, paramtable, 285 with_builtins or item.module.name != "__builtins__" or item.astnode.doc is not None) 286 assert item.location == len(self.raw_code) - 1 287 288 elif isinstance(item, micropython.data.Const): 289 assert item.location == len(self.raw_code) 290 self.raw_code += item.as_raw(objtable, paramtable) 291 292 elif isinstance(item, micropython.data.Function): 293 assert item.location == len(self.raw_code) 294 self.raw_code += item.as_raw(objtable, paramtable) 295 296 # Check the code location only where the code has been generated. 297 298 assert (not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None) or \ 299 item.name is not None and item.code_location == len(self.raw_code) + len(item.defaults) or \ 300 item.name is None and item.code_location == len(self.raw_code) 301 302 elif isinstance(item, micropython.data.Module): 303 assert item.location == len(self.raw_code) 304 self.raw_code += item.as_raw(objtable, paramtable) 305 306 else: 307 self.raw_code.append(item) 308 309 # Set the code body location for items now that the code blocks have 310 # been positioned. 311 312 if isinstance(item, (micropython.data.Class, micropython.data.Function)): 313 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 314 item.code_body_location = item.full_name() 315 else: 316 item.code_body_location = item.get_body_block().location 317 318 # Fix the module locations. 319 320 for module in self.importer.modules_ordered: 321 322 if not with_builtins and module.name == "__builtins__": 323 continue 324 325 module.code_location = module.blocks[0].location 326 327 self.code_location = self.importer.modules["__main__"].code_location 328 return self.raw_code 329 330 def get_object_table(self): 331 332 "Return a table with details of attributes for classes and modules." 333 334 if self.objtable is None: 335 self.importer.vacuum() 336 337 t = self.objtable = micropython.table.ObjectTable() 338 for module in self.importer.get_modules(): 339 t.add(module.full_name(), module.module_attributes()) 340 341 # Add class and instance attributes for all classes, together 342 # with descendant information. 343 344 for obj in module.all_objects: 345 if isinstance(obj, micropython.inspect.Class): 346 347 # Prevent ambiguous classes. 348 349 full_name = obj.full_name() 350 name = obj.name 351 352 if module.has_key(name) and module[name].defines_ambiguous_class(): 353 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 354 355 # Define a table entry for the class. 356 357 attributes = {full_name : obj} 358 attributes.update(obj.all_attributes()) 359 attributes.update(obj.all_descendants()) 360 t.add(full_name, attributes) 361 362 return self.objtable 363 364 def get_parameter_table(self): 365 366 "Return a table with details of parameters for functions and methods." 367 368 # Need the object table to get at class details. 369 370 objtable = self.get_object_table() 371 372 if self.paramtable is None: 373 t = self.paramtable = micropython.table.ParameterTable() 374 375 # Visit each module, getting function and method details. 376 377 for module in self.importer.get_modules(): 378 for obj in module.all_objects: 379 if isinstance(obj, micropython.inspect.Function): 380 t.add(obj.full_name(), obj.parameters()) 381 382 # Classes are callable, too. 383 # Take details of the appropriate __init__ method to make an 384 # entry for an instantiation function for the class. 385 386 elif isinstance(obj, micropython.inspect.Class): 387 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 388 389 # Filter out all parameter table entries not referenced by keyword 390 # arguments. 391 392 keyword_names = set() 393 394 for module in self.importer.get_modules(): 395 keyword_names.update(module.keyword_names) 396 397 for function_name, parameters in t.table.items(): 398 for name in parameters.keys(): 399 if name in keyword_names: 400 break 401 else: 402 del t.table[function_name] 403 404 return self.paramtable 405 406 class Importer: 407 408 "An import machine, searching for and loading modules." 409 410 predefined_constants = { 411 "None" : None, 412 "True" : True, 413 "False" : False, 414 #"Ellipsis" : Ellipsis, 415 "NotImplemented" : NotImplemented 416 } 417 418 def __init__(self, path=None, verbose=0, optimisations=None): 419 420 """ 421 Initialise the importer with the given search 'path' - a list of 422 directories to search for Python modules. 423 424 The optional 'verbose' parameter causes output concerning the activities 425 of the object to be produced if set to a true value (not the default). 426 427 The optional 'optimisations' cause certain techniques to be used in 428 reducing program size and improving program efficiency. 429 """ 430 431 self.path = path or [os.getcwd()] 432 self.verbose = verbose 433 self.optimisations = optimisations or set() 434 435 self.modules = {} 436 self.modules_ordered = [] 437 self.loading = set() 438 439 # Constant records. 440 441 self.constant_values = {} 442 self.constant_list = None # cache for constants 443 self.init_predefined_constants() 444 445 # Name records (used to track actual use of names). 446 # Include names which may not be explicitly used in programs. 447 # NOTE: Potentially declare these when inspecting. 448 449 self.names_used = set(["__init__", "__call__", "__bool__"]) 450 451 # Status information. 452 453 self.vacuumed = 0 454 self.finalised = 0 455 456 def get_modules(self): 457 458 "Return all modules known to the importer." 459 460 return self.modules.values() 461 462 def get_module(self, name): 463 464 "Return the module with the given 'name'." 465 466 return self.modules[name] 467 468 # General maintenance. 469 470 def vacuum(self): 471 472 "Tidy up the modules." 473 474 if self.vacuumed: 475 return 476 477 for name, module in self.modules.items(): 478 if module.loaded: 479 module.vacuum() 480 else: 481 del self.modules[name] 482 483 self.vacuumed = 1 484 485 def finalise(self): 486 487 "Finalise the program." 488 489 if self.finalised: 490 return 491 492 self.vacuum() 493 494 for module in self.get_modules(): 495 module.finalise() 496 497 self.finalised = 1 498 499 # Name accounting. 500 501 def use_name(self, name): 502 503 "Register the given 'name' as being used in the program." 504 505 self.names_used.add(name) 506 507 # Constant accounting. 508 509 def init_predefined_constants(self): 510 511 "Ensure the predefined constants." 512 513 for name, value in self.predefined_constants.items(): 514 self.make_constant(value) 515 516 def get_predefined_constant(self, name): 517 518 "Return the predefined constant for the given 'name'." 519 520 return self.make_constant(self.predefined_constants[name]) 521 522 def get_constant(self, value): 523 524 "Return a constant for the given 'value'." 525 526 const = micropython.data.Const(value) 527 return self.constant_values[const] 528 529 def make_constant(self, value): 530 531 "Make and return a constant for the given 'value'." 532 533 # Ensure the presence of the constant's type. 534 535 name = value.__class__.__name__ 536 if self.modules.has_key("__builtins__"): 537 attr = self.modules["__builtins__"].get(name) 538 if attr is not None: 539 attr.set_referenced() 540 541 # Make a constant object and return it. 542 543 const = micropython.data.Const(value) 544 if not self.constant_values.has_key(const): 545 self.constant_values[const] = const 546 return self.constant_values[const] 547 548 def constants(self): 549 550 "Return a list of constants." 551 552 if self.constant_list is None: 553 self.constant_list = list(self.constant_values.values()) 554 555 return self.constant_list 556 557 # Import methods. 558 559 def find_in_path(self, name): 560 561 """ 562 Find the given module 'name' in the search path, returning None where no 563 such module could be found, or a 2-tuple from the 'find' method 564 otherwise. 565 """ 566 567 for d in self.path: 568 m = self.find(d, name) 569 if m: return m 570 return None 571 572 def find(self, d, name): 573 574 """ 575 In the directory 'd', find the given module 'name', where 'name' can 576 either refer to a single file module or to a package. Return None if the 577 'name' cannot be associated with either a file or a package directory, 578 or a 2-tuple from '_find_package' or '_find_module' otherwise. 579 """ 580 581 m = self._find_package(d, name) 582 if m: return m 583 m = self._find_module(d, name) 584 if m: return m 585 return None 586 587 def _find_module(self, d, name): 588 589 """ 590 In the directory 'd', find the given module 'name', returning None where 591 no suitable file exists in the directory, or a 2-tuple consisting of 592 None (indicating that no package directory is involved) and a filename 593 indicating the location of the module. 594 """ 595 596 name_py = name + os.extsep + "py" 597 filename = self._find_file(d, name_py) 598 if filename: 599 return None, filename 600 return None 601 602 def _find_package(self, d, name): 603 604 """ 605 In the directory 'd', find the given package 'name', returning None 606 where no suitable package directory exists, or a 2-tuple consisting of 607 a directory (indicating the location of the package directory itself) 608 and a filename indicating the location of the __init__.py module which 609 declares the package's top-level contents. 610 """ 611 612 filename = self._find_file(d, name) 613 if filename: 614 init_py = "__init__" + os.path.extsep + "py" 615 init_py_filename = self._find_file(filename, init_py) 616 if init_py_filename: 617 return filename, init_py_filename 618 return None 619 620 def _find_file(self, d, filename): 621 622 """ 623 Return the filename obtained when searching the directory 'd' for the 624 given 'filename', or None if no actual file exists for the filename. 625 """ 626 627 filename = os.path.join(d, filename) 628 if os.path.exists(filename): 629 return filename 630 else: 631 return None 632 633 def load(self, name, return_leaf=0): 634 635 """ 636 Load the module or package with the given 'name'. Return an object 637 referencing the loaded module or package, or None if no such module or 638 package exists. 639 """ 640 641 if self.modules.has_key(name) and self.modules[name].loaded: 642 #print "Cached (%s)" % name 643 return self.modules[name] 644 if self.verbose: 645 print "Loading", name 646 647 # Split the name into path components, and try to find the uppermost in 648 # the search path. 649 650 path = name.split(".") 651 m = self.find_in_path(path[0]) 652 if not m: 653 if self.verbose: 654 print "Not found (%s)" % path[0] 655 return None # NOTE: Import error. 656 d, filename = m 657 658 # Either acquire a reference to an already-imported module, or load the 659 # module from a file. 660 661 top = module = self.load_from_file(filename, path[0]) 662 663 # For hierarchical names, traverse each path component... 664 665 if len(path) > 1: 666 if not d: 667 if self.verbose: 668 print "No package (%s)" % filename 669 return None # NOTE: Import error (package not found). 670 else: 671 self.add_submodules(d, module) 672 673 path_so_far = path[:1] 674 for p in path[1:]: 675 path_so_far.append(p) 676 677 # Find the package or module concerned. 678 679 m = self.find(d, p) 680 if not m: 681 if self.verbose: 682 print "Not found (%s)" % p 683 return None # NOTE: Import error. 684 d, filename = m 685 module_name = ".".join(path_so_far) 686 687 # Either reference an imported module or load one from a file. 688 689 submodule = self.load_from_file(filename, module_name) 690 691 if d: 692 self.add_submodules(d, module) 693 694 # Store the submodule within its parent module. 695 696 module.set_module(p, submodule) 697 module = submodule 698 699 # Return either the deepest or the uppermost module. 700 701 if return_leaf: 702 return module 703 else: 704 return top 705 706 def load_from_file(self, name, module_name=None): 707 708 """ 709 Load the module with the given 'name' (which may be a full module path). 710 """ 711 712 if module_name is None: 713 module_name = "__main__" 714 715 module = self.add_module(module_name) 716 if not module.loaded and module not in self.loading: 717 self.loading.add(module) 718 #print "Parsing", name 719 module.parse(name) 720 #print "Done", name 721 self.loading.remove(module) 722 module.loaded = 1 723 724 # Record the module. 725 726 #print "Loaded", module_name, "with namespace", module.namespace.keys() 727 return module 728 729 def add_module(self, module_name): 730 731 """ 732 Return the module with the given 'module_name', adding a new module 733 object if one does not already exist. 734 """ 735 736 if not self.modules.has_key(module_name): 737 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 738 self.modules_ordered.append(module) 739 else: 740 module = self.modules[module_name] 741 return module 742 743 def add_submodules(self, pathname, module): 744 745 """ 746 Work around insufficient __all__ declarations and examine the directory 747 with the given 'pathname', adding submodules to the given 'module'. 748 """ 749 750 for filename in os.listdir(pathname): 751 submodule, ext = os.path.splitext(filename) 752 if ext not in ("", ".py"): 753 continue 754 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 755 756 # vim: tabstop=4 expandtab shiftwidth=4