1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 51 try: 52 set 53 except NameError: 54 from sets import Set as set 55 56 class Program: 57 58 "This class supports the generation of a program image." 59 60 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 61 62 def __init__(self, importer, optimisations=None): 63 64 """ 65 Initialise the program representation with an 'importer' which is able 66 to locate and load Python modules. 67 68 The optional 'optimisations' cause certain techniques to be used in 69 reducing program size and improving program efficiency. 70 """ 71 72 self.importer = importer 73 self.optimisations = optimisations or set() 74 75 # Remember the tables once generated. 76 77 self.objtable = None 78 self.paramtable = None 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def get_importer(self): 86 return self.importer 87 88 # Access to finalised program information. 89 90 def get_image(self, with_builtins=0): 91 92 "Return a dictionary mapping modules to structures." 93 94 if self.code is not None: 95 return self.code 96 97 objtable = self.get_object_table() 98 paramtable = self.get_parameter_table() 99 self.importer.finalise() 100 101 self.code = [] 102 103 # Append constants to the image. 104 105 for pos, const in enumerate(self.importer.constants()): 106 self.code.append(const) 107 108 last_module = self.importer.modules_ordered[-1] 109 110 for module in self.importer.modules_ordered: 111 pos = len(self.code) 112 suppress_builtins = not with_builtins and module.name == "__builtins__" 113 114 # Position the module in the image and make a translation. 115 116 trans = micropython.ast.Translation(module, self) 117 118 # Add header details. 119 120 self.code.append(module) 121 pos += 1 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 pos += len(attributes.keys()) 128 129 # Append classes and functions to the image. 130 131 for obj in module.all_objects: 132 if isinstance(obj, micropython.inspect.Class): 133 134 # Add header details. 135 136 self.code.append(obj) 137 pos += 1 138 139 # Append class attributes to the image. 140 141 attributes = obj.class_attributes() 142 self.code += obj.attributes_as_list() 143 pos += len(attributes.keys()) 144 145 # Omit built-in function code where requested. 146 147 if suppress_builtins and not obj.astnode.doc is not None: 148 continue 149 150 # Generate the instantiator/initialiser. 151 # Append the function code to the image. 152 153 instantiator = obj.get_instantiator() 154 code = trans.get_instantiator_code(obj) 155 self.code += code 156 pos += len(code) 157 158 # Class-level code is generated separately at the module 159 # level, and the code location is set within the code 160 # generation process for the module. 161 162 elif isinstance(obj, micropython.inspect.Function): 163 164 # Add header details. 165 166 self.code.append(obj) 167 pos += 1 168 169 # Append any default values to the image. 170 # Only do this for named functions (not lambdas). 171 172 if obj.name is not None: 173 self.code += obj.default_attrs 174 pos += len(obj.default_attrs) 175 176 # Omit built-in function code where requested. 177 178 if suppress_builtins and not obj.astnode.doc is not None: 179 pass 180 181 # Append the function code to the image. 182 183 else: 184 code = trans.get_code(obj) 185 self.code += code 186 pos += len(code) 187 188 # Omit built-in module code where requested. 189 190 if suppress_builtins: 191 pass 192 193 # Append the module top-level code to the image. 194 195 else: 196 code = trans.get_module_code(final=(module is last_module)) 197 self.code += code 198 pos += len(code) 199 200 return self.code 201 202 def get_raw_image(self, with_builtins=0): 203 204 "Return the raw image representation of the program." 205 206 self.get_image(with_builtins) 207 208 objtable = self.get_object_table() 209 paramtable = self.get_parameter_table() 210 211 # Position the objects. 212 213 pos = 0 214 215 for item in self.code: 216 217 # Blocks are positioned leaving space for their expansion. 218 219 if isinstance(item, Block): 220 item.location = pos 221 pos += len(item.code) 222 223 # Other multi-location objects. 224 225 elif isinstance(item, ( 226 micropython.data.Class, micropython.data.Const, 227 micropython.data.Function, micropython.data.Module 228 )): 229 230 if isinstance(item, micropython.data.Class): 231 232 # Append a template of an instance for use when 233 # instantiating classes. 234 235 item.instance_template_location = pos 236 pos += 1 237 238 # Record the location of the object. 239 240 item.location = pos 241 pos += 1 242 243 # Code and details are associated with certain objects. 244 245 if isinstance(item, micropython.data.Function): 246 247 # Set the code location only where the code has been 248 # generated. 249 250 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 251 item.code_location = item.full_name() 252 253 # Skip any defaults for named functions. 254 255 elif item.name is not None: 256 item.code_location = pos + len(item.defaults) 257 258 # Skip any defaults for lambda functions. 259 260 else: 261 item.code_location = pos 262 263 elif isinstance(item, micropython.data.Const): 264 pos += len(item.raw_data()) 265 266 else: 267 pos += 1 268 269 # Generate the raw code. 270 271 self.raw_code = [] 272 273 for item in self.code: 274 275 if isinstance(item, micropython.data.Attr): 276 self.raw_code += item.as_raw(objtable, paramtable) 277 278 elif isinstance(item, Block): 279 assert item.location == len(self.raw_code) 280 self.raw_code += item.as_raw(objtable, paramtable) 281 282 elif isinstance(item, micropython.data.Class): 283 assert item.instance_template_location == len(self.raw_code) 284 self.raw_code += item.as_raw(objtable, paramtable, 285 with_builtins or item.module.name != "__builtins__" or item.astnode.doc is not None) 286 assert item.location == len(self.raw_code) - 1 287 288 elif isinstance(item, micropython.data.Const): 289 assert item.location == len(self.raw_code) 290 self.raw_code += item.as_raw(objtable, paramtable) 291 292 elif isinstance(item, micropython.data.Function): 293 assert item.location == len(self.raw_code) 294 self.raw_code += item.as_raw(objtable, paramtable) 295 296 # Check the code location only where the code has been generated. 297 298 assert (not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None) or \ 299 item.name is not None and item.code_location == len(self.raw_code) + len(item.defaults) or \ 300 item.name is None and item.code_location == len(self.raw_code) 301 302 elif isinstance(item, micropython.data.Module): 303 assert item.location == len(self.raw_code) 304 self.raw_code += item.as_raw(objtable, paramtable) 305 306 else: 307 self.raw_code.append(item) 308 309 # Set the code body location for items now that the code blocks have 310 # been positioned. 311 312 if isinstance(item, (micropython.data.Class, micropython.data.Function)): 313 if not with_builtins and item.module.name == "__builtins__" and item.astnode.doc is None: 314 item.code_body_location = item.full_name() 315 else: 316 item.code_body_location = item.get_body_block().location 317 318 # Fix the module locations. 319 320 for module in self.importer.modules_ordered: 321 322 if not with_builtins and module.name == "__builtins__": 323 continue 324 325 module.code_location = module.blocks[0].location 326 327 self.code_location = self.importer.modules["__main__"].code_location 328 return self.raw_code 329 330 def get_object_table(self): 331 332 "Return a table with details of attributes for classes and modules." 333 334 if self.objtable is None: 335 self.importer.vacuum() 336 337 t = self.objtable = micropython.table.ObjectTable() 338 for module in self.importer.get_modules(): 339 t.add(module.full_name(), module.module_attributes()) 340 341 # Add class and instance attributes for all classes, together 342 # with descendant information. 343 344 for obj in module.all_objects: 345 if isinstance(obj, micropython.inspect.Class): 346 attributes = {obj.full_name() : obj} 347 attributes.update(obj.all_attributes()) 348 attributes.update(obj.all_descendants()) 349 t.add(obj.full_name(), attributes) 350 351 return self.objtable 352 353 def get_parameter_table(self): 354 355 "Return a table with details of parameters for functions and methods." 356 357 # Need the object table to get at class details. 358 359 objtable = self.get_object_table() 360 361 if self.paramtable is None: 362 t = self.paramtable = micropython.table.ParameterTable() 363 364 # Visit each module, getting function and method details. 365 366 for module in self.importer.get_modules(): 367 for obj in module.all_objects: 368 if isinstance(obj, micropython.inspect.Function): 369 t.add(obj.full_name(), obj.parameters()) 370 371 # Classes are callable, too. 372 # Take details of the appropriate __init__ method to make an 373 # entry for an instantiation function for the class. 374 375 elif isinstance(obj, micropython.inspect.Class): 376 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 377 378 # Filter out all parameter table entries not referenced by keyword 379 # arguments. 380 381 keyword_names = set() 382 383 for module in self.importer.get_modules(): 384 keyword_names.update(module.keyword_names) 385 386 for function_name, parameters in t.table.items(): 387 for name in parameters.keys(): 388 if name in keyword_names: 389 break 390 else: 391 del t.table[function_name] 392 393 return self.paramtable 394 395 class Importer: 396 397 "An import machine, searching for and loading modules." 398 399 predefined_constants = { 400 "None" : None, 401 "True" : True, 402 "False" : False, 403 #"Ellipsis" : Ellipsis, 404 "NotImplemented" : NotImplemented 405 } 406 407 def __init__(self, path=None, verbose=0, optimisations=None): 408 409 """ 410 Initialise the importer with the given search 'path' - a list of 411 directories to search for Python modules. 412 413 The optional 'verbose' parameter causes output concerning the activities 414 of the object to be produced if set to a true value (not the default). 415 416 The optional 'optimisations' cause certain techniques to be used in 417 reducing program size and improving program efficiency. 418 """ 419 420 self.path = path or [os.getcwd()] 421 self.verbose = verbose 422 self.optimisations = optimisations or set() 423 424 self.modules = {} 425 self.modules_ordered = [] 426 self.loading = set() 427 428 # Constant records. 429 430 self.constant_values = {} 431 self.constant_list = None # cache for constants 432 self.init_predefined_constants() 433 434 # Name records (used to track actual use of names). 435 # Include names which may not be explicitly used in programs. 436 # NOTE: Potentially declare these when inspecting. 437 438 self.names_used = set(["__init__", "__call__", "__bool__"]) 439 440 # Status information. 441 442 self.vacuumed = 0 443 self.finalised = 0 444 445 def get_modules(self): 446 447 "Return all modules known to the importer." 448 449 return self.modules.values() 450 451 def get_module(self, name): 452 453 "Return the module with the given 'name'." 454 455 return self.modules[name] 456 457 # General maintenance. 458 459 def vacuum(self): 460 461 "Tidy up the modules." 462 463 if self.vacuumed: 464 return 465 466 for name, module in self.modules.items(): 467 if module.loaded: 468 module.vacuum() 469 else: 470 del self.modules[name] 471 472 self.vacuumed = 1 473 474 def finalise(self): 475 476 "Finalise the program." 477 478 if self.finalised: 479 return 480 481 self.vacuum() 482 483 for module in self.get_modules(): 484 module.finalise() 485 486 self.finalised = 1 487 488 # Name accounting. 489 490 def use_name(self, name): 491 492 "Register the given 'name' as being used in the program." 493 494 self.names_used.add(name) 495 496 # Constant accounting. 497 498 def init_predefined_constants(self): 499 500 "Ensure the predefined constants." 501 502 for name, value in self.predefined_constants.items(): 503 self.make_constant(value) 504 505 def get_predefined_constant(self, name): 506 507 "Return the predefined constant for the given 'name'." 508 509 return self.make_constant(self.predefined_constants[name]) 510 511 def get_constant(self, value): 512 513 "Return a constant for the given 'value'." 514 515 const = micropython.data.Const(value) 516 return self.constant_values[const] 517 518 def make_constant(self, value): 519 520 "Make and return a constant for the given 'value'." 521 522 # Ensure the presence of the constant's type. 523 524 name = value.__class__.__name__ 525 if self.modules.has_key("__builtins__"): 526 attr = self.modules["__builtins__"].get(name) 527 if attr is not None: 528 attr.set_referenced() 529 530 # Make a constant object and return it. 531 532 const = micropython.data.Const(value) 533 if not self.constant_values.has_key(const): 534 self.constant_values[const] = const 535 return self.constant_values[const] 536 537 def constants(self): 538 539 "Return a list of constants." 540 541 if self.constant_list is None: 542 self.constant_list = list(self.constant_values.values()) 543 544 return self.constant_list 545 546 # Import methods. 547 548 def find_in_path(self, name): 549 550 """ 551 Find the given module 'name' in the search path, returning None where no 552 such module could be found, or a 2-tuple from the 'find' method 553 otherwise. 554 """ 555 556 for d in self.path: 557 m = self.find(d, name) 558 if m: return m 559 return None 560 561 def find(self, d, name): 562 563 """ 564 In the directory 'd', find the given module 'name', where 'name' can 565 either refer to a single file module or to a package. Return None if the 566 'name' cannot be associated with either a file or a package directory, 567 or a 2-tuple from '_find_package' or '_find_module' otherwise. 568 """ 569 570 m = self._find_package(d, name) 571 if m: return m 572 m = self._find_module(d, name) 573 if m: return m 574 return None 575 576 def _find_module(self, d, name): 577 578 """ 579 In the directory 'd', find the given module 'name', returning None where 580 no suitable file exists in the directory, or a 2-tuple consisting of 581 None (indicating that no package directory is involved) and a filename 582 indicating the location of the module. 583 """ 584 585 name_py = name + os.extsep + "py" 586 filename = self._find_file(d, name_py) 587 if filename: 588 return None, filename 589 return None 590 591 def _find_package(self, d, name): 592 593 """ 594 In the directory 'd', find the given package 'name', returning None 595 where no suitable package directory exists, or a 2-tuple consisting of 596 a directory (indicating the location of the package directory itself) 597 and a filename indicating the location of the __init__.py module which 598 declares the package's top-level contents. 599 """ 600 601 filename = self._find_file(d, name) 602 if filename: 603 init_py = "__init__" + os.path.extsep + "py" 604 init_py_filename = self._find_file(filename, init_py) 605 if init_py_filename: 606 return filename, init_py_filename 607 return None 608 609 def _find_file(self, d, filename): 610 611 """ 612 Return the filename obtained when searching the directory 'd' for the 613 given 'filename', or None if no actual file exists for the filename. 614 """ 615 616 filename = os.path.join(d, filename) 617 if os.path.exists(filename): 618 return filename 619 else: 620 return None 621 622 def load(self, name, return_leaf=0): 623 624 """ 625 Load the module or package with the given 'name'. Return an object 626 referencing the loaded module or package, or None if no such module or 627 package exists. 628 """ 629 630 if self.modules.has_key(name) and self.modules[name].loaded: 631 #print "Cached (%s)" % name 632 return self.modules[name] 633 if self.verbose: 634 print "Loading", name 635 636 # Split the name into path components, and try to find the uppermost in 637 # the search path. 638 639 path = name.split(".") 640 m = self.find_in_path(path[0]) 641 if not m: 642 if self.verbose: 643 print "Not found (%s)" % path[0] 644 return None # NOTE: Import error. 645 d, filename = m 646 647 # Either acquire a reference to an already-imported module, or load the 648 # module from a file. 649 650 top = module = self.load_from_file(filename, path[0]) 651 652 # For hierarchical names, traverse each path component... 653 654 if len(path) > 1: 655 if not d: 656 if self.verbose: 657 print "No package (%s)" % filename 658 return None # NOTE: Import error (package not found). 659 else: 660 self.add_submodules(d, module) 661 662 path_so_far = path[:1] 663 for p in path[1:]: 664 path_so_far.append(p) 665 666 # Find the package or module concerned. 667 668 m = self.find(d, p) 669 if not m: 670 if self.verbose: 671 print "Not found (%s)" % p 672 return None # NOTE: Import error. 673 d, filename = m 674 module_name = ".".join(path_so_far) 675 676 # Either reference an imported module or load one from a file. 677 678 submodule = self.load_from_file(filename, module_name) 679 680 if d: 681 self.add_submodules(d, module) 682 683 # Store the submodule within its parent module. 684 685 module.set_module(p, submodule) 686 module = submodule 687 688 # Return either the deepest or the uppermost module. 689 690 if return_leaf: 691 return module 692 else: 693 return top 694 695 def load_from_file(self, name, module_name=None): 696 697 """ 698 Load the module with the given 'name' (which may be a full module path). 699 """ 700 701 if module_name is None: 702 module_name = "__main__" 703 704 module = self.add_module(module_name) 705 if not module.loaded and module not in self.loading: 706 self.loading.add(module) 707 #print "Parsing", name 708 module.parse(name) 709 #print "Done", name 710 self.loading.remove(module) 711 module.loaded = 1 712 713 # Record the module. 714 715 #print "Loaded", module_name, "with namespace", module.namespace.keys() 716 return module 717 718 def add_module(self, module_name): 719 720 """ 721 Return the module with the given 'module_name', adding a new module 722 object if one does not already exist. 723 """ 724 725 if not self.modules.has_key(module_name): 726 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 727 self.modules_ordered.append(module) 728 else: 729 module = self.modules[module_name] 730 return module 731 732 def add_submodules(self, pathname, module): 733 734 """ 735 Work around insufficient __all__ declarations and examine the directory 736 with the given 'pathname', adding submodules to the given 'module'. 737 """ 738 739 for filename in os.listdir(pathname): 740 submodule, ext = os.path.splitext(filename) 741 if ext not in ("", ".py"): 742 continue 743 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 744 745 # vim: tabstop=4 expandtab shiftwidth=4