1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class Program: 56 57 "This class supports the generation of a program image." 58 59 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 60 61 def __init__(self, importer, optimisations=None): 62 63 """ 64 Initialise the program representation with an 'importer' which is able 65 to locate and load Python modules. 66 67 The optional 'optimisations' cause certain techniques to be used in 68 reducing program size and improving program efficiency. 69 """ 70 71 self.importer = importer 72 self.optimisations = optimisations or set() 73 74 # Remember the tables once generated. 75 76 self.objtable = None 77 self.paramtable = None 78 79 # Main program information. 80 81 self.code = None 82 self.code_location = None 83 84 def get_importer(self): 85 return self.importer 86 87 # Access to finalised program information. 88 89 def get_image(self, with_builtins=0): 90 91 "Return a dictionary mapping modules to structures." 92 93 if self.code is not None: 94 return self.code 95 96 objtable = self.get_object_table() 97 paramtable = self.get_parameter_table() 98 self.importer.finalise() 99 100 self.code = [] 101 102 # Append constants to the image. 103 104 for pos, const in enumerate(self.importer.constants()): 105 self.code.append(const) 106 107 last_module = self.importer.modules_ordered[-1] 108 109 for module in self.importer.modules_ordered: 110 pos = len(self.code) 111 suppress_builtins = not with_builtins and module.name == "__builtins__" 112 113 # Position the module in the image and make a translation. 114 115 trans = micropython.ast.Translation(module, self) 116 117 # Add header details. 118 119 self.code.append(module) 120 pos += 1 121 122 # Append module attributes to the image. 123 124 attributes = module.module_attributes() 125 self.code += module.attributes_as_list() 126 pos += len(attributes.keys()) 127 128 # Append classes and functions to the image. 129 130 for obj in module.all_objects: 131 if isinstance(obj, micropython.inspect.Class): 132 133 # Add header details. 134 135 self.code.append(obj) 136 pos += 1 137 138 # Append class attributes to the image. 139 140 attributes = obj.class_attributes() 141 self.code += obj.attributes_as_list() 142 pos += len(attributes.keys()) 143 144 # Omit built-in function code where requested. 145 146 if suppress_builtins: 147 continue 148 149 # Generate the instantiator/initialiser. 150 # Append the function code to the image. 151 152 instantiator = obj.get_instantiator() 153 code = trans.get_instantiator_code(obj) 154 self.code += code 155 pos += len(code) 156 157 # Class-level code is generated separately at the module 158 # level, and the code location is set within the code 159 # generation process for the module. 160 161 elif isinstance(obj, micropython.inspect.Function): 162 163 # Add header details. 164 165 self.code.append(obj) 166 pos += 1 167 168 # Append any default values to the image. 169 # Only do this for named functions (not lambdas). 170 171 if obj.name is not None: 172 self.code += obj.default_attrs 173 pos += len(obj.default_attrs) 174 175 # Omit built-in function code where requested. 176 177 if suppress_builtins: 178 pass 179 180 # Append the function code to the image. 181 182 else: 183 code = trans.get_code(obj) 184 self.code += code 185 pos += len(code) 186 187 # Omit built-in module code where requested. 188 189 if suppress_builtins: 190 pass 191 192 # Append the module top-level code to the image. 193 194 else: 195 code = trans.get_module_code(final=(module is last_module)) 196 self.code += code 197 pos += len(code) 198 199 return self.code 200 201 def get_raw_image(self, with_builtins=0): 202 203 "Return the raw image representation of the program." 204 205 self.get_image(with_builtins) 206 207 objtable = self.get_object_table() 208 paramtable = self.get_parameter_table() 209 210 # Position the objects. 211 212 pos = 0 213 current_function = None 214 215 for item in self.code: 216 217 # Blocks are positioned leaving space for their expansion. 218 219 if isinstance(item, Block): 220 item.location = pos 221 pos += len(item.code) 222 223 # Set code body information on functions, assuming that the 224 # first block is for argument checks. 225 226 if current_function is not None: 227 current_function.code_body_location = pos 228 229 current_function = None 230 231 # Other multi-location objects. 232 233 elif isinstance(item, ( 234 micropython.data.Class, micropython.data.Const, 235 micropython.data.Function, micropython.data.Module 236 )): 237 238 if isinstance(item, micropython.data.Class): 239 240 # Append a template of an instance for use when 241 # instantiating classes. 242 243 item.instance_template_location = pos 244 pos += 1 245 246 # Record the location of the object. 247 248 item.location = pos 249 pos += 1 250 251 # Code and details are associated with certain objects. 252 253 if isinstance(item, micropython.data.Function): 254 255 # Set the code location only where the code has been 256 # generated. 257 258 if not with_builtins and item.module.name == "__builtins__": 259 item.code_location = item.full_name() 260 261 # Skip any defaults. 262 263 else: 264 item.code_location = pos + len(item.defaults) 265 266 current_function = item 267 268 elif isinstance(item, micropython.data.Const): 269 pos += len(item.raw_data()) 270 current_function = None 271 272 else: 273 current_function = None 274 275 else: 276 pos += 1 277 278 # Generate the raw code. 279 280 self.raw_code = [] 281 282 for item in self.code: 283 284 if isinstance(item, micropython.data.Attr): 285 self.raw_code += item.as_raw(objtable, paramtable) 286 287 elif isinstance(item, Block): 288 assert item.location == len(self.raw_code) 289 self.raw_code += item.as_raw(objtable, paramtable) 290 291 # Using classcode, attrcode, codeaddr, codedetails, instance. 292 293 elif isinstance(item, micropython.data.Class): 294 assert item.instance_template_location == len(self.raw_code) 295 self.raw_code += item.as_raw(objtable, paramtable, with_builtins or item.module.name != "__builtins__") 296 assert item.location == len(self.raw_code) - 1 297 298 elif isinstance(item, micropython.data.Const): 299 assert item.location == len(self.raw_code) 300 self.raw_code += item.as_raw(objtable, paramtable) 301 302 elif isinstance(item, micropython.data.Function): 303 assert item.location == len(self.raw_code) 304 self.raw_code += item.as_raw(objtable, paramtable) 305 306 # Check the code location only where the code has been generated. 307 308 assert (not with_builtins and item.module.name == "__builtins__") or \ 309 item.code_location == len(self.raw_code) + len(item.defaults) 310 311 elif isinstance(item, micropython.data.Module): 312 assert item.location == len(self.raw_code) 313 self.raw_code += item.as_raw(objtable, paramtable) 314 315 else: 316 self.raw_code.append(item) 317 318 # Fix the module locations. 319 320 for module in self.importer.modules_ordered: 321 322 if not with_builtins and module.name == "__builtins__": 323 continue 324 325 module.code_location = module.blocks[0].location 326 327 self.code_location = self.importer.modules["__main__"].code_location 328 return self.raw_code 329 330 def get_object_table(self): 331 332 "Return a table with details of attributes for classes and modules." 333 334 if self.objtable is None: 335 self.importer.vacuum() 336 337 t = self.objtable = micropython.table.ObjectTable() 338 for module in self.importer.get_modules(): 339 t.add(module.full_name(), module.module_attributes()) 340 341 # Add class and instance attributes for all classes, together 342 # with descendant information. 343 344 for obj in module.all_objects: 345 if isinstance(obj, micropython.inspect.Class): 346 attributes = {obj.full_name() : obj} 347 attributes.update(obj.all_attributes()) 348 attributes.update(obj.all_descendants()) 349 t.add(obj.full_name(), attributes) 350 351 return self.objtable 352 353 def get_parameter_table(self): 354 355 "Return a table with details of parameters for functions and methods." 356 357 # Need the object table to get at class details. 358 359 objtable = self.get_object_table() 360 361 if self.paramtable is None: 362 t = self.paramtable = micropython.table.ParameterTable() 363 364 # Visit each module, getting function and method details. 365 366 for module in self.importer.get_modules(): 367 for obj in module.all_objects: 368 if isinstance(obj, micropython.inspect.Function): 369 t.add(obj.full_name(), obj.parameters()) 370 371 # Classes are callable, too. 372 # Take details of the appropriate __init__ method to make an 373 # entry for an instantiation function for the class. 374 375 elif isinstance(obj, micropython.inspect.Class): 376 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 377 378 # Filter out all parameter table entries not referenced by keyword 379 # arguments. 380 381 keyword_names = set() 382 383 for module in self.importer.get_modules(): 384 keyword_names.update(module.keyword_names) 385 386 for function_name, parameters in t.table.items(): 387 for name in parameters.keys(): 388 if name in keyword_names: 389 break 390 else: 391 del t.table[function_name] 392 393 return self.paramtable 394 395 class Importer: 396 397 "An import machine, searching for and loading modules." 398 399 predefined_constants = { 400 "None" : None, 401 "True" : True, 402 "False" : False, 403 #"Ellipsis" : Ellipsis, 404 "NotImplemented" : NotImplemented 405 } 406 407 def __init__(self, path=None, verbose=0, optimisations=None): 408 409 """ 410 Initialise the importer with the given search 'path' - a list of 411 directories to search for Python modules. 412 413 The optional 'verbose' parameter causes output concerning the activities 414 of the object to be produced if set to a true value (not the default). 415 416 The optional 'optimisations' cause certain techniques to be used in 417 reducing program size and improving program efficiency. 418 """ 419 420 self.path = path or [os.getcwd()] 421 self.verbose = verbose 422 self.optimisations = optimisations or set() 423 424 self.modules = {} 425 self.modules_ordered = [] 426 self.loading = set() 427 428 # Constant records. 429 430 self.constant_values = {} 431 self.constant_list = None # cache for constants 432 self.init_predefined_constants() 433 434 # Name records (used to track actual use of names). 435 # Include names which may not be explicitly used in programs. 436 # NOTE: Potentially declare these when inspecting. 437 438 self.names_used = set(["__init__", "__call__", "__bool__"]) 439 440 # Status information. 441 442 self.vacuumed = 0 443 self.finalised = 0 444 445 def get_modules(self): 446 447 "Return all modules known to the importer." 448 449 return self.modules.values() 450 451 # General maintenance. 452 453 def vacuum(self): 454 455 "Tidy up the modules." 456 457 if self.vacuumed: 458 return 459 460 for name, module in self.modules.items(): 461 if module.loaded: 462 module.vacuum() 463 else: 464 del self.modules[name] 465 466 self.vacuumed = 1 467 468 def finalise(self): 469 470 "Finalise the program." 471 472 if self.finalised: 473 return 474 475 self.vacuum() 476 477 for module in self.get_modules(): 478 module.finalise() 479 480 self.finalised = 1 481 482 # Name accounting. 483 484 def use_name(self, name): 485 486 "Register the given 'name' as being used in the program." 487 488 self.names_used.add(name) 489 490 # Constant accounting. 491 492 def init_predefined_constants(self): 493 494 "Ensure the predefined constants." 495 496 for name, value in self.predefined_constants.items(): 497 self.make_constant(value) 498 499 def get_predefined_constant(self, name): 500 501 "Return the predefined constant for the given 'name'." 502 503 return self.make_constant(self.predefined_constants[name]) 504 505 def get_constant(self, value): 506 507 "Return a constant for the given 'value'." 508 509 const = micropython.data.Const(value) 510 return self.constant_values[const] 511 512 def make_constant(self, value): 513 514 "Make and return a constant for the given 'value'." 515 516 # Ensure the presence of the constant's type. 517 518 name = value.__class__.__name__ 519 if self.modules.has_key("__builtins__"): 520 attr = self.modules["__builtins__"].get(name) 521 if attr is not None: 522 attr.set_referenced() 523 524 # Make a constant object and return it. 525 526 const = micropython.data.Const(value) 527 if not self.constant_values.has_key(const): 528 self.constant_values[const] = const 529 return self.constant_values[const] 530 531 def constants(self): 532 533 "Return a list of constants." 534 535 if self.constant_list is None: 536 self.constant_list = list(self.constant_values.values()) 537 538 return self.constant_list 539 540 # Import methods. 541 542 def find_in_path(self, name): 543 544 """ 545 Find the given module 'name' in the search path, returning None where no 546 such module could be found, or a 2-tuple from the 'find' method 547 otherwise. 548 """ 549 550 for d in self.path: 551 m = self.find(d, name) 552 if m: return m 553 return None 554 555 def find(self, d, name): 556 557 """ 558 In the directory 'd', find the given module 'name', where 'name' can 559 either refer to a single file module or to a package. Return None if the 560 'name' cannot be associated with either a file or a package directory, 561 or a 2-tuple from '_find_package' or '_find_module' otherwise. 562 """ 563 564 m = self._find_package(d, name) 565 if m: return m 566 m = self._find_module(d, name) 567 if m: return m 568 return None 569 570 def _find_module(self, d, name): 571 572 """ 573 In the directory 'd', find the given module 'name', returning None where 574 no suitable file exists in the directory, or a 2-tuple consisting of 575 None (indicating that no package directory is involved) and a filename 576 indicating the location of the module. 577 """ 578 579 name_py = name + os.extsep + "py" 580 filename = self._find_file(d, name_py) 581 if filename: 582 return None, filename 583 return None 584 585 def _find_package(self, d, name): 586 587 """ 588 In the directory 'd', find the given package 'name', returning None 589 where no suitable package directory exists, or a 2-tuple consisting of 590 a directory (indicating the location of the package directory itself) 591 and a filename indicating the location of the __init__.py module which 592 declares the package's top-level contents. 593 """ 594 595 filename = self._find_file(d, name) 596 if filename: 597 init_py = "__init__" + os.path.extsep + "py" 598 init_py_filename = self._find_file(filename, init_py) 599 if init_py_filename: 600 return filename, init_py_filename 601 return None 602 603 def _find_file(self, d, filename): 604 605 """ 606 Return the filename obtained when searching the directory 'd' for the 607 given 'filename', or None if no actual file exists for the filename. 608 """ 609 610 filename = os.path.join(d, filename) 611 if os.path.exists(filename): 612 return filename 613 else: 614 return None 615 616 def load(self, name, return_leaf=0): 617 618 """ 619 Load the module or package with the given 'name'. Return an object 620 referencing the loaded module or package, or None if no such module or 621 package exists. 622 """ 623 624 if self.modules.has_key(name) and self.modules[name].loaded: 625 #print "Cached (%s)" % name 626 return self.modules[name] 627 if self.verbose: 628 print "Loading", name 629 630 # Split the name into path components, and try to find the uppermost in 631 # the search path. 632 633 path = name.split(".") 634 m = self.find_in_path(path[0]) 635 if not m: 636 if self.verbose: 637 print "Not found (%s)" % path[0] 638 return None # NOTE: Import error. 639 d, filename = m 640 641 # Either acquire a reference to an already-imported module, or load the 642 # module from a file. 643 644 top = module = self.load_from_file(filename, path[0]) 645 646 # For hierarchical names, traverse each path component... 647 648 if len(path) > 1: 649 if not d: 650 if self.verbose: 651 print "No package (%s)" % filename 652 return None # NOTE: Import error (package not found). 653 else: 654 self.add_submodules(d, module) 655 656 path_so_far = path[:1] 657 for p in path[1:]: 658 path_so_far.append(p) 659 660 # Find the package or module concerned. 661 662 m = self.find(d, p) 663 if not m: 664 if self.verbose: 665 print "Not found (%s)" % p 666 return None # NOTE: Import error. 667 d, filename = m 668 module_name = ".".join(path_so_far) 669 670 # Either reference an imported module or load one from a file. 671 672 submodule = self.load_from_file(filename, module_name) 673 674 if d: 675 self.add_submodules(d, module) 676 677 # Store the submodule within its parent module. 678 679 module.set_module(p, submodule) 680 module = submodule 681 682 # Return either the deepest or the uppermost module. 683 684 if return_leaf: 685 return module 686 else: 687 return top 688 689 def load_from_file(self, name, module_name=None): 690 691 """ 692 Load the module with the given 'name' (which may be a full module path). 693 """ 694 695 if module_name is None: 696 module_name = "__main__" 697 698 module = self.add_module(module_name) 699 if not module.loaded and module not in self.loading: 700 self.loading.add(module) 701 #print "Parsing", name 702 module.parse(name) 703 #print "Done", name 704 self.loading.remove(module) 705 module.loaded = 1 706 707 # Record the module. 708 709 #print "Loaded", module_name, "with namespace", module.namespace.keys() 710 return module 711 712 def add_module(self, module_name): 713 714 """ 715 Return the module with the given 'module_name', adding a new module 716 object if one does not already exist. 717 """ 718 719 if not self.modules.has_key(module_name): 720 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 721 self.modules_ordered.append(module) 722 else: 723 module = self.modules[module_name] 724 return module 725 726 def add_submodules(self, pathname, module): 727 728 """ 729 Work around insufficient __all__ declarations and examine the directory 730 with the given 'pathname', adding submodules to the given 'module'. 731 """ 732 733 for filename in os.listdir(pathname): 734 submodule, ext = os.path.splitext(filename) 735 if ext not in ("", ".py"): 736 continue 737 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 738 739 # vim: tabstop=4 expandtab shiftwidth=4