1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.data 39 import micropython.opt 40 import micropython.inspect 41 import micropython.table 42 import os 43 try: 44 set 45 except NameError: 46 from sets import Set as set 47 48 class Program: 49 50 "This class supports the generation of a program image." 51 52 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 53 54 def __init__(self, importer, optimisations=None): 55 56 """ 57 Initialise the program representation with an 'importer' which is able 58 to locate and load Python modules. 59 60 The optional 'optimisations' cause certain techniques to be used in 61 reducing program size and improving program efficiency. 62 """ 63 64 self.importer = importer 65 self.optimisations = optimisations or set() 66 67 # Remember the tables once generated. 68 69 self.objtable = None 70 self.paramtable = None 71 self.clstable = None 72 73 # Main program information. 74 75 self.code = None 76 self.code_location = None 77 78 def get_importer(self): 79 return self.importer 80 81 # Access to finalised program information. 82 83 def get_image(self, with_builtins=0): 84 85 "Return a dictionary mapping modules to structures." 86 87 if self.code is not None: 88 return self.code 89 90 objtable = self.get_object_table() 91 paramtable = self.get_parameter_table() 92 self.importer.finalise() 93 94 self.code = [] 95 96 # Append constants to the image. 97 98 for pos, const in enumerate(self.importer.constants()): 99 self.code.append(const) 100 101 last_module = self.importer.modules_ordered[-1] 102 103 for module in self.importer.modules_ordered: 104 105 if not with_builtins and module.name == "__builtins__": 106 continue 107 108 pos = len(self.code) 109 110 # Position the module in the image and make a translation. 111 112 trans = micropython.ast.Translation(module, self) 113 114 # Add header details. 115 116 self.code.append(module) 117 pos += 1 118 119 # Append module attributes to the image. 120 121 attributes = module.module_attributes() 122 self.code += module.attributes_as_list() 123 pos += len(attributes.keys()) 124 125 # Append classes and functions to the image. 126 127 for obj in module.all_objects: 128 if isinstance(obj, micropython.inspect.Class): 129 130 # Add header details. 131 132 self.code.append(obj) 133 pos += 1 134 135 # Append class attributes to the image. 136 137 attributes = obj.class_attributes() 138 self.code += obj.attributes_as_list() 139 pos += len(attributes.keys()) 140 141 # Generate the instantiator/initialiser. 142 # Append the function code to the image. 143 144 instantiator = obj.get_instantiator() 145 code = trans.get_instantiator_code(obj) 146 self.code += code 147 pos += len(code) 148 149 # Class-level code is generated separately at the module 150 # level, and the code location is set within the code 151 # generation process for the module. 152 153 elif isinstance(obj, micropython.inspect.Function): 154 155 # Add header details. 156 157 self.code.append(obj) 158 pos += 1 159 160 # Append any default values to the image. 161 # Only do this for named functions (not lambdas). 162 163 if obj.name is not None: 164 self.code += obj.default_attrs 165 pos += len(obj.default_attrs) 166 167 # Append the function code to the image. 168 169 code = trans.get_code(obj) 170 self.code += code 171 pos += len(code) 172 173 # Append the module top-level code to the image. 174 175 code = trans.get_module_code(final=(module is last_module)) 176 self.code += code 177 pos += len(code) 178 179 return self.code 180 181 def get_raw_image(self, with_builtins=0): 182 183 "Return the raw image representation of the program." 184 185 self.get_image(with_builtins) 186 187 objtable = self.get_object_table() 188 paramtable = self.get_parameter_table() 189 190 # Position the objects. 191 192 pos = 0 193 for item in self.code: 194 if isinstance(item, Block): 195 item.location = pos 196 pos += len(item.code) 197 elif isinstance(item, (micropython.data.Class, micropython.data.Const, 198 micropython.data.Function, micropython.data.Module)): 199 item.location = pos 200 pos += 1 201 if isinstance(item, micropython.data.Function): 202 item.code_location = pos 203 elif isinstance(item, micropython.data.Const): 204 pos += len(self.raw_data(item)) 205 else: 206 pos += 1 207 208 # Generate the raw code. 209 210 self.raw_code = [] 211 212 for item in self.code: 213 214 if isinstance(item, micropython.data.Attr): 215 self.raw_code.append(( 216 item.context and item.context.location, 217 item.value and item.value.location # no useful context is provided 218 )) 219 220 elif isinstance(item, Block): 221 self.raw_code += self.raw_block(item) 222 223 # Using classcode, attrcode, codeaddr, codedetails, instance. 224 225 elif isinstance(item, micropython.data.Class): 226 assert item.location == len(self.raw_code) 227 228 # NOTE: The instantiator code is the first block of the class. 229 230 instantiator_code_location = item.get_instantiator().blocks[0].location 231 232 # NOTE: Need initialiser details! 233 self.raw_code.append(( 234 objtable.as_list().get_code(item.full_name()), 235 objtable.get_index(item.full_name()), 236 instantiator_code_location, 237 ( 238 len(item.get_instantiator().positional_names), 239 len(item.get_instantiator().defaults) 240 ), 241 0 242 )) 243 244 elif isinstance(item, micropython.data.Const): 245 assert item.location == len(self.raw_code) 246 247 # NOTE: Need class details! 248 self.raw_code.append(( 249 objtable.as_list().get_code(item.value_type_name()), 250 objtable.get_index(item.value_type_name()), 251 None, 252 None, 253 1 254 )) 255 256 self.raw_code += self.raw_data(item) 257 258 elif isinstance(item, micropython.data.Function): 259 assert item.location == len(self.raw_code) 260 261 # NOTE: Need class and parameter details! Should arguably be types.FunctionType. 262 self.raw_code.append(( 263 objtable.as_list().get_code("__builtins__.function"), 264 objtable.get_index("__builtins__.function"), 265 item.code_location, 266 ( 267 len(item.positional_names), 268 len(item.defaults) 269 ), 270 0 271 )) 272 273 assert item.code_location == len(self.raw_code) 274 275 elif isinstance(item, micropython.data.Module): 276 assert item.location == len(self.raw_code) 277 278 self.raw_code.append(( 279 objtable.as_list().get_code(item.full_name()), 280 None, # module name not used as an attribute 281 None, 282 None, 283 0 284 )) 285 286 else: 287 self.raw_code.append(item) 288 289 # Fix the module locations. 290 291 for module in self.importer.modules_ordered: 292 293 if not with_builtins and module.name == "__builtins__": 294 continue 295 296 module.code_location = module.blocks[0].location 297 298 self.code_location = self.importer.modules["__main__"].code_location 299 return self.raw_code 300 301 def raw_block(self, block): 302 303 "Return the code for the given 'block'." 304 305 assert block.location == len(self.raw_code) 306 for i, item in enumerate(block.code): 307 if hasattr(item, "location"): 308 item.location = location + i 309 return block.code 310 311 def raw_data(self, item): 312 313 "Return the data for the given 'item'." 314 315 datatype = item.value_type_name() 316 317 # NOTE: Start simple and use single entries for most types. 318 319 if datatype in ("__builtins__.tuple", "__builtins__.list"): 320 return [len(item.value)] + list(item.value) 321 else: 322 return [item.value] 323 324 def get_object_table(self): 325 326 "Return a table with details of attributes for classes and modules." 327 328 if self.objtable is None: 329 self.importer.vacuum() 330 331 t = self.objtable = micropython.table.ObjectTable() 332 for module in self.importer.get_modules(): 333 t.add(module.full_name(), module.module_attributes()) 334 335 # Add class and instance attributes for all classes, together 336 # with descendant information. 337 338 for obj in module.all_objects: 339 if isinstance(obj, micropython.inspect.Class): 340 attributes = {obj.full_name() : obj} 341 attributes.update(obj.all_attributes()) 342 attributes.update(obj.all_descendants()) 343 t.add(obj.full_name(), attributes) 344 345 return self.objtable 346 347 def get_parameter_table(self): 348 349 "Return a table with details of parameters for functions and methods." 350 351 # Need the object table to get at class details. 352 353 objtable = self.get_object_table() 354 355 if self.paramtable is None: 356 t = self.paramtable = micropython.table.ParameterTable() 357 358 # Visit each module, getting function and method details. 359 360 for module in self.importer.get_modules(): 361 for obj in module.all_objects: 362 if isinstance(obj, micropython.inspect.Function): 363 t.add(obj.full_name(), obj.parameters()) 364 365 # Classes are callable, too. 366 # Take details of the appropriate __init__ method to make an 367 # entry for an instantiation function for the class. 368 369 elif isinstance(obj, micropython.inspect.Class): 370 t.add(obj.full_name(), obj.get_instantiator().parameters()) 371 372 # Filter out all parameter table entries not referenced by keyword 373 # arguments. 374 375 keyword_names = set() 376 377 for module in self.importer.get_modules(): 378 keyword_names.update(module.keyword_names) 379 380 for function_name, parameters in t.table.items(): 381 for name in parameters.keys(): 382 if name in keyword_names: 383 break 384 else: 385 del t.table[function_name] 386 387 return self.paramtable 388 389 class Importer: 390 391 "An import machine, searching for and loading modules." 392 393 predefined_constants = { 394 "None" : None, 395 "True" : True, 396 "False" : False, 397 "Ellipsis" : Ellipsis, 398 "NotImplemented" : NotImplemented 399 } 400 401 def __init__(self, path=None, verbose=0, optimisations=None): 402 403 """ 404 Initialise the importer with the given search 'path' - a list of 405 directories to search for Python modules. 406 407 The optional 'verbose' parameter causes output concerning the activities 408 of the object to be produced if set to a true value (not the default). 409 410 The optional 'optimisations' cause certain techniques to be used in 411 reducing program size and improving program efficiency. 412 """ 413 414 self.path = path or [os.getcwd()] 415 self.verbose = verbose 416 self.optimisations = optimisations or set() 417 418 self.modules = {} 419 self.modules_ordered = [] 420 self.loading = set() 421 422 # Constant records. 423 424 self.constant_values = {} 425 self.constant_list = None # cache for constants 426 self.init_predefined_constants() 427 428 # Name records (used to track actual use of names). 429 # Include names which may not be explicitly used in programs. 430 431 self.names_used = set(["__init__"]) 432 433 # Status information. 434 435 self.vacuumed = 0 436 self.finalised = 0 437 438 def get_modules(self): 439 440 "Return all modules known to the importer." 441 442 return self.modules.values() 443 444 # General maintenance. 445 446 def vacuum(self): 447 448 "Tidy up the modules." 449 450 if self.vacuumed: 451 return 452 453 for name, module in self.modules.items(): 454 if module.loaded: 455 module.vacuum() 456 else: 457 del self.modules[name] 458 459 self.vacuumed = 1 460 461 def finalise(self): 462 463 "Finalise the program." 464 465 if self.finalised: 466 return 467 468 self.vacuum() 469 470 for module in self.get_modules(): 471 module.finalise() 472 473 self.finalised = 1 474 475 # Name accounting. 476 477 def use_name(self, name): 478 479 "Register the given 'name' as being used in the program." 480 481 self.names_used.add(name) 482 483 # Constant accounting. 484 485 def init_predefined_constants(self): 486 487 "Ensure the predefined constants." 488 489 for name, value in self.predefined_constants.items(): 490 self.make_constant(value) 491 492 def get_predefined_constant(self, name): 493 494 "Return the predefined constant for the given 'name'." 495 496 return self.make_constant(self.predefined_constants[name]) 497 498 def get_constant(self, value): 499 500 "Return a constant for the given 'value'." 501 502 const = micropython.data.Const(value) 503 return self.constant_values[const] 504 505 def make_constant(self, value): 506 507 "Make and return a constant for the given 'value'." 508 509 const = micropython.data.Const(value) 510 if not self.constant_values.has_key(const): 511 self.constant_values[const] = const 512 return self.constant_values[const] 513 514 def constants(self): 515 516 "Return a list of constants." 517 518 if self.constant_list is None: 519 self.constant_list = list(self.constant_values.values()) 520 521 return self.constant_list 522 523 # Import methods. 524 525 def find_in_path(self, name): 526 527 """ 528 Find the given module 'name' in the search path, returning None where no 529 such module could be found, or a 2-tuple from the 'find' method 530 otherwise. 531 """ 532 533 for d in self.path: 534 m = self.find(d, name) 535 if m: return m 536 return None 537 538 def find(self, d, name): 539 540 """ 541 In the directory 'd', find the given module 'name', where 'name' can 542 either refer to a single file module or to a package. Return None if the 543 'name' cannot be associated with either a file or a package directory, 544 or a 2-tuple from '_find_package' or '_find_module' otherwise. 545 """ 546 547 m = self._find_package(d, name) 548 if m: return m 549 m = self._find_module(d, name) 550 if m: return m 551 return None 552 553 def _find_module(self, d, name): 554 555 """ 556 In the directory 'd', find the given module 'name', returning None where 557 no suitable file exists in the directory, or a 2-tuple consisting of 558 None (indicating that no package directory is involved) and a filename 559 indicating the location of the module. 560 """ 561 562 name_py = name + os.extsep + "py" 563 filename = self._find_file(d, name_py) 564 if filename: 565 return None, filename 566 return None 567 568 def _find_package(self, d, name): 569 570 """ 571 In the directory 'd', find the given package 'name', returning None 572 where no suitable package directory exists, or a 2-tuple consisting of 573 a directory (indicating the location of the package directory itself) 574 and a filename indicating the location of the __init__.py module which 575 declares the package's top-level contents. 576 """ 577 578 filename = self._find_file(d, name) 579 if filename: 580 init_py = "__init__" + os.path.extsep + "py" 581 init_py_filename = self._find_file(filename, init_py) 582 if init_py_filename: 583 return filename, init_py_filename 584 return None 585 586 def _find_file(self, d, filename): 587 588 """ 589 Return the filename obtained when searching the directory 'd' for the 590 given 'filename', or None if no actual file exists for the filename. 591 """ 592 593 filename = os.path.join(d, filename) 594 if os.path.exists(filename): 595 return filename 596 else: 597 return None 598 599 def load(self, name, return_leaf=0): 600 601 """ 602 Load the module or package with the given 'name'. Return an object 603 referencing the loaded module or package, or None if no such module or 604 package exists. 605 """ 606 607 if self.modules.has_key(name) and self.modules[name].loaded: 608 #print "Cached (%s)" % name 609 return self.modules[name] 610 if self.verbose: 611 print "Loading", name 612 613 # Split the name into path components, and try to find the uppermost in 614 # the search path. 615 616 path = name.split(".") 617 m = self.find_in_path(path[0]) 618 if not m: 619 if self.verbose: 620 print "Not found (%s)" % path[0] 621 return None # NOTE: Import error. 622 d, filename = m 623 624 # Either acquire a reference to an already-imported module, or load the 625 # module from a file. 626 627 top = module = self.load_from_file(filename, path[0]) 628 629 # For hierarchical names, traverse each path component... 630 631 if len(path) > 1: 632 if not d: 633 if self.verbose: 634 print "No package (%s)" % filename 635 return None # NOTE: Import error (package not found). 636 else: 637 self.add_submodules(d, module) 638 639 path_so_far = path[:1] 640 for p in path[1:]: 641 path_so_far.append(p) 642 643 # Find the package or module concerned. 644 645 m = self.find(d, p) 646 if not m: 647 if self.verbose: 648 print "Not found (%s)" % p 649 return None # NOTE: Import error. 650 d, filename = m 651 module_name = ".".join(path_so_far) 652 653 # Either reference an imported module or load one from a file. 654 655 submodule = self.load_from_file(filename, module_name) 656 657 if d: 658 self.add_submodules(d, module) 659 660 # Store the submodule within its parent module. 661 662 module.set_module(p, submodule) 663 module = submodule 664 665 # Return either the deepest or the uppermost module. 666 667 if return_leaf: 668 return module 669 else: 670 return top 671 672 def load_from_file(self, name, module_name=None): 673 674 """ 675 Load the module with the given 'name' (which may be a full module path). 676 """ 677 678 if module_name is None: 679 module_name = "__main__" 680 681 module = self.add_module(module_name) 682 if not module.loaded and module not in self.loading: 683 self.loading.add(module) 684 #print "Parsing", name 685 module.parse(name) 686 #print "Done", name 687 self.loading.remove(module) 688 module.loaded = 1 689 690 # Record the module. 691 692 #print "Loaded", module_name, "with namespace", module.namespace.keys() 693 return module 694 695 def add_module(self, module_name): 696 697 """ 698 Return the module with the given 'module_name', adding a new module 699 object if one does not already exist. 700 """ 701 702 if not self.modules.has_key(module_name): 703 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 704 self.modules_ordered.append(module) 705 else: 706 module = self.modules[module_name] 707 return module 708 709 def add_submodules(self, pathname, module): 710 711 """ 712 Work around insufficient __all__ declarations and examine the directory 713 with the given 'pathname', adding submodules to the given 'module'. 714 """ 715 716 for filename in os.listdir(pathname): 717 submodule, ext = os.path.splitext(filename) 718 if ext not in ("", ".py"): 719 continue 720 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 721 722 # vim: tabstop=4 expandtab shiftwidth=4