1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.data 39 import micropython.opt 40 import micropython.inspect 41 import micropython.table 42 import os 43 try: 44 set 45 except NameError: 46 from sets import Set as set 47 48 class Program: 49 50 "This class supports the generation of a program image." 51 52 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 53 54 def __init__(self, importer, optimisations=None): 55 56 """ 57 Initialise the program representation with an 'importer' which is able 58 to locate and load Python modules. 59 60 The optional 'optimisations' cause certain techniques to be used in 61 reducing program size and improving program efficiency. 62 """ 63 64 self.importer = importer 65 self.optimisations = optimisations or set() 66 67 # Remember the tables once generated. 68 69 self.objtable = None 70 self.paramtable = None 71 self.clstable = None 72 73 # Main program information. 74 75 self.code = None 76 self.code_location = None 77 78 def get_importer(self): 79 return self.importer 80 81 # Access to finalised program information. 82 83 def get_image(self, with_builtins=0): 84 85 "Return a dictionary mapping modules to structures." 86 87 if self.code is not None: 88 return self.code 89 90 objtable = self.get_object_table() 91 paramtable = self.get_parameter_table() 92 self.importer.finalise() 93 94 self.code = [] 95 96 # Append constants to the image. 97 98 for pos, const in enumerate(self.importer.constants()): 99 self.code.append(const) 100 101 last_module = self.importer.modules_ordered[-1] 102 103 for module in self.importer.modules_ordered: 104 105 if not with_builtins and module.name == "__builtins__": 106 continue 107 108 pos = len(self.code) 109 110 # Position the module in the image and make a translation. 111 112 trans = micropython.ast.Translation(module, self) 113 114 # Add header details. 115 116 self.code.append(module) 117 pos += 1 118 119 # Append module attributes to the image. 120 121 attributes = module.module_attributes() 122 self.code += module.attributes_as_list() 123 pos += len(attributes.keys()) 124 125 # Append classes and functions to the image. 126 127 for obj in module.all_objects: 128 if isinstance(obj, micropython.inspect.Class): 129 130 # Add header details. 131 132 self.code.append(obj) 133 pos += 1 134 135 # Append class attributes to the image. 136 137 attributes = obj.class_attributes() 138 self.code += obj.attributes_as_list() 139 pos += len(attributes.keys()) 140 141 # Generate the instantiator/initialiser. 142 # Append the function code to the image. 143 144 instantiator = obj.get_instantiator() 145 code = trans.get_instantiator_code(obj) 146 self.code += code 147 pos += len(code) 148 149 # Class-level code is generated separately at the module 150 # level, and the code location is set within the code 151 # generation process for the module. 152 153 elif isinstance(obj, micropython.inspect.Function): 154 155 # Add header details. 156 157 self.code.append(obj) 158 pos += 1 159 160 # Append any default values to the image. 161 # Only do this for named functions (not lambdas). 162 163 if obj.name is not None: 164 self.code += obj.default_attrs 165 pos += len(obj.default_attrs) 166 167 # Append the function code to the image. 168 169 code = trans.get_code(obj) 170 self.code += code 171 pos += len(code) 172 173 # Append the module top-level code to the image. 174 175 code = trans.get_module_code(final=(module is last_module)) 176 self.code += code 177 pos += len(code) 178 179 return self.code 180 181 def get_raw_image(self, with_builtins=0): 182 183 "Return the raw image representation of the program." 184 185 self.get_image(with_builtins) 186 187 objtable = self.get_object_table() 188 paramtable = self.get_parameter_table() 189 190 # Position the objects. 191 192 pos = 0 193 for item in self.code: 194 if isinstance(item, Block): 195 item.location = pos 196 pos += len(item.code) 197 elif isinstance(item, (micropython.data.Class, micropython.data.Const, 198 micropython.data.Function, micropython.data.Module)): 199 item.location = pos 200 pos += 1 201 if isinstance(item, micropython.data.Function): 202 item.code_location = pos 203 elif isinstance(item, micropython.data.Const): 204 pos += len(self.raw_data(item)) 205 else: 206 pos += 1 207 208 # Generate the raw code. 209 210 self.raw_code = [] 211 212 for item in self.code: 213 214 if isinstance(item, micropython.data.Attr): 215 self.raw_code.append(( 216 item.context and item.context.location, 217 item.value and item.value.location # no useful context is provided 218 )) 219 220 elif isinstance(item, Block): 221 self.raw_code += self.raw_block(item) 222 223 # Using classcode, attrcode, codeaddr, codedetails, instance. 224 225 elif isinstance(item, micropython.data.Class): 226 assert item.location == len(self.raw_code) 227 228 # NOTE: The instantiator code is the first block of the class. 229 230 instantiator_code_location = item.get_instantiator().blocks[0].location 231 232 # NOTE: Need initialiser details! 233 self.raw_code.append(( 234 objtable.as_list().get_code(item.full_name()), 235 objtable.get_index(item.full_name()), 236 instantiator_code_location, 237 ( 238 len(item.get_instantiator().positional_names), 239 len(item.get_instantiator().defaults) 240 ), 241 0 242 )) 243 244 elif isinstance(item, micropython.data.Const): 245 assert item.location == len(self.raw_code) 246 247 # NOTE: Need class details! 248 self.raw_code.append(( 249 objtable.as_list().get_code(item.value_type_name()), 250 objtable.get_index(item.value_type_name()), 251 None, 252 None, 253 1 254 )) 255 256 self.raw_code += self.raw_data(item) 257 258 elif isinstance(item, micropython.data.Function): 259 assert item.location == len(self.raw_code) 260 261 # NOTE: Need class and parameter details! Should arguably be types.FunctionType. 262 self.raw_code.append(( 263 objtable.as_list().get_code("__builtins__.function"), 264 objtable.get_index("__builtins__.function"), 265 item.code_location, 266 ( 267 len(item.positional_names), 268 len(item.defaults) 269 ), 270 0 271 )) 272 273 assert item.code_location == len(self.raw_code) 274 275 elif isinstance(item, micropython.data.Module): 276 assert item.location == len(self.raw_code) 277 278 self.raw_code.append(( 279 objtable.as_list().get_code(item.full_name()), 280 None, # module name not used as an attribute 281 None, 282 None, 283 0 284 )) 285 286 else: 287 self.raw_code.append(item) 288 289 # Fix the module locations. 290 291 for module in self.importer.modules_ordered: 292 293 if not with_builtins and module.name == "__builtins__": 294 continue 295 296 module.code_location = module.blocks[0].location 297 298 self.code_location = self.importer.modules["__main__"].code_location 299 return self.raw_code 300 301 def raw_block(self, block): 302 303 "Return the code for the given 'block'." 304 305 assert block.location == len(self.raw_code) 306 for i, item in enumerate(block.code): 307 if hasattr(item, "location"): 308 item.location = location + i 309 return block.code 310 311 def raw_data(self, item): 312 313 "Return the data for the given 'item'." 314 315 datatype = item.value_type_name() 316 317 # NOTE: Start simple and use single entries for most types. 318 319 if datatype in ("__builtins__.tuple", "__builtins__.list"): 320 return [len(item.value)] + list(item.value) 321 else: 322 return [item.value] 323 324 def get_object_table(self): 325 326 "Return a table with details of attributes for classes and modules." 327 328 if self.objtable is None: 329 self.importer.vacuum() 330 331 t = self.objtable = micropython.table.ObjectTable() 332 for module in self.importer.get_modules(): 333 t.add(module.full_name(), module.module_attributes()) 334 335 # Add class and instance attributes for all classes, together 336 # with descendant information. 337 338 for obj in module.all_objects: 339 if isinstance(obj, micropython.inspect.Class): 340 attributes = {obj.full_name() : obj} 341 attributes.update(obj.all_attributes()) 342 attributes.update(obj.all_descendants()) 343 t.add(obj.full_name(), attributes) 344 345 return self.objtable 346 347 def get_parameter_table(self): 348 349 "Return a table with details of parameters for functions and methods." 350 351 # Need the object table to get at class details. 352 353 objtable = self.get_object_table() 354 355 if self.paramtable is None: 356 t = self.paramtable = micropython.table.ParameterTable() 357 358 # Visit each module, getting function and method details. 359 360 for module in self.importer.get_modules(): 361 for obj in module.all_objects: 362 if isinstance(obj, micropython.inspect.Function): 363 t.add(obj.full_name(), obj.parameters()) 364 365 # Classes are callable, too. 366 # Take details of the appropriate __init__ method to make an 367 # entry for an instantiation function for the class. 368 369 elif isinstance(obj, micropython.inspect.Class): 370 t.add(obj.full_name(), obj.get_instantiator().parameters()) 371 372 # Filter out all parameter table entries not referenced by keyword 373 # arguments. 374 375 keyword_names = set() 376 377 for module in self.importer.get_modules(): 378 keyword_names.update(module.keyword_names) 379 380 for function_name, parameters in t.table.items(): 381 for name in parameters.keys(): 382 if name in keyword_names: 383 break 384 else: 385 del t.table[function_name] 386 387 return self.paramtable 388 389 class Importer: 390 391 "An import machine, searching for and loading modules." 392 393 predefined_constants = { 394 "None" : None, 395 "True" : True, 396 "False" : False, 397 "Ellipsis" : Ellipsis, 398 "NotImplemented" : NotImplemented 399 } 400 401 def __init__(self, path=None, verbose=0, optimisations=None): 402 403 """ 404 Initialise the importer with the given search 'path' - a list of 405 directories to search for Python modules. 406 407 The optional 'verbose' parameter causes output concerning the activities 408 of the object to be produced if set to a true value (not the default). 409 410 The optional 'optimisations' cause certain techniques to be used in 411 reducing program size and improving program efficiency. 412 """ 413 414 self.path = path or [os.getcwd()] 415 self.verbose = verbose 416 self.optimisations = optimisations or set() 417 418 self.modules = {} 419 self.modules_ordered = [] 420 self.loading = set() 421 422 # Constant records. 423 424 self.constant_values = {} 425 self.constant_list = None # cache for constants 426 self.init_predefined_constants() 427 428 # Name records (used to track actual use of names). 429 # Include names which may not be explicitly used in programs. 430 431 self.names_used = set(["__init__"]) 432 433 # Status information. 434 435 self.vacuumed = 0 436 self.finalised = 0 437 438 def get_modules(self): 439 440 "Return all modules known to the importer." 441 442 return self.modules.values() 443 444 # General maintenance. 445 446 def vacuum(self): 447 448 "Tidy up the modules." 449 450 if self.vacuumed: 451 return 452 453 for name, module in self.modules.items(): 454 if module.loaded: 455 module.vacuum() 456 else: 457 del self.modules[name] 458 459 self.vacuumed = 1 460 461 def finalise(self): 462 463 "Finalise the program." 464 465 if self.finalised: 466 return 467 468 self.vacuum() 469 470 for module in self.get_modules(): 471 module.finalise() 472 473 self.finalised = 1 474 475 # Name accounting. 476 477 def use_name(self, name): 478 479 "Register the given 'name' as being used in the program." 480 481 self.names_used.add(name) 482 483 # Constant accounting. 484 485 def init_predefined_constants(self): 486 487 "Ensure the predefined constants." 488 489 for name, value in self.predefined_constants.items(): 490 self.make_constant(value) 491 492 def get_predefined_constant(self, name): 493 494 "Return the predefined constant for the given 'name'." 495 496 return self.make_constant(self.predefined_constants[name]) 497 498 def get_constant(self, value): 499 500 "Return a constant for the given 'value'." 501 502 return self.constant_values[value] 503 504 def make_constant(self, value): 505 506 "Make and return a constant for the given 'value'." 507 508 if not self.constant_values.has_key(value): 509 const = micropython.data.Const(value) 510 self.constant_values[value] = const 511 return self.constant_values[value] 512 513 def constants(self): 514 515 "Return a list of constants." 516 517 if self.constant_list is None: 518 self.constant_list = list(self.constant_values.values()) 519 520 return self.constant_list 521 522 # Import methods. 523 524 def find_in_path(self, name): 525 526 """ 527 Find the given module 'name' in the search path, returning None where no 528 such module could be found, or a 2-tuple from the 'find' method 529 otherwise. 530 """ 531 532 for d in self.path: 533 m = self.find(d, name) 534 if m: return m 535 return None 536 537 def find(self, d, name): 538 539 """ 540 In the directory 'd', find the given module 'name', where 'name' can 541 either refer to a single file module or to a package. Return None if the 542 'name' cannot be associated with either a file or a package directory, 543 or a 2-tuple from '_find_package' or '_find_module' otherwise. 544 """ 545 546 m = self._find_package(d, name) 547 if m: return m 548 m = self._find_module(d, name) 549 if m: return m 550 return None 551 552 def _find_module(self, d, name): 553 554 """ 555 In the directory 'd', find the given module 'name', returning None where 556 no suitable file exists in the directory, or a 2-tuple consisting of 557 None (indicating that no package directory is involved) and a filename 558 indicating the location of the module. 559 """ 560 561 name_py = name + os.extsep + "py" 562 filename = self._find_file(d, name_py) 563 if filename: 564 return None, filename 565 return None 566 567 def _find_package(self, d, name): 568 569 """ 570 In the directory 'd', find the given package 'name', returning None 571 where no suitable package directory exists, or a 2-tuple consisting of 572 a directory (indicating the location of the package directory itself) 573 and a filename indicating the location of the __init__.py module which 574 declares the package's top-level contents. 575 """ 576 577 filename = self._find_file(d, name) 578 if filename: 579 init_py = "__init__" + os.path.extsep + "py" 580 init_py_filename = self._find_file(filename, init_py) 581 if init_py_filename: 582 return filename, init_py_filename 583 return None 584 585 def _find_file(self, d, filename): 586 587 """ 588 Return the filename obtained when searching the directory 'd' for the 589 given 'filename', or None if no actual file exists for the filename. 590 """ 591 592 filename = os.path.join(d, filename) 593 if os.path.exists(filename): 594 return filename 595 else: 596 return None 597 598 def load(self, name, return_leaf=0): 599 600 """ 601 Load the module or package with the given 'name'. Return an object 602 referencing the loaded module or package, or None if no such module or 603 package exists. 604 """ 605 606 if self.modules.has_key(name) and self.modules[name].loaded: 607 #print "Cached (%s)" % name 608 return self.modules[name] 609 if self.verbose: 610 print "Loading", name 611 612 # Split the name into path components, and try to find the uppermost in 613 # the search path. 614 615 path = name.split(".") 616 m = self.find_in_path(path[0]) 617 if not m: 618 if self.verbose: 619 print "Not found (%s)" % path[0] 620 return None # NOTE: Import error. 621 d, filename = m 622 623 # Either acquire a reference to an already-imported module, or load the 624 # module from a file. 625 626 top = module = self.load_from_file(filename, path[0]) 627 628 # For hierarchical names, traverse each path component... 629 630 if len(path) > 1: 631 if not d: 632 if self.verbose: 633 print "No package (%s)" % filename 634 return None # NOTE: Import error (package not found). 635 else: 636 self.add_submodules(d, module) 637 638 path_so_far = path[:1] 639 for p in path[1:]: 640 path_so_far.append(p) 641 642 # Find the package or module concerned. 643 644 m = self.find(d, p) 645 if not m: 646 if self.verbose: 647 print "Not found (%s)" % p 648 return None # NOTE: Import error. 649 d, filename = m 650 module_name = ".".join(path_so_far) 651 652 # Either reference an imported module or load one from a file. 653 654 submodule = self.load_from_file(filename, module_name) 655 656 if d: 657 self.add_submodules(d, module) 658 659 # Store the submodule within its parent module. 660 661 module.set_module(p, submodule) 662 module = submodule 663 664 # Return either the deepest or the uppermost module. 665 666 if return_leaf: 667 return module 668 else: 669 return top 670 671 def load_from_file(self, name, module_name=None): 672 673 """ 674 Load the module with the given 'name' (which may be a full module path). 675 """ 676 677 if module_name is None: 678 module_name = "__main__" 679 680 module = self.add_module(module_name) 681 if not module.loaded and module not in self.loading: 682 self.loading.add(module) 683 #print "Parsing", name 684 module.parse(name) 685 #print "Done", name 686 self.loading.remove(module) 687 module.loaded = 1 688 689 # Record the module. 690 691 #print "Loaded", module_name, "with namespace", module.namespace.keys() 692 return module 693 694 def add_module(self, module_name): 695 696 """ 697 Return the module with the given 'module_name', adding a new module 698 object if one does not already exist. 699 """ 700 701 if not self.modules.has_key(module_name): 702 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 703 self.modules_ordered.append(module) 704 else: 705 module = self.modules[module_name] 706 return module 707 708 def add_submodules(self, pathname, module): 709 710 """ 711 Work around insufficient __all__ declarations and examine the directory 712 with the given 'pathname', adding submodules to the given 'module'. 713 """ 714 715 for filename in os.listdir(pathname): 716 submodule, ext = os.path.splitext(filename) 717 if ext not in ("", ".py"): 718 continue 719 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 720 721 # vim: tabstop=4 expandtab shiftwidth=4