1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class Program: 56 57 "This class supports the generation of a program image." 58 59 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 60 61 def __init__(self, importer, optimisations=None): 62 63 """ 64 Initialise the program representation with an 'importer' which is able 65 to locate and load Python modules. 66 67 The optional 'optimisations' cause certain techniques to be used in 68 reducing program size and improving program efficiency. 69 """ 70 71 self.importer = importer 72 self.optimisations = optimisations or set() 73 74 # Remember the tables once generated. 75 76 self.objtable = None 77 self.paramtable = None 78 79 # Main program information. 80 81 self.code = None 82 self.code_location = None 83 84 def get_importer(self): 85 return self.importer 86 87 # Access to finalised program information. 88 89 def get_image(self, with_builtins=0): 90 91 "Return a dictionary mapping modules to structures." 92 93 if self.code is not None: 94 return self.code 95 96 objtable = self.get_object_table() 97 paramtable = self.get_parameter_table() 98 self.importer.finalise() 99 100 self.code = [] 101 102 # Append constants to the image. 103 104 for pos, const in enumerate(self.importer.constants()): 105 self.code.append(const) 106 107 last_module = self.importer.modules_ordered[-1] 108 109 for module in self.importer.modules_ordered: 110 pos = len(self.code) 111 112 # Position the module in the image and make a translation. 113 114 trans = micropython.ast.Translation(module, self) 115 116 # Add header details. 117 118 self.code.append(module) 119 pos += 1 120 121 # Append module attributes to the image. 122 123 attributes = module.module_attributes() 124 self.code += module.attributes_as_list() 125 pos += len(attributes.keys()) 126 127 # Append classes and functions to the image. 128 129 for obj in module.all_objects: 130 if isinstance(obj, micropython.inspect.Class): 131 132 # Add header details. 133 134 self.code.append(obj) 135 pos += 1 136 137 # Append class attributes to the image. 138 139 attributes = obj.class_attributes() 140 self.code += obj.attributes_as_list() 141 pos += len(attributes.keys()) 142 143 # Generate the instantiator/initialiser. 144 # Append the function code to the image. 145 146 instantiator = obj.get_instantiator() 147 code = trans.get_instantiator_code(obj) 148 self.code += code 149 pos += len(code) 150 151 # Class-level code is generated separately at the module 152 # level, and the code location is set within the code 153 # generation process for the module. 154 155 elif isinstance(obj, micropython.inspect.Function): 156 157 # Add header details. 158 159 self.code.append(obj) 160 pos += 1 161 162 # Append any default values to the image. 163 # Only do this for named functions (not lambdas). 164 165 if obj.name is not None: 166 self.code += obj.default_attrs 167 pos += len(obj.default_attrs) 168 169 # Omit built-in function code where requested. 170 171 if not with_builtins and module.name == "__builtins__": 172 pass 173 174 # Append the function code to the image. 175 176 else: 177 code = trans.get_code(obj) 178 self.code += code 179 pos += len(code) 180 181 # Omit built-in module code where requested. 182 183 if not with_builtins and module.name == "__builtins__": 184 pass 185 186 # Append the module top-level code to the image. 187 188 else: 189 code = trans.get_module_code(final=(module is last_module)) 190 self.code += code 191 pos += len(code) 192 193 return self.code 194 195 def get_raw_image(self, with_builtins=0): 196 197 "Return the raw image representation of the program." 198 199 self.get_image(with_builtins) 200 201 objtable = self.get_object_table() 202 paramtable = self.get_parameter_table() 203 204 # Position the objects. 205 206 pos = 0 207 for item in self.code: 208 209 # Blocks are positioned leaving space for their expansion. 210 211 if isinstance(item, Block): 212 item.location = pos 213 pos += len(item.code) 214 215 # Other multi-location objects. 216 217 elif isinstance(item, ( 218 micropython.data.Class, micropython.data.Const, 219 micropython.data.Function, micropython.data.Module 220 )): 221 222 if isinstance(item, micropython.data.Class): 223 224 # Append a template of an instance for use when 225 # instantiating classes. 226 227 item.instance_template_location = pos 228 pos += 1 229 230 # Record the location of the object. 231 232 item.location = pos 233 pos += 1 234 235 # Code and details are associated with certain objects. 236 237 if isinstance(item, micropython.data.Function): 238 239 # Set the code location only where the code has been 240 # generated. 241 242 if not with_builtins and item.module.name == "__builtins__": 243 item.code_location = item.full_name() 244 245 # Skip any defaults. 246 247 else: 248 item.code_location = pos + len(item.defaults) 249 250 elif isinstance(item, micropython.data.Const): 251 pos += len(item.raw_data()) 252 253 else: 254 pos += 1 255 256 # Generate the raw code. 257 258 self.raw_code = [] 259 260 for item in self.code: 261 262 if isinstance(item, micropython.data.Attr): 263 self.raw_code += item.as_raw(objtable, paramtable) 264 265 elif isinstance(item, Block): 266 assert item.location == len(self.raw_code) 267 self.raw_code += item.as_raw(objtable, paramtable) 268 269 # Using classcode, attrcode, codeaddr, codedetails, instance. 270 271 elif isinstance(item, micropython.data.Class): 272 assert item.instance_template_location == len(self.raw_code) 273 self.raw_code += item.as_raw(objtable, paramtable) 274 assert item.location == len(self.raw_code) - 1 275 276 elif isinstance(item, micropython.data.Const): 277 assert item.location == len(self.raw_code) 278 self.raw_code += item.as_raw(objtable, paramtable) 279 280 elif isinstance(item, micropython.data.Function): 281 assert item.location == len(self.raw_code) 282 self.raw_code += item.as_raw(objtable, paramtable) 283 284 # Check the code location only where the code has been generated. 285 286 assert (not with_builtins and item.module.name == "__builtins__") or \ 287 item.code_location == len(self.raw_code) + len(item.defaults) 288 289 elif isinstance(item, micropython.data.Module): 290 assert item.location == len(self.raw_code) 291 self.raw_code += item.as_raw(objtable, paramtable) 292 293 else: 294 self.raw_code.append(item) 295 296 # Fix the module locations. 297 298 for module in self.importer.modules_ordered: 299 300 if not with_builtins and module.name == "__builtins__": 301 continue 302 303 module.code_location = module.blocks[0].location 304 305 self.code_location = self.importer.modules["__main__"].code_location 306 return self.raw_code 307 308 def get_object_table(self): 309 310 "Return a table with details of attributes for classes and modules." 311 312 if self.objtable is None: 313 self.importer.vacuum() 314 315 t = self.objtable = micropython.table.ObjectTable() 316 for module in self.importer.get_modules(): 317 t.add(module.full_name(), module.module_attributes()) 318 319 # Add class and instance attributes for all classes, together 320 # with descendant information. 321 322 for obj in module.all_objects: 323 if isinstance(obj, micropython.inspect.Class): 324 attributes = {obj.full_name() : obj} 325 attributes.update(obj.all_attributes()) 326 attributes.update(obj.all_descendants()) 327 t.add(obj.full_name(), attributes) 328 329 return self.objtable 330 331 def get_parameter_table(self): 332 333 "Return a table with details of parameters for functions and methods." 334 335 # Need the object table to get at class details. 336 337 objtable = self.get_object_table() 338 339 if self.paramtable is None: 340 t = self.paramtable = micropython.table.ParameterTable() 341 342 # Visit each module, getting function and method details. 343 344 for module in self.importer.get_modules(): 345 for obj in module.all_objects: 346 if isinstance(obj, micropython.inspect.Function): 347 t.add(obj.full_name(), obj.parameters()) 348 349 # Classes are callable, too. 350 # Take details of the appropriate __init__ method to make an 351 # entry for an instantiation function for the class. 352 353 elif isinstance(obj, micropython.inspect.Class): 354 t.add(obj.full_name(), obj.get_instantiator().parameters()) 355 356 # Filter out all parameter table entries not referenced by keyword 357 # arguments. 358 359 keyword_names = set() 360 361 for module in self.importer.get_modules(): 362 keyword_names.update(module.keyword_names) 363 364 for function_name, parameters in t.table.items(): 365 for name in parameters.keys(): 366 if name in keyword_names: 367 break 368 else: 369 del t.table[function_name] 370 371 return self.paramtable 372 373 class Importer: 374 375 "An import machine, searching for and loading modules." 376 377 predefined_constants = { 378 "None" : None, 379 "True" : True, 380 "False" : False, 381 "Ellipsis" : Ellipsis, 382 "NotImplemented" : NotImplemented 383 } 384 385 def __init__(self, path=None, verbose=0, optimisations=None): 386 387 """ 388 Initialise the importer with the given search 'path' - a list of 389 directories to search for Python modules. 390 391 The optional 'verbose' parameter causes output concerning the activities 392 of the object to be produced if set to a true value (not the default). 393 394 The optional 'optimisations' cause certain techniques to be used in 395 reducing program size and improving program efficiency. 396 """ 397 398 self.path = path or [os.getcwd()] 399 self.verbose = verbose 400 self.optimisations = optimisations or set() 401 402 self.modules = {} 403 self.modules_ordered = [] 404 self.loading = set() 405 406 # Constant records. 407 408 self.constant_values = {} 409 self.constant_list = None # cache for constants 410 self.init_predefined_constants() 411 412 # Name records (used to track actual use of names). 413 # Include names which may not be explicitly used in programs. 414 # NOTE: Potentially declare these when inspecting. For example, __iter__ 415 # NOTE: and next are usually implicitly invoked by "for" statements. 416 417 self.names_used = set(["__init__", "__call__", "__getitem__", "__bool__", "__iter__", "next"]) 418 419 # Status information. 420 421 self.vacuumed = 0 422 self.finalised = 0 423 424 def get_modules(self): 425 426 "Return all modules known to the importer." 427 428 return self.modules.values() 429 430 # General maintenance. 431 432 def vacuum(self): 433 434 "Tidy up the modules." 435 436 if self.vacuumed: 437 return 438 439 for name, module in self.modules.items(): 440 if module.loaded: 441 module.vacuum() 442 else: 443 del self.modules[name] 444 445 self.vacuumed = 1 446 447 def finalise(self): 448 449 "Finalise the program." 450 451 if self.finalised: 452 return 453 454 self.vacuum() 455 456 for module in self.get_modules(): 457 module.finalise() 458 459 self.finalised = 1 460 461 # Name accounting. 462 463 def use_name(self, name): 464 465 "Register the given 'name' as being used in the program." 466 467 self.names_used.add(name) 468 469 # Constant accounting. 470 471 def init_predefined_constants(self): 472 473 "Ensure the predefined constants." 474 475 for name, value in self.predefined_constants.items(): 476 self.make_constant(value) 477 478 def get_predefined_constant(self, name): 479 480 "Return the predefined constant for the given 'name'." 481 482 return self.make_constant(self.predefined_constants[name]) 483 484 def get_constant(self, value): 485 486 "Return a constant for the given 'value'." 487 488 const = micropython.data.Const(value) 489 return self.constant_values[const] 490 491 def make_constant(self, value): 492 493 "Make and return a constant for the given 'value'." 494 495 const = micropython.data.Const(value) 496 if not self.constant_values.has_key(const): 497 self.constant_values[const] = const 498 return self.constant_values[const] 499 500 def constants(self): 501 502 "Return a list of constants." 503 504 if self.constant_list is None: 505 self.constant_list = list(self.constant_values.values()) 506 507 return self.constant_list 508 509 # Import methods. 510 511 def find_in_path(self, name): 512 513 """ 514 Find the given module 'name' in the search path, returning None where no 515 such module could be found, or a 2-tuple from the 'find' method 516 otherwise. 517 """ 518 519 for d in self.path: 520 m = self.find(d, name) 521 if m: return m 522 return None 523 524 def find(self, d, name): 525 526 """ 527 In the directory 'd', find the given module 'name', where 'name' can 528 either refer to a single file module or to a package. Return None if the 529 'name' cannot be associated with either a file or a package directory, 530 or a 2-tuple from '_find_package' or '_find_module' otherwise. 531 """ 532 533 m = self._find_package(d, name) 534 if m: return m 535 m = self._find_module(d, name) 536 if m: return m 537 return None 538 539 def _find_module(self, d, name): 540 541 """ 542 In the directory 'd', find the given module 'name', returning None where 543 no suitable file exists in the directory, or a 2-tuple consisting of 544 None (indicating that no package directory is involved) and a filename 545 indicating the location of the module. 546 """ 547 548 name_py = name + os.extsep + "py" 549 filename = self._find_file(d, name_py) 550 if filename: 551 return None, filename 552 return None 553 554 def _find_package(self, d, name): 555 556 """ 557 In the directory 'd', find the given package 'name', returning None 558 where no suitable package directory exists, or a 2-tuple consisting of 559 a directory (indicating the location of the package directory itself) 560 and a filename indicating the location of the __init__.py module which 561 declares the package's top-level contents. 562 """ 563 564 filename = self._find_file(d, name) 565 if filename: 566 init_py = "__init__" + os.path.extsep + "py" 567 init_py_filename = self._find_file(filename, init_py) 568 if init_py_filename: 569 return filename, init_py_filename 570 return None 571 572 def _find_file(self, d, filename): 573 574 """ 575 Return the filename obtained when searching the directory 'd' for the 576 given 'filename', or None if no actual file exists for the filename. 577 """ 578 579 filename = os.path.join(d, filename) 580 if os.path.exists(filename): 581 return filename 582 else: 583 return None 584 585 def load(self, name, return_leaf=0): 586 587 """ 588 Load the module or package with the given 'name'. Return an object 589 referencing the loaded module or package, or None if no such module or 590 package exists. 591 """ 592 593 if self.modules.has_key(name) and self.modules[name].loaded: 594 #print "Cached (%s)" % name 595 return self.modules[name] 596 if self.verbose: 597 print "Loading", name 598 599 # Split the name into path components, and try to find the uppermost in 600 # the search path. 601 602 path = name.split(".") 603 m = self.find_in_path(path[0]) 604 if not m: 605 if self.verbose: 606 print "Not found (%s)" % path[0] 607 return None # NOTE: Import error. 608 d, filename = m 609 610 # Either acquire a reference to an already-imported module, or load the 611 # module from a file. 612 613 top = module = self.load_from_file(filename, path[0]) 614 615 # For hierarchical names, traverse each path component... 616 617 if len(path) > 1: 618 if not d: 619 if self.verbose: 620 print "No package (%s)" % filename 621 return None # NOTE: Import error (package not found). 622 else: 623 self.add_submodules(d, module) 624 625 path_so_far = path[:1] 626 for p in path[1:]: 627 path_so_far.append(p) 628 629 # Find the package or module concerned. 630 631 m = self.find(d, p) 632 if not m: 633 if self.verbose: 634 print "Not found (%s)" % p 635 return None # NOTE: Import error. 636 d, filename = m 637 module_name = ".".join(path_so_far) 638 639 # Either reference an imported module or load one from a file. 640 641 submodule = self.load_from_file(filename, module_name) 642 643 if d: 644 self.add_submodules(d, module) 645 646 # Store the submodule within its parent module. 647 648 module.set_module(p, submodule) 649 module = submodule 650 651 # Return either the deepest or the uppermost module. 652 653 if return_leaf: 654 return module 655 else: 656 return top 657 658 def load_from_file(self, name, module_name=None): 659 660 """ 661 Load the module with the given 'name' (which may be a full module path). 662 """ 663 664 if module_name is None: 665 module_name = "__main__" 666 667 module = self.add_module(module_name) 668 if not module.loaded and module not in self.loading: 669 self.loading.add(module) 670 #print "Parsing", name 671 module.parse(name) 672 #print "Done", name 673 self.loading.remove(module) 674 module.loaded = 1 675 676 # Record the module. 677 678 #print "Loaded", module_name, "with namespace", module.namespace.keys() 679 return module 680 681 def add_module(self, module_name): 682 683 """ 684 Return the module with the given 'module_name', adding a new module 685 object if one does not already exist. 686 """ 687 688 if not self.modules.has_key(module_name): 689 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 690 self.modules_ordered.append(module) 691 else: 692 module = self.modules[module_name] 693 return module 694 695 def add_submodules(self, pathname, module): 696 697 """ 698 Work around insufficient __all__ declarations and examine the directory 699 with the given 'pathname', adding submodules to the given 'module'. 700 """ 701 702 for filename in os.listdir(pathname): 703 submodule, ext = os.path.splitext(filename) 704 if ext not in ("", ".py"): 705 continue 706 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 707 708 # vim: tabstop=4 expandtab shiftwidth=4