1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class Program: 56 57 "This class supports the generation of a program image." 58 59 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 60 61 def __init__(self, importer, optimisations=None): 62 63 """ 64 Initialise the program representation with an 'importer' which is able 65 to locate and load Python modules. 66 67 The optional 'optimisations' cause certain techniques to be used in 68 reducing program size and improving program efficiency. 69 """ 70 71 self.importer = importer 72 self.optimisations = optimisations or set() 73 74 # Remember the tables once generated. 75 76 self.objtable = None 77 self.paramtable = None 78 79 # Main program information. 80 81 self.code = None 82 self.code_location = None 83 84 def get_importer(self): 85 return self.importer 86 87 # Access to finalised program information. 88 89 def get_image(self, with_builtins=0): 90 91 "Return a dictionary mapping modules to structures." 92 93 if self.code is not None: 94 return self.code 95 96 objtable = self.get_object_table() 97 paramtable = self.get_parameter_table() 98 self.importer.finalise() 99 100 self.code = [] 101 102 # Append constants to the image. 103 104 for pos, const in enumerate(self.importer.constants()): 105 self.code.append(const) 106 107 last_module = self.importer.modules_ordered[-1] 108 109 for module in self.importer.modules_ordered: 110 pos = len(self.code) 111 112 # Position the module in the image and make a translation. 113 114 trans = micropython.ast.Translation(module, self) 115 116 # Add header details. 117 118 self.code.append(module) 119 pos += 1 120 121 # Append module attributes to the image. 122 123 attributes = module.module_attributes() 124 self.code += module.attributes_as_list() 125 pos += len(attributes.keys()) 126 127 # Append classes and functions to the image. 128 129 for obj in module.all_objects: 130 if isinstance(obj, micropython.inspect.Class): 131 132 # Add header details. 133 134 self.code.append(obj) 135 pos += 1 136 137 # Append class attributes to the image. 138 139 attributes = obj.class_attributes() 140 self.code += obj.attributes_as_list() 141 pos += len(attributes.keys()) 142 143 # Omit built-in function code where requested. 144 145 if not with_builtins and module.name == "__builtins__": 146 continue 147 148 # Generate the instantiator/initialiser. 149 # Append the function code to the image. 150 151 instantiator = obj.get_instantiator() 152 code = trans.get_instantiator_code(obj) 153 self.code += code 154 pos += len(code) 155 156 # Class-level code is generated separately at the module 157 # level, and the code location is set within the code 158 # generation process for the module. 159 160 elif isinstance(obj, micropython.inspect.Function): 161 162 # Add header details. 163 164 self.code.append(obj) 165 pos += 1 166 167 # Append any default values to the image. 168 # Only do this for named functions (not lambdas). 169 170 if obj.name is not None: 171 self.code += obj.default_attrs 172 pos += len(obj.default_attrs) 173 174 # Omit built-in function code where requested. 175 176 if not with_builtins and module.name == "__builtins__": 177 pass 178 179 # Append the function code to the image. 180 181 else: 182 code = trans.get_code(obj) 183 self.code += code 184 pos += len(code) 185 186 # Omit built-in module code where requested. 187 188 if not with_builtins and module.name == "__builtins__": 189 pass 190 191 # Append the module top-level code to the image. 192 193 else: 194 code = trans.get_module_code(final=(module is last_module)) 195 self.code += code 196 pos += len(code) 197 198 return self.code 199 200 def get_raw_image(self, with_builtins=0): 201 202 "Return the raw image representation of the program." 203 204 self.get_image(with_builtins) 205 206 objtable = self.get_object_table() 207 paramtable = self.get_parameter_table() 208 209 # Position the objects. 210 211 pos = 0 212 for item in self.code: 213 214 # Blocks are positioned leaving space for their expansion. 215 216 if isinstance(item, Block): 217 item.location = pos 218 pos += len(item.code) 219 220 # Other multi-location objects. 221 222 elif isinstance(item, ( 223 micropython.data.Class, micropython.data.Const, 224 micropython.data.Function, micropython.data.Module 225 )): 226 227 if isinstance(item, micropython.data.Class): 228 229 # Append a template of an instance for use when 230 # instantiating classes. 231 232 item.instance_template_location = pos 233 pos += 1 234 235 # Record the location of the object. 236 237 item.location = pos 238 pos += 1 239 240 # Code and details are associated with certain objects. 241 242 if isinstance(item, micropython.data.Function): 243 244 # Set the code location only where the code has been 245 # generated. 246 247 if not with_builtins and item.module.name == "__builtins__": 248 item.code_location = item.full_name() 249 250 # Skip any defaults. 251 252 else: 253 item.code_location = pos + len(item.defaults) 254 255 elif isinstance(item, micropython.data.Const): 256 pos += len(item.raw_data()) 257 258 else: 259 pos += 1 260 261 # Generate the raw code. 262 263 self.raw_code = [] 264 265 for item in self.code: 266 267 if isinstance(item, micropython.data.Attr): 268 self.raw_code += item.as_raw(objtable, paramtable) 269 270 elif isinstance(item, Block): 271 assert item.location == len(self.raw_code) 272 self.raw_code += item.as_raw(objtable, paramtable) 273 274 # Using classcode, attrcode, codeaddr, codedetails, instance. 275 276 elif isinstance(item, micropython.data.Class): 277 assert item.instance_template_location == len(self.raw_code) 278 self.raw_code += item.as_raw(objtable, paramtable, with_builtins or item.module.name != "__builtins__") 279 assert item.location == len(self.raw_code) - 1 280 281 elif isinstance(item, micropython.data.Const): 282 assert item.location == len(self.raw_code) 283 self.raw_code += item.as_raw(objtable, paramtable) 284 285 elif isinstance(item, micropython.data.Function): 286 assert item.location == len(self.raw_code) 287 self.raw_code += item.as_raw(objtable, paramtable) 288 289 # Check the code location only where the code has been generated. 290 291 assert (not with_builtins and item.module.name == "__builtins__") or \ 292 item.code_location == len(self.raw_code) + len(item.defaults) 293 294 elif isinstance(item, micropython.data.Module): 295 assert item.location == len(self.raw_code) 296 self.raw_code += item.as_raw(objtable, paramtable) 297 298 else: 299 self.raw_code.append(item) 300 301 # Fix the module locations. 302 303 for module in self.importer.modules_ordered: 304 305 if not with_builtins and module.name == "__builtins__": 306 continue 307 308 module.code_location = module.blocks[0].location 309 310 self.code_location = self.importer.modules["__main__"].code_location 311 return self.raw_code 312 313 def get_object_table(self): 314 315 "Return a table with details of attributes for classes and modules." 316 317 if self.objtable is None: 318 self.importer.vacuum() 319 320 t = self.objtable = micropython.table.ObjectTable() 321 for module in self.importer.get_modules(): 322 t.add(module.full_name(), module.module_attributes()) 323 324 # Add class and instance attributes for all classes, together 325 # with descendant information. 326 327 for obj in module.all_objects: 328 if isinstance(obj, micropython.inspect.Class): 329 attributes = {obj.full_name() : obj} 330 attributes.update(obj.all_attributes()) 331 attributes.update(obj.all_descendants()) 332 t.add(obj.full_name(), attributes) 333 334 return self.objtable 335 336 def get_parameter_table(self): 337 338 "Return a table with details of parameters for functions and methods." 339 340 # Need the object table to get at class details. 341 342 objtable = self.get_object_table() 343 344 if self.paramtable is None: 345 t = self.paramtable = micropython.table.ParameterTable() 346 347 # Visit each module, getting function and method details. 348 349 for module in self.importer.get_modules(): 350 for obj in module.all_objects: 351 if isinstance(obj, micropython.inspect.Function): 352 t.add(obj.full_name(), obj.parameters()) 353 354 # Classes are callable, too. 355 # Take details of the appropriate __init__ method to make an 356 # entry for an instantiation function for the class. 357 358 elif isinstance(obj, micropython.inspect.Class): 359 t.add(obj.full_name(), obj.get_instantiator().parameters()) 360 361 # Filter out all parameter table entries not referenced by keyword 362 # arguments. 363 364 keyword_names = set() 365 366 for module in self.importer.get_modules(): 367 keyword_names.update(module.keyword_names) 368 369 for function_name, parameters in t.table.items(): 370 for name in parameters.keys(): 371 if name in keyword_names: 372 break 373 else: 374 del t.table[function_name] 375 376 return self.paramtable 377 378 class Importer: 379 380 "An import machine, searching for and loading modules." 381 382 predefined_constants = { 383 "None" : None, 384 "True" : True, 385 "False" : False, 386 #"Ellipsis" : Ellipsis, 387 "NotImplemented" : NotImplemented 388 } 389 390 def __init__(self, path=None, verbose=0, optimisations=None): 391 392 """ 393 Initialise the importer with the given search 'path' - a list of 394 directories to search for Python modules. 395 396 The optional 'verbose' parameter causes output concerning the activities 397 of the object to be produced if set to a true value (not the default). 398 399 The optional 'optimisations' cause certain techniques to be used in 400 reducing program size and improving program efficiency. 401 """ 402 403 self.path = path or [os.getcwd()] 404 self.verbose = verbose 405 self.optimisations = optimisations or set() 406 407 self.modules = {} 408 self.modules_ordered = [] 409 self.loading = set() 410 411 # Constant records. 412 413 self.constant_values = {} 414 self.constant_list = None # cache for constants 415 self.init_predefined_constants() 416 417 # Name records (used to track actual use of names). 418 # Include names which may not be explicitly used in programs. 419 # NOTE: Potentially declare these when inspecting. For example, __iter__ 420 # NOTE: and next are usually implicitly invoked by "for" statements. 421 422 self.names_used = set(["__init__", "__call__", "__getitem__", "__bool__", "__iter__", "next"]) 423 424 # Status information. 425 426 self.vacuumed = 0 427 self.finalised = 0 428 429 def get_modules(self): 430 431 "Return all modules known to the importer." 432 433 return self.modules.values() 434 435 # General maintenance. 436 437 def vacuum(self): 438 439 "Tidy up the modules." 440 441 if self.vacuumed: 442 return 443 444 for name, module in self.modules.items(): 445 if module.loaded: 446 module.vacuum() 447 else: 448 del self.modules[name] 449 450 self.vacuumed = 1 451 452 def finalise(self): 453 454 "Finalise the program." 455 456 if self.finalised: 457 return 458 459 self.vacuum() 460 461 for module in self.get_modules(): 462 module.finalise() 463 464 self.finalised = 1 465 466 # Name accounting. 467 468 def use_name(self, name): 469 470 "Register the given 'name' as being used in the program." 471 472 self.names_used.add(name) 473 474 # Constant accounting. 475 476 def init_predefined_constants(self): 477 478 "Ensure the predefined constants." 479 480 for name, value in self.predefined_constants.items(): 481 self.make_constant(value) 482 483 def get_predefined_constant(self, name): 484 485 "Return the predefined constant for the given 'name'." 486 487 return self.make_constant(self.predefined_constants[name]) 488 489 def get_constant(self, value): 490 491 "Return a constant for the given 'value'." 492 493 const = micropython.data.Const(value) 494 return self.constant_values[const] 495 496 def make_constant(self, value): 497 498 "Make and return a constant for the given 'value'." 499 500 const = micropython.data.Const(value) 501 if not self.constant_values.has_key(const): 502 self.constant_values[const] = const 503 return self.constant_values[const] 504 505 def constants(self): 506 507 "Return a list of constants." 508 509 if self.constant_list is None: 510 self.constant_list = list(self.constant_values.values()) 511 512 return self.constant_list 513 514 # Import methods. 515 516 def find_in_path(self, name): 517 518 """ 519 Find the given module 'name' in the search path, returning None where no 520 such module could be found, or a 2-tuple from the 'find' method 521 otherwise. 522 """ 523 524 for d in self.path: 525 m = self.find(d, name) 526 if m: return m 527 return None 528 529 def find(self, d, name): 530 531 """ 532 In the directory 'd', find the given module 'name', where 'name' can 533 either refer to a single file module or to a package. Return None if the 534 'name' cannot be associated with either a file or a package directory, 535 or a 2-tuple from '_find_package' or '_find_module' otherwise. 536 """ 537 538 m = self._find_package(d, name) 539 if m: return m 540 m = self._find_module(d, name) 541 if m: return m 542 return None 543 544 def _find_module(self, d, name): 545 546 """ 547 In the directory 'd', find the given module 'name', returning None where 548 no suitable file exists in the directory, or a 2-tuple consisting of 549 None (indicating that no package directory is involved) and a filename 550 indicating the location of the module. 551 """ 552 553 name_py = name + os.extsep + "py" 554 filename = self._find_file(d, name_py) 555 if filename: 556 return None, filename 557 return None 558 559 def _find_package(self, d, name): 560 561 """ 562 In the directory 'd', find the given package 'name', returning None 563 where no suitable package directory exists, or a 2-tuple consisting of 564 a directory (indicating the location of the package directory itself) 565 and a filename indicating the location of the __init__.py module which 566 declares the package's top-level contents. 567 """ 568 569 filename = self._find_file(d, name) 570 if filename: 571 init_py = "__init__" + os.path.extsep + "py" 572 init_py_filename = self._find_file(filename, init_py) 573 if init_py_filename: 574 return filename, init_py_filename 575 return None 576 577 def _find_file(self, d, filename): 578 579 """ 580 Return the filename obtained when searching the directory 'd' for the 581 given 'filename', or None if no actual file exists for the filename. 582 """ 583 584 filename = os.path.join(d, filename) 585 if os.path.exists(filename): 586 return filename 587 else: 588 return None 589 590 def load(self, name, return_leaf=0): 591 592 """ 593 Load the module or package with the given 'name'. Return an object 594 referencing the loaded module or package, or None if no such module or 595 package exists. 596 """ 597 598 if self.modules.has_key(name) and self.modules[name].loaded: 599 #print "Cached (%s)" % name 600 return self.modules[name] 601 if self.verbose: 602 print "Loading", name 603 604 # Split the name into path components, and try to find the uppermost in 605 # the search path. 606 607 path = name.split(".") 608 m = self.find_in_path(path[0]) 609 if not m: 610 if self.verbose: 611 print "Not found (%s)" % path[0] 612 return None # NOTE: Import error. 613 d, filename = m 614 615 # Either acquire a reference to an already-imported module, or load the 616 # module from a file. 617 618 top = module = self.load_from_file(filename, path[0]) 619 620 # For hierarchical names, traverse each path component... 621 622 if len(path) > 1: 623 if not d: 624 if self.verbose: 625 print "No package (%s)" % filename 626 return None # NOTE: Import error (package not found). 627 else: 628 self.add_submodules(d, module) 629 630 path_so_far = path[:1] 631 for p in path[1:]: 632 path_so_far.append(p) 633 634 # Find the package or module concerned. 635 636 m = self.find(d, p) 637 if not m: 638 if self.verbose: 639 print "Not found (%s)" % p 640 return None # NOTE: Import error. 641 d, filename = m 642 module_name = ".".join(path_so_far) 643 644 # Either reference an imported module or load one from a file. 645 646 submodule = self.load_from_file(filename, module_name) 647 648 if d: 649 self.add_submodules(d, module) 650 651 # Store the submodule within its parent module. 652 653 module.set_module(p, submodule) 654 module = submodule 655 656 # Return either the deepest or the uppermost module. 657 658 if return_leaf: 659 return module 660 else: 661 return top 662 663 def load_from_file(self, name, module_name=None): 664 665 """ 666 Load the module with the given 'name' (which may be a full module path). 667 """ 668 669 if module_name is None: 670 module_name = "__main__" 671 672 module = self.add_module(module_name) 673 if not module.loaded and module not in self.loading: 674 self.loading.add(module) 675 #print "Parsing", name 676 module.parse(name) 677 #print "Done", name 678 self.loading.remove(module) 679 module.loaded = 1 680 681 # Record the module. 682 683 #print "Loaded", module_name, "with namespace", module.namespace.keys() 684 return module 685 686 def add_module(self, module_name): 687 688 """ 689 Return the module with the given 'module_name', adding a new module 690 object if one does not already exist. 691 """ 692 693 if not self.modules.has_key(module_name): 694 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 695 self.modules_ordered.append(module) 696 else: 697 module = self.modules[module_name] 698 return module 699 700 def add_submodules(self, pathname, module): 701 702 """ 703 Work around insufficient __all__ declarations and examine the directory 704 with the given 'pathname', adding submodules to the given 'module'. 705 """ 706 707 for filename in os.listdir(pathname): 708 submodule, ext = os.path.splitext(filename) 709 if ext not in ("", ".py"): 710 continue 711 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 712 713 # vim: tabstop=4 expandtab shiftwidth=4