1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 import micropython.ast 44 import micropython.data 45 import micropython.opt 46 import micropython.inspect 47 import micropython.table 48 import os 49 try: 50 set 51 except NameError: 52 from sets import Set as set 53 54 class Program: 55 56 "This class supports the generation of a program image." 57 58 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 59 60 def __init__(self, importer, optimisations=None): 61 62 """ 63 Initialise the program representation with an 'importer' which is able 64 to locate and load Python modules. 65 66 The optional 'optimisations' cause certain techniques to be used in 67 reducing program size and improving program efficiency. 68 """ 69 70 self.importer = importer 71 self.optimisations = optimisations or set() 72 73 # Remember the tables once generated. 74 75 self.objtable = None 76 self.paramtable = None 77 78 # Main program information. 79 80 self.code = None 81 self.code_location = None 82 83 def get_importer(self): 84 return self.importer 85 86 # Access to finalised program information. 87 88 def get_image(self, with_builtins=0): 89 90 "Return a dictionary mapping modules to structures." 91 92 if self.code is not None: 93 return self.code 94 95 objtable = self.get_object_table() 96 paramtable = self.get_parameter_table() 97 self.importer.finalise() 98 99 self.code = [] 100 101 # Append constants to the image. 102 103 for pos, const in enumerate(self.importer.constants()): 104 self.code.append(const) 105 106 last_module = self.importer.modules_ordered[-1] 107 108 for module in self.importer.modules_ordered: 109 pos = len(self.code) 110 111 # Position the module in the image and make a translation. 112 113 trans = micropython.ast.Translation(module, self) 114 115 # Add header details. 116 117 self.code.append(module) 118 pos += 1 119 120 # Append module attributes to the image. 121 122 attributes = module.module_attributes() 123 self.code += module.attributes_as_list() 124 pos += len(attributes.keys()) 125 126 # Append classes and functions to the image. 127 128 for obj in module.all_objects: 129 if isinstance(obj, micropython.inspect.Class): 130 131 # Add header details. 132 133 self.code.append(obj) 134 pos += 1 135 136 # Append class attributes to the image. 137 138 attributes = obj.class_attributes() 139 self.code += obj.attributes_as_list() 140 pos += len(attributes.keys()) 141 142 # Generate the instantiator/initialiser. 143 # Append the function code to the image. 144 145 instantiator = obj.get_instantiator() 146 code = trans.get_instantiator_code(obj) 147 self.code += code 148 pos += len(code) 149 150 # Class-level code is generated separately at the module 151 # level, and the code location is set within the code 152 # generation process for the module. 153 154 elif isinstance(obj, micropython.inspect.Function): 155 156 # Add header details. 157 158 self.code.append(obj) 159 pos += 1 160 161 # Append any default values to the image. 162 # Only do this for named functions (not lambdas). 163 164 if obj.name is not None: 165 self.code += obj.default_attrs 166 pos += len(obj.default_attrs) 167 168 # Omit built-in function code where requested. 169 170 if not with_builtins and module.name == "__builtins__": 171 pass 172 173 # Append the function code to the image. 174 175 else: 176 code = trans.get_code(obj) 177 self.code += code 178 pos += len(code) 179 180 # Omit built-in module code where requested. 181 182 if not with_builtins and module.name == "__builtins__": 183 pass 184 185 # Append the module top-level code to the image. 186 187 else: 188 code = trans.get_module_code(final=(module is last_module)) 189 self.code += code 190 pos += len(code) 191 192 return self.code 193 194 def get_raw_image(self, with_builtins=0): 195 196 "Return the raw image representation of the program." 197 198 self.get_image(with_builtins) 199 200 objtable = self.get_object_table() 201 paramtable = self.get_parameter_table() 202 203 # Position the objects. 204 205 pos = 0 206 for item in self.code: 207 208 # Blocks are positioned leaving space for their expansion. 209 210 if isinstance(item, Block): 211 item.location = pos 212 pos += len(item.code) 213 214 # Other multi-location objects. 215 216 elif isinstance(item, ( 217 micropython.data.Class, micropython.data.Const, 218 micropython.data.Function, micropython.data.Module 219 )): 220 221 if isinstance(item, micropython.data.Class): 222 223 # Append a template of an instance for use when 224 # instantiating classes. 225 226 item.instance_template_location = pos 227 pos += 1 228 229 # Record the location of the object. 230 231 item.location = pos 232 pos += 1 233 234 # Code and details are associated with certain objects. 235 236 if isinstance(item, micropython.data.Function): 237 238 # Set the code location only where the code has been 239 # generated. 240 241 if not with_builtins and item.module.name == "__builtins__": 242 item.code_location = item.full_name() 243 244 # Skip any defaults. 245 246 else: 247 item.code_location = pos + len(item.defaults) 248 249 elif isinstance(item, micropython.data.Const): 250 pos += len(item.raw_data()) 251 252 else: 253 pos += 1 254 255 # Generate the raw code. 256 257 self.raw_code = [] 258 259 for item in self.code: 260 261 if isinstance(item, micropython.data.Attr): 262 self.raw_code += item.as_raw(objtable, paramtable) 263 264 elif isinstance(item, Block): 265 assert item.location == len(self.raw_code) 266 self.raw_code += item.as_raw(objtable, paramtable) 267 268 # Using classcode, attrcode, codeaddr, codedetails, instance. 269 270 elif isinstance(item, micropython.data.Class): 271 assert item.instance_template_location == len(self.raw_code) 272 self.raw_code += item.as_raw(objtable, paramtable) 273 assert item.location == len(self.raw_code) - 1 274 275 elif isinstance(item, micropython.data.Const): 276 assert item.location == len(self.raw_code) 277 self.raw_code += item.as_raw(objtable, paramtable) 278 279 elif isinstance(item, micropython.data.Function): 280 assert item.location == len(self.raw_code) 281 self.raw_code += item.as_raw(objtable, paramtable) 282 283 # Check the code location only where the code has been generated. 284 285 assert (not with_builtins and item.module.name == "__builtins__") or \ 286 item.code_location == len(self.raw_code) + len(item.defaults) 287 288 elif isinstance(item, micropython.data.Module): 289 assert item.location == len(self.raw_code) 290 self.raw_code += item.as_raw(objtable, paramtable) 291 292 else: 293 self.raw_code.append(item) 294 295 # Fix the module locations. 296 297 for module in self.importer.modules_ordered: 298 299 if not with_builtins and module.name == "__builtins__": 300 continue 301 302 module.code_location = module.blocks[0].location 303 304 self.code_location = self.importer.modules["__main__"].code_location 305 return self.raw_code 306 307 def get_object_table(self): 308 309 "Return a table with details of attributes for classes and modules." 310 311 if self.objtable is None: 312 self.importer.vacuum() 313 314 t = self.objtable = micropython.table.ObjectTable() 315 for module in self.importer.get_modules(): 316 t.add(module.full_name(), module.module_attributes()) 317 318 # Add class and instance attributes for all classes, together 319 # with descendant information. 320 321 for obj in module.all_objects: 322 if isinstance(obj, micropython.inspect.Class): 323 attributes = {obj.full_name() : obj} 324 attributes.update(obj.all_attributes()) 325 attributes.update(obj.all_descendants()) 326 t.add(obj.full_name(), attributes) 327 328 return self.objtable 329 330 def get_parameter_table(self): 331 332 "Return a table with details of parameters for functions and methods." 333 334 # Need the object table to get at class details. 335 336 objtable = self.get_object_table() 337 338 if self.paramtable is None: 339 t = self.paramtable = micropython.table.ParameterTable() 340 341 # Visit each module, getting function and method details. 342 343 for module in self.importer.get_modules(): 344 for obj in module.all_objects: 345 if isinstance(obj, micropython.inspect.Function): 346 t.add(obj.full_name(), obj.parameters()) 347 348 # Classes are callable, too. 349 # Take details of the appropriate __init__ method to make an 350 # entry for an instantiation function for the class. 351 352 elif isinstance(obj, micropython.inspect.Class): 353 t.add(obj.full_name(), obj.get_instantiator().parameters()) 354 355 # Filter out all parameter table entries not referenced by keyword 356 # arguments. 357 358 keyword_names = set() 359 360 for module in self.importer.get_modules(): 361 keyword_names.update(module.keyword_names) 362 363 for function_name, parameters in t.table.items(): 364 for name in parameters.keys(): 365 if name in keyword_names: 366 break 367 else: 368 del t.table[function_name] 369 370 return self.paramtable 371 372 class Importer: 373 374 "An import machine, searching for and loading modules." 375 376 predefined_constants = { 377 "None" : None, 378 "True" : True, 379 "False" : False, 380 "Ellipsis" : Ellipsis, 381 "NotImplemented" : NotImplemented 382 } 383 384 def __init__(self, path=None, verbose=0, optimisations=None): 385 386 """ 387 Initialise the importer with the given search 'path' - a list of 388 directories to search for Python modules. 389 390 The optional 'verbose' parameter causes output concerning the activities 391 of the object to be produced if set to a true value (not the default). 392 393 The optional 'optimisations' cause certain techniques to be used in 394 reducing program size and improving program efficiency. 395 """ 396 397 self.path = path or [os.getcwd()] 398 self.verbose = verbose 399 self.optimisations = optimisations or set() 400 401 self.modules = {} 402 self.modules_ordered = [] 403 self.loading = set() 404 405 # Constant records. 406 407 self.constant_values = {} 408 self.constant_list = None # cache for constants 409 self.init_predefined_constants() 410 411 # Name records (used to track actual use of names). 412 # Include names which may not be explicitly used in programs. 413 414 self.names_used = set(["__init__", "__call__", "__getitem__", "__bool__"]) 415 416 # Status information. 417 418 self.vacuumed = 0 419 self.finalised = 0 420 421 def get_modules(self): 422 423 "Return all modules known to the importer." 424 425 return self.modules.values() 426 427 # General maintenance. 428 429 def vacuum(self): 430 431 "Tidy up the modules." 432 433 if self.vacuumed: 434 return 435 436 for name, module in self.modules.items(): 437 if module.loaded: 438 module.vacuum() 439 else: 440 del self.modules[name] 441 442 self.vacuumed = 1 443 444 def finalise(self): 445 446 "Finalise the program." 447 448 if self.finalised: 449 return 450 451 self.vacuum() 452 453 for module in self.get_modules(): 454 module.finalise() 455 456 self.finalised = 1 457 458 # Name accounting. 459 460 def use_name(self, name): 461 462 "Register the given 'name' as being used in the program." 463 464 self.names_used.add(name) 465 466 # Constant accounting. 467 468 def init_predefined_constants(self): 469 470 "Ensure the predefined constants." 471 472 for name, value in self.predefined_constants.items(): 473 self.make_constant(value) 474 475 def get_predefined_constant(self, name): 476 477 "Return the predefined constant for the given 'name'." 478 479 return self.make_constant(self.predefined_constants[name]) 480 481 def get_constant(self, value): 482 483 "Return a constant for the given 'value'." 484 485 const = micropython.data.Const(value) 486 return self.constant_values[const] 487 488 def make_constant(self, value): 489 490 "Make and return a constant for the given 'value'." 491 492 const = micropython.data.Const(value) 493 if not self.constant_values.has_key(const): 494 self.constant_values[const] = const 495 return self.constant_values[const] 496 497 def constants(self): 498 499 "Return a list of constants." 500 501 if self.constant_list is None: 502 self.constant_list = list(self.constant_values.values()) 503 504 return self.constant_list 505 506 # Import methods. 507 508 def find_in_path(self, name): 509 510 """ 511 Find the given module 'name' in the search path, returning None where no 512 such module could be found, or a 2-tuple from the 'find' method 513 otherwise. 514 """ 515 516 for d in self.path: 517 m = self.find(d, name) 518 if m: return m 519 return None 520 521 def find(self, d, name): 522 523 """ 524 In the directory 'd', find the given module 'name', where 'name' can 525 either refer to a single file module or to a package. Return None if the 526 'name' cannot be associated with either a file or a package directory, 527 or a 2-tuple from '_find_package' or '_find_module' otherwise. 528 """ 529 530 m = self._find_package(d, name) 531 if m: return m 532 m = self._find_module(d, name) 533 if m: return m 534 return None 535 536 def _find_module(self, d, name): 537 538 """ 539 In the directory 'd', find the given module 'name', returning None where 540 no suitable file exists in the directory, or a 2-tuple consisting of 541 None (indicating that no package directory is involved) and a filename 542 indicating the location of the module. 543 """ 544 545 name_py = name + os.extsep + "py" 546 filename = self._find_file(d, name_py) 547 if filename: 548 return None, filename 549 return None 550 551 def _find_package(self, d, name): 552 553 """ 554 In the directory 'd', find the given package 'name', returning None 555 where no suitable package directory exists, or a 2-tuple consisting of 556 a directory (indicating the location of the package directory itself) 557 and a filename indicating the location of the __init__.py module which 558 declares the package's top-level contents. 559 """ 560 561 filename = self._find_file(d, name) 562 if filename: 563 init_py = "__init__" + os.path.extsep + "py" 564 init_py_filename = self._find_file(filename, init_py) 565 if init_py_filename: 566 return filename, init_py_filename 567 return None 568 569 def _find_file(self, d, filename): 570 571 """ 572 Return the filename obtained when searching the directory 'd' for the 573 given 'filename', or None if no actual file exists for the filename. 574 """ 575 576 filename = os.path.join(d, filename) 577 if os.path.exists(filename): 578 return filename 579 else: 580 return None 581 582 def load(self, name, return_leaf=0): 583 584 """ 585 Load the module or package with the given 'name'. Return an object 586 referencing the loaded module or package, or None if no such module or 587 package exists. 588 """ 589 590 if self.modules.has_key(name) and self.modules[name].loaded: 591 #print "Cached (%s)" % name 592 return self.modules[name] 593 if self.verbose: 594 print "Loading", name 595 596 # Split the name into path components, and try to find the uppermost in 597 # the search path. 598 599 path = name.split(".") 600 m = self.find_in_path(path[0]) 601 if not m: 602 if self.verbose: 603 print "Not found (%s)" % path[0] 604 return None # NOTE: Import error. 605 d, filename = m 606 607 # Either acquire a reference to an already-imported module, or load the 608 # module from a file. 609 610 top = module = self.load_from_file(filename, path[0]) 611 612 # For hierarchical names, traverse each path component... 613 614 if len(path) > 1: 615 if not d: 616 if self.verbose: 617 print "No package (%s)" % filename 618 return None # NOTE: Import error (package not found). 619 else: 620 self.add_submodules(d, module) 621 622 path_so_far = path[:1] 623 for p in path[1:]: 624 path_so_far.append(p) 625 626 # Find the package or module concerned. 627 628 m = self.find(d, p) 629 if not m: 630 if self.verbose: 631 print "Not found (%s)" % p 632 return None # NOTE: Import error. 633 d, filename = m 634 module_name = ".".join(path_so_far) 635 636 # Either reference an imported module or load one from a file. 637 638 submodule = self.load_from_file(filename, module_name) 639 640 if d: 641 self.add_submodules(d, module) 642 643 # Store the submodule within its parent module. 644 645 module.set_module(p, submodule) 646 module = submodule 647 648 # Return either the deepest or the uppermost module. 649 650 if return_leaf: 651 return module 652 else: 653 return top 654 655 def load_from_file(self, name, module_name=None): 656 657 """ 658 Load the module with the given 'name' (which may be a full module path). 659 """ 660 661 if module_name is None: 662 module_name = "__main__" 663 664 module = self.add_module(module_name) 665 if not module.loaded and module not in self.loading: 666 self.loading.add(module) 667 #print "Parsing", name 668 module.parse(name) 669 #print "Done", name 670 self.loading.remove(module) 671 module.loaded = 1 672 673 # Record the module. 674 675 #print "Loaded", module_name, "with namespace", module.namespace.keys() 676 return module 677 678 def add_module(self, module_name): 679 680 """ 681 Return the module with the given 'module_name', adding a new module 682 object if one does not already exist. 683 """ 684 685 if not self.modules.has_key(module_name): 686 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 687 self.modules_ordered.append(module) 688 else: 689 module = self.modules[module_name] 690 return module 691 692 def add_submodules(self, pathname, module): 693 694 """ 695 Work around insufficient __all__ declarations and examine the directory 696 with the given 'pathname', adding submodules to the given 'module'. 697 """ 698 699 for filename in os.listdir(pathname): 700 submodule, ext = os.path.splitext(filename) 701 if ext not in ("", ".py"): 702 continue 703 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 704 705 # vim: tabstop=4 expandtab shiftwidth=4