1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.data 39 import micropython.opt 40 import micropython.inspect 41 import micropython.table 42 import os 43 try: 44 set 45 except NameError: 46 from sets import Set as set 47 48 class Importer: 49 50 """ 51 An import machine, searching for and loading modules. In addition, this 52 class supports the generation of a program image. 53 """ 54 55 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 56 57 predefined_constants = { 58 "None" : None, 59 "True" : True, 60 "False" : False, 61 "Ellipsis" : Ellipsis, 62 "NotImplemented" : NotImplemented 63 } 64 65 def __init__(self, path=None, optimisations=None, verbose=0): 66 67 """ 68 Initialise the importer with the given search 'path' - a list of 69 directories to search for Python modules. 70 71 The optional 'optimisations' cause certain techniques to be used in 72 reducing program size and improving program efficiency. 73 74 The optional 'verbose' parameter causes output concerning the activities 75 of the object to be produced if set to a true value (not the default). 76 """ 77 78 self.path = path or [os.getcwd()] 79 self.optimisations = optimisations 80 self.verbose = verbose 81 self.modules = {} 82 self.modules_ordered = [] 83 self.loading = set() 84 85 # Remember the tables once generated. 86 87 self.objtable = None 88 self.paramtable = None 89 self.clstable = None 90 91 # Constant records. 92 93 self.constant_values = {} 94 self.constant_list = None # cache for constants 95 self.init_predefined_constants() 96 97 # Name records (used to track actual use of names). 98 # Include names which may not be explicitly used in programs. 99 100 self.names_used = set(["__init__"]) 101 102 # Main program information. 103 104 self.code = None 105 self.code_location = None 106 107 # Status information. 108 109 self.vacuumed = 0 110 self.finalised = 0 111 112 # Name accounting. 113 114 def use_name(self, name): 115 116 "Register the given 'name' as being used in the program." 117 118 self.names_used.add(name) 119 120 # Constant accounting. 121 122 def init_predefined_constants(self): 123 124 "Ensure the predefined constants." 125 126 for name, value in self.predefined_constants.items(): 127 self.make_constant(value) 128 129 def get_predefined_constant(self, name): 130 131 "Return the predefined constant for the given 'name'." 132 133 return self.make_constant(self.predefined_constants[name]) 134 135 def get_constant(self, value): 136 137 "Return a constant for the given 'value'." 138 139 return self.constant_values[value] 140 141 def make_constant(self, value): 142 143 "Make and return a constant for the given 'value'." 144 145 if not self.constant_values.has_key(value): 146 const = micropython.data.Const(value) 147 self.constant_values[value] = const 148 return self.constant_values[value] 149 150 def constants(self): 151 152 "Return a list of constants." 153 154 if self.constant_list is None: 155 self.constant_list = list(self.constant_values.values()) 156 157 return self.constant_list 158 159 # General maintenance. 160 161 def vacuum(self): 162 163 "Tidy up the modules." 164 165 if self.vacuumed: 166 return 167 168 for name, module in self.modules.items(): 169 if module.loaded: 170 module.vacuum() 171 else: 172 del self.modules[name] 173 174 self.vacuumed = 1 175 176 def finalise(self): 177 178 "Finalise the program." 179 180 if self.finalised: 181 return 182 183 self.vacuum() 184 185 for module in self.get_modules(): 186 module.finalise() 187 188 self.finalised = 1 189 190 # Access to finalised program information. 191 192 def get_modules(self): 193 194 "Return all modules known to the importer." 195 196 return self.modules.values() 197 198 def get_image(self, with_builtins=0): 199 200 "Return a dictionary mapping modules to structures." 201 202 if self.code is not None: 203 return self.code 204 205 objtable = self.get_object_table() 206 paramtable = self.get_parameter_table() 207 self.finalise() 208 209 image = [] 210 211 # Append constants to the image. 212 213 for pos, const in enumerate(self.constants()): 214 const.location = pos 215 image.append(const) 216 217 last_module = self.modules_ordered[-1] 218 219 for module in self.modules_ordered: 220 221 if not with_builtins and module.name == "__builtins__": 222 continue 223 224 pos = len(image) 225 226 # Position the module in the image and make a translation. 227 228 module.location = pos 229 trans = micropython.ast.Translation(module, self) 230 231 # Add header details. 232 233 image.append(module) 234 pos += 1 235 236 # Append module attributes to the image. 237 238 attributes = module.module_attributes() 239 image += module.attributes_as_list() 240 pos += len(attributes.keys()) 241 242 # Append classes and functions to the image. 243 244 for obj in module.all_objects: 245 if isinstance(obj, micropython.inspect.Class): 246 247 # Position the class in the image. 248 249 obj.location = pos 250 251 # Add header details. 252 253 image.append(obj) 254 pos += 1 255 256 # Append class attributes to the image. 257 258 attributes = obj.class_attributes() 259 image += obj.attributes_as_list() 260 pos += len(attributes.keys()) 261 262 # Generate the instantiator/initialiser. 263 # Append the function code to the image. 264 265 instantiator = obj.get_instantiator() 266 instantiator.code_location = pos 267 code = trans.get_instantiator_code(obj) 268 image += code 269 pos += len(code) 270 271 # Class-level code is generated separately at the module 272 # level, and the code location is set within the code 273 # generation process for the module. 274 275 elif isinstance(obj, micropython.inspect.Function): 276 277 # Position the function in the image. 278 279 obj.location = pos 280 281 # Add header details. 282 283 image.append(obj) 284 pos += 1 285 286 # Append any default values to the image. 287 # Only do this for named functions (not lambdas). 288 289 if obj.name is not None: 290 image += obj.default_attrs 291 pos += len(obj.default_attrs) 292 293 # Append the function code to the image. 294 295 obj.code_location = pos 296 code = trans.get_code(obj) 297 image += code 298 pos += len(code) 299 300 # Remember the position of the module code. 301 302 module.code_location = pos 303 304 # Append the module top-level code to the image. 305 306 code = trans.get_module_code(final=(module is last_module)) 307 image += code 308 pos += len(code) 309 310 # Remember the generated code and the location of the first instruction. 311 312 self.code = image 313 self.code_location = self.modules["__main__"].code_location 314 return image 315 316 def get_object_table(self): 317 318 "Return a table with details of attributes for classes and modules." 319 320 if self.objtable is None: 321 self.vacuum() 322 323 t = self.objtable = micropython.table.ObjectTable() 324 for module in self.get_modules(): 325 t.add(module.full_name(), module.module_attributes()) 326 327 # Add class and instance attributes for all classes, together 328 # with descendant information. 329 330 for obj in module.all_objects: 331 if isinstance(obj, micropython.inspect.Class): 332 attributes = {obj.full_name() : obj} 333 attributes.update(obj.all_attributes()) 334 attributes.update(obj.all_descendants()) 335 t.add(obj.full_name(), attributes) 336 337 return self.objtable 338 339 def get_parameter_table(self): 340 341 "Return a table with details of parameters for functions and methods." 342 343 # Need the object table to get at class details. 344 345 objtable = self.get_object_table() 346 347 if self.paramtable is None: 348 t = self.paramtable = micropython.table.ParameterTable() 349 350 # Visit each module, getting function and method details. 351 352 for module in self.get_modules(): 353 for obj in module.all_objects: 354 if isinstance(obj, micropython.inspect.Function): 355 t.add(obj.full_name(), obj.parameters()) 356 357 # Classes are callable, too. 358 # Take details of the appropriate __init__ method to make an 359 # entry for an instantiation function for the class. 360 361 elif isinstance(obj, micropython.inspect.Class): 362 t.add(obj.full_name(), obj.get_instantiator().parameters()) 363 364 # Filter out all parameter table entries not referenced by keyword 365 # arguments. 366 367 keyword_names = set() 368 369 for module in self.get_modules(): 370 keyword_names.update(module.keyword_names) 371 372 for function_name, parameters in t.table.items(): 373 for name in parameters.keys(): 374 if name in keyword_names: 375 break 376 else: 377 del t.table[function_name] 378 379 return self.paramtable 380 381 # Import methods. 382 383 def find_in_path(self, name): 384 385 """ 386 Find the given module 'name' in the search path, returning None where no 387 such module could be found, or a 2-tuple from the 'find' method 388 otherwise. 389 """ 390 391 for d in self.path: 392 m = self.find(d, name) 393 if m: return m 394 return None 395 396 def find(self, d, name): 397 398 """ 399 In the directory 'd', find the given module 'name', where 'name' can 400 either refer to a single file module or to a package. Return None if the 401 'name' cannot be associated with either a file or a package directory, 402 or a 2-tuple from '_find_package' or '_find_module' otherwise. 403 """ 404 405 m = self._find_package(d, name) 406 if m: return m 407 m = self._find_module(d, name) 408 if m: return m 409 return None 410 411 def _find_module(self, d, name): 412 413 """ 414 In the directory 'd', find the given module 'name', returning None where 415 no suitable file exists in the directory, or a 2-tuple consisting of 416 None (indicating that no package directory is involved) and a filename 417 indicating the location of the module. 418 """ 419 420 name_py = name + os.extsep + "py" 421 filename = self._find_file(d, name_py) 422 if filename: 423 return None, filename 424 return None 425 426 def _find_package(self, d, name): 427 428 """ 429 In the directory 'd', find the given package 'name', returning None 430 where no suitable package directory exists, or a 2-tuple consisting of 431 a directory (indicating the location of the package directory itself) 432 and a filename indicating the location of the __init__.py module which 433 declares the package's top-level contents. 434 """ 435 436 filename = self._find_file(d, name) 437 if filename: 438 init_py = "__init__" + os.path.extsep + "py" 439 init_py_filename = self._find_file(filename, init_py) 440 if init_py_filename: 441 return filename, init_py_filename 442 return None 443 444 def _find_file(self, d, filename): 445 446 """ 447 Return the filename obtained when searching the directory 'd' for the 448 given 'filename', or None if no actual file exists for the filename. 449 """ 450 451 filename = os.path.join(d, filename) 452 if os.path.exists(filename): 453 return filename 454 else: 455 return None 456 457 def load(self, name, return_leaf=0): 458 459 """ 460 Load the module or package with the given 'name'. Return an object 461 referencing the loaded module or package, or None if no such module or 462 package exists. 463 """ 464 465 if self.modules.has_key(name) and self.modules[name].loaded: 466 #print "Cached (%s)" % name 467 return self.modules[name] 468 if self.verbose: 469 print "Loading", name 470 471 # Split the name into path components, and try to find the uppermost in 472 # the search path. 473 474 path = name.split(".") 475 m = self.find_in_path(path[0]) 476 if not m: 477 if self.verbose: 478 print "Not found (%s)" % path[0] 479 return None # NOTE: Import error. 480 d, filename = m 481 482 # Either acquire a reference to an already-imported module, or load the 483 # module from a file. 484 485 top = module = self.load_from_file(filename, path[0]) 486 487 # For hierarchical names, traverse each path component... 488 489 if len(path) > 1: 490 if not d: 491 if self.verbose: 492 print "No package (%s)" % filename 493 return None # NOTE: Import error (package not found). 494 else: 495 self.add_submodules(d, module) 496 497 path_so_far = path[:1] 498 for p in path[1:]: 499 path_so_far.append(p) 500 501 # Find the package or module concerned. 502 503 m = self.find(d, p) 504 if not m: 505 if self.verbose: 506 print "Not found (%s)" % p 507 return None # NOTE: Import error. 508 d, filename = m 509 module_name = ".".join(path_so_far) 510 511 # Either reference an imported module or load one from a file. 512 513 submodule = self.load_from_file(filename, module_name) 514 515 if d: 516 self.add_submodules(d, module) 517 518 # Store the submodule within its parent module. 519 520 module.set_module(p, submodule) 521 module = submodule 522 523 # Return either the deepest or the uppermost module. 524 525 if return_leaf: 526 return module 527 else: 528 return top 529 530 def load_from_file(self, name, module_name=None): 531 532 """ 533 Load the module with the given 'name' (which may be a full module path). 534 """ 535 536 if module_name is None: 537 module_name = "__main__" 538 539 module = self.add_module(module_name) 540 if not module.loaded and module not in self.loading: 541 self.loading.add(module) 542 #print "Parsing", name 543 module.parse(name) 544 #print "Done", name 545 self.loading.remove(module) 546 module.loaded = 1 547 548 # Record the module. 549 550 #print "Loaded", module_name, "with namespace", module.namespace.keys() 551 return module 552 553 def add_module(self, module_name): 554 555 """ 556 Return the module with the given 'module_name', adding a new module 557 object if one does not already exist. 558 """ 559 560 if not self.modules.has_key(module_name): 561 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 562 self.modules_ordered.append(module) 563 else: 564 module = self.modules[module_name] 565 return module 566 567 def add_submodules(self, pathname, module): 568 569 """ 570 Work around insufficient __all__ declarations and examine the directory 571 with the given 'pathname', adding submodules to the given 'module'. 572 """ 573 574 for filename in os.listdir(pathname): 575 submodule, ext = os.path.splitext(filename) 576 if ext not in ("", ".py"): 577 continue 578 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 579 580 # vim: tabstop=4 expandtab shiftwidth=4