1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.data 39 import micropython.opt 40 import micropython.inspect 41 import micropython.table 42 import os 43 try: 44 set 45 except NameError: 46 from sets import Set as set 47 48 class Importer: 49 50 """ 51 An import machine, searching for and loading modules. In addition, this 52 class supports the generation of a program image. 53 """ 54 55 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 56 57 predefined_constants = { 58 "None" : None, 59 "True" : True, 60 "False" : False, 61 "Ellipsis" : Ellipsis, 62 "NotImplemented" : NotImplemented 63 } 64 65 def __init__(self, path=None, optimisations=None, verbose=0): 66 67 """ 68 Initialise the importer with the given search 'path' - a list of 69 directories to search for Python modules. 70 71 The optional 'optimisations' cause certain techniques to be used in 72 reducing program size and improving program efficiency. 73 74 The optional 'verbose' parameter causes output concerning the activities 75 of the object to be produced if set to a true value (not the default). 76 """ 77 78 self.path = path or [os.getcwd()] 79 self.optimisations = optimisations 80 self.verbose = verbose 81 self.modules = {} 82 self.modules_ordered = [] 83 self.loading = set() 84 85 # Remember the tables once generated. 86 87 self.objtable = None 88 self.paramtable = None 89 self.clstable = None 90 91 # Constant records. 92 93 self.constant_values = {} 94 self.constant_list = None # cache for constants 95 self.init_predefined_constants() 96 97 # Name records (used to track actual use of names). 98 99 self.names_used = set() 100 101 # Main program information. 102 103 self.code = None 104 self.code_location = None 105 106 # Name accounting. 107 108 def use_name(self, name): 109 110 "Register the given 'name' as being used in the program." 111 112 self.names_used.add(name) 113 114 # Constant accounting. 115 116 def init_predefined_constants(self): 117 118 "Ensure the predefined constants." 119 120 for name, value in self.predefined_constants.items(): 121 self.make_constant(value) 122 123 def get_predefined_constant(self, name): 124 125 "Return the predefined constant for the given 'name'." 126 127 return self.make_constant(self.predefined_constants[name]) 128 129 def get_constant(self, value): 130 131 "Return a constant for the given 'value'." 132 133 return self.constant_values[value] 134 135 def make_constant(self, value): 136 137 "Make and return a constant for the given 'value'." 138 139 if not self.constant_values.has_key(value): 140 const = micropython.data.Const(value) 141 self.constant_values[value] = const 142 return self.constant_values[value] 143 144 def constants(self): 145 146 "Return a list of constants." 147 148 if self.constant_list is None: 149 self.constant_list = list(self.constant_values.values()) 150 151 return self.constant_list 152 153 # General maintenance. 154 155 def vacuum(self): 156 157 "Tidy up the modules." 158 159 for name, module in self.modules.items(): 160 if module.loaded: 161 module.vacuum() 162 else: 163 del self.modules[name] 164 165 def finalise(self): 166 167 "Finalise the program." 168 169 for module in self.get_modules(): 170 171 # Fix the attributes. 172 173 module.finalise_attributes() 174 175 for obj in module.all_objects: 176 if isinstance(obj, (micropython.inspect.Class, micropython.inspect.Function)): 177 obj.finalise_attributes() 178 179 # Access to finalised program information. 180 181 def get_modules(self): 182 183 "Return all modules known to the importer." 184 185 return self.modules.values() 186 187 def get_image(self, with_builtins=0): 188 189 "Return a dictionary mapping modules to structures." 190 191 if self.code is not None: 192 return self.code 193 194 objtable = self.get_object_table() 195 paramtable = self.get_parameter_table() 196 self.finalise() 197 198 image = [] 199 200 # Append constants to the image. 201 202 for pos, const in enumerate(self.constants()): 203 const.location = pos 204 image.append(const) 205 206 last_module = self.modules_ordered[-1] 207 208 for module in self.modules_ordered: 209 210 if not with_builtins and module.name == "__builtins__": 211 continue 212 213 pos = len(image) 214 215 # Position the module in the image and make a translation. 216 217 module.location = pos 218 trans = micropython.ast.Translation(module, self) 219 220 # Add header details. 221 222 image.append(module) 223 pos += 1 224 225 # Append module attributes to the image. 226 227 attributes = module.module_attributes() 228 image += module.attributes_as_list() 229 pos += len(attributes.keys()) 230 231 # Append classes and functions to the image. 232 233 for obj in module.all_objects: 234 if isinstance(obj, micropython.inspect.Class): 235 236 # Position the class in the image. 237 238 obj.location = pos 239 240 # Add header details. 241 242 image.append(obj) 243 pos += 1 244 245 # Append class attributes to the image. 246 247 attributes = obj.class_attributes() 248 image += obj.attributes_as_list() 249 pos += len(attributes.keys()) 250 251 # Generate the instantiator/initialiser. 252 # Append the function code to the image. 253 254 instantiator = obj.get_instantiator() 255 instantiator.code_location = pos 256 code = trans.get_instantiator_code(obj) 257 image += code 258 pos += len(code) 259 260 # Class-level code is generated separately at the module 261 # level, and the code location is set within the code 262 # generation process for the module. 263 264 elif isinstance(obj, micropython.inspect.Function): 265 266 # Position the function in the image. 267 268 obj.location = pos 269 270 # Add header details. 271 272 image.append(obj) 273 pos += 1 274 275 # Append any default values to the image. 276 # Only do this for named functions (not lambdas). 277 278 if obj.name is not None: 279 image += obj.default_attrs 280 pos += len(obj.default_attrs) 281 282 # Append the function code to the image. 283 284 obj.code_location = pos 285 code = trans.get_code(obj) 286 image += code 287 pos += len(code) 288 289 # Remember the position of the module code. 290 291 module.code_location = pos 292 293 # Append the module top-level code to the image. 294 295 code = trans.get_module_code(final=(module is last_module)) 296 image += code 297 pos += len(code) 298 299 # Remember the generated code and the location of the first instruction. 300 301 self.code = image 302 self.code_location = self.modules["__main__"].code_location 303 return image 304 305 def get_object_table(self): 306 307 "Return a table with details of attributes for classes and modules." 308 309 if self.objtable is None: 310 t = self.objtable = micropython.table.ObjectTable() 311 for module in self.get_modules(): 312 t.add(module.full_name(), module.module_attributes()) 313 314 # Add class and instance attributes for all classes, together 315 # with descendant information. 316 317 for obj in module.all_objects: 318 if isinstance(obj, micropython.inspect.Class): 319 attributes = {obj.full_name() : obj} 320 attributes.update(obj.all_attributes()) 321 attributes.update(obj.all_descendants()) 322 t.add(obj.full_name(), attributes) 323 324 return self.objtable 325 326 def get_parameter_table(self): 327 328 "Return a table with details of parameters for functions and methods." 329 330 # Need the object table to get at class details. 331 332 objtable = self.get_object_table() 333 334 if self.paramtable is None: 335 t = self.paramtable = micropython.table.ParameterTable() 336 337 # Visit each module, getting function and method details. 338 339 for module in self.get_modules(): 340 for obj in module.all_objects: 341 if isinstance(obj, micropython.inspect.Function): 342 t.add(obj.full_name(), obj.parameters()) 343 344 # Classes are callable, too. 345 # Take details of the appropriate __init__ method to make an 346 # entry for an instantiation function for the class. 347 348 elif isinstance(obj, micropython.inspect.Class): 349 t.add(obj.full_name(), obj.get_instantiator().parameters()) 350 351 # Filter out all parameter table entries not referenced by keyword 352 # arguments. 353 354 keyword_names = set() 355 356 for module in self.get_modules(): 357 keyword_names.update(module.keyword_names) 358 359 for function_name, parameters in t.table.items(): 360 for name in parameters.keys(): 361 if name in keyword_names: 362 break 363 else: 364 del t.table[function_name] 365 366 return self.paramtable 367 368 # Import methods. 369 370 def find_in_path(self, name): 371 372 """ 373 Find the given module 'name' in the search path, returning None where no 374 such module could be found, or a 2-tuple from the 'find' method 375 otherwise. 376 """ 377 378 for d in self.path: 379 m = self.find(d, name) 380 if m: return m 381 return None 382 383 def find(self, d, name): 384 385 """ 386 In the directory 'd', find the given module 'name', where 'name' can 387 either refer to a single file module or to a package. Return None if the 388 'name' cannot be associated with either a file or a package directory, 389 or a 2-tuple from '_find_package' or '_find_module' otherwise. 390 """ 391 392 m = self._find_package(d, name) 393 if m: return m 394 m = self._find_module(d, name) 395 if m: return m 396 return None 397 398 def _find_module(self, d, name): 399 400 """ 401 In the directory 'd', find the given module 'name', returning None where 402 no suitable file exists in the directory, or a 2-tuple consisting of 403 None (indicating that no package directory is involved) and a filename 404 indicating the location of the module. 405 """ 406 407 name_py = name + os.extsep + "py" 408 filename = self._find_file(d, name_py) 409 if filename: 410 return None, filename 411 return None 412 413 def _find_package(self, d, name): 414 415 """ 416 In the directory 'd', find the given package 'name', returning None 417 where no suitable package directory exists, or a 2-tuple consisting of 418 a directory (indicating the location of the package directory itself) 419 and a filename indicating the location of the __init__.py module which 420 declares the package's top-level contents. 421 """ 422 423 filename = self._find_file(d, name) 424 if filename: 425 init_py = "__init__" + os.path.extsep + "py" 426 init_py_filename = self._find_file(filename, init_py) 427 if init_py_filename: 428 return filename, init_py_filename 429 return None 430 431 def _find_file(self, d, filename): 432 433 """ 434 Return the filename obtained when searching the directory 'd' for the 435 given 'filename', or None if no actual file exists for the filename. 436 """ 437 438 filename = os.path.join(d, filename) 439 if os.path.exists(filename): 440 return filename 441 else: 442 return None 443 444 def load(self, name, return_leaf=0): 445 446 """ 447 Load the module or package with the given 'name'. Return an object 448 referencing the loaded module or package, or None if no such module or 449 package exists. 450 """ 451 452 if self.modules.has_key(name) and self.modules[name].loaded: 453 #print "Cached (%s)" % name 454 return self.modules[name] 455 if self.verbose: 456 print "Loading", name 457 458 # Split the name into path components, and try to find the uppermost in 459 # the search path. 460 461 path = name.split(".") 462 m = self.find_in_path(path[0]) 463 if not m: 464 if self.verbose: 465 print "Not found (%s)" % path[0] 466 return None # NOTE: Import error. 467 d, filename = m 468 469 # Either acquire a reference to an already-imported module, or load the 470 # module from a file. 471 472 top = module = self.load_from_file(filename, path[0]) 473 474 # For hierarchical names, traverse each path component... 475 476 if len(path) > 1: 477 if not d: 478 if self.verbose: 479 print "No package (%s)" % filename 480 return None # NOTE: Import error (package not found). 481 else: 482 self.add_submodules(d, module) 483 484 path_so_far = path[:1] 485 for p in path[1:]: 486 path_so_far.append(p) 487 488 # Find the package or module concerned. 489 490 m = self.find(d, p) 491 if not m: 492 if self.verbose: 493 print "Not found (%s)" % p 494 return None # NOTE: Import error. 495 d, filename = m 496 module_name = ".".join(path_so_far) 497 498 # Either reference an imported module or load one from a file. 499 500 submodule = self.load_from_file(filename, module_name) 501 502 if d: 503 self.add_submodules(d, module) 504 505 # Store the submodule within its parent module. 506 507 module.set_module(p, submodule) 508 module = submodule 509 510 # Return either the deepest or the uppermost module. 511 512 if return_leaf: 513 return module 514 else: 515 return top 516 517 def load_from_file(self, name, module_name=None): 518 519 """ 520 Load the module with the given 'name' (which may be a full module path). 521 """ 522 523 if module_name is None: 524 module_name = "__main__" 525 526 module = self.add_module(module_name) 527 if not module.loaded and module not in self.loading: 528 self.loading.add(module) 529 #print "Parsing", name 530 module.parse(name) 531 #print "Done", name 532 self.loading.remove(module) 533 module.loaded = 1 534 535 # Record the module. 536 537 #print "Loaded", module_name, "with namespace", module.namespace.keys() 538 return module 539 540 def add_module(self, module_name): 541 542 """ 543 Return the module with the given 'module_name', adding a new module 544 object if one does not already exist. 545 """ 546 547 if not self.modules.has_key(module_name): 548 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 549 self.modules_ordered.append(module) 550 else: 551 module = self.modules[module_name] 552 return module 553 554 def add_submodules(self, pathname, module): 555 556 """ 557 Work around insufficient __all__ declarations and examine the directory 558 with the given 'pathname', adding submodules to the given 'module'. 559 """ 560 561 for filename in os.listdir(pathname): 562 submodule, ext = os.path.splitext(filename) 563 if ext not in ("", ".py"): 564 continue 565 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 566 567 # vim: tabstop=4 expandtab shiftwidth=4