1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.opt 39 import micropython.inspect 40 import micropython.table 41 import os 42 try: 43 set 44 except NameError: 45 from sets import Set as set 46 47 class Importer: 48 49 """ 50 An import machine, searching for and loading modules. In addition, this 51 class supports the generation of a program image. 52 """ 53 54 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 55 56 def __init__(self, path=None, verbose=0): 57 58 """ 59 Initialise the importer with the given search 'path' - a list of 60 directories to search for Python modules. 61 62 The optional 'verbose' parameter causes output concerning the activities 63 of the object to be produced if set to a true value (not the default). 64 """ 65 66 self.path = path or [os.getcwd()] 67 self.verbose = verbose 68 self.modules = {} 69 self.modules_ordered = [] 70 self.loading = set() 71 72 # Remember the tables once generated. 73 74 self.objtable = None 75 self.paramtable = None 76 self.clstable = None 77 78 # Constant records. 79 80 self.constant_values = {} 81 self.constant_list = None # cache for constants 82 83 # Main program information. 84 85 self.code = None 86 self.code_location = None 87 88 def constants(self): 89 90 "Return a list of constants." 91 92 if self.constant_list is None: 93 self.constant_list = list(self.constant_values.values()) 94 95 return self.constant_list 96 97 def vacuum(self): 98 99 "Tidy up the modules." 100 101 for name, module in self.modules.items(): 102 if module.loaded: 103 module.vacuum() 104 else: 105 del self.modules[name] 106 107 def finalise(self): 108 109 "Finalise the program." 110 111 for module in self.get_modules(): 112 113 # Fix the attributes. 114 115 module.finalise_attributes() 116 117 for obj in module.all_objects: 118 if isinstance(obj, (micropython.inspect.Class, micropython.inspect.Function)): 119 obj.finalise_attributes() 120 121 def get_modules(self): 122 123 "Return all modules known to the importer." 124 125 return self.modules.values() 126 127 def get_image(self, with_builtins=0, optimisations=None): 128 129 "Return a dictionary mapping modules to structures." 130 131 if self.code is not None: 132 return self.code 133 134 objtable = self.get_object_table() 135 paramtable = self.get_parameter_table() 136 self.finalise() 137 138 image = [] 139 140 # Append constants to the image. 141 142 for pos, const in enumerate(self.constants()): 143 const.location = pos 144 image.append(const) 145 146 last_module = self.modules_ordered[-1] 147 148 for module in self.modules_ordered: 149 150 if not with_builtins and module.name == "__builtins__": 151 continue 152 153 pos = len(image) 154 155 # Position the module in the image and make a translation. 156 157 module.location = pos 158 trans = micropython.ast.Translation(module, self, optimisations) 159 160 # Add header details. 161 162 image.append(module) 163 pos += 1 164 165 # Append module attributes to the image. 166 167 attributes = module.module_attributes() 168 image += module.attributes_as_list() 169 pos += len(attributes.keys()) 170 171 # Append classes and functions to the image. 172 173 for obj in module.all_objects: 174 if isinstance(obj, micropython.inspect.Class): 175 176 # Position the class in the image. 177 178 obj.location = pos 179 180 # Add header details. 181 182 image.append(obj) 183 pos += 1 184 185 # Append class attributes to the image. 186 187 attributes = obj.class_attributes() 188 image += obj.attributes_as_list() 189 pos += len(attributes.keys()) 190 191 # Generate the instantiator/initialiser. 192 # Append the function code to the image. 193 194 instantiator = obj.get_instantiator() 195 instantiator.code_location = pos 196 code = trans.get_instantiator_code(obj) 197 image += code 198 pos += len(code) 199 200 # Class-level code is generated separately at the module 201 # level, and the code location is set within the code 202 # generation process for the module. 203 204 elif isinstance(obj, micropython.inspect.Function): 205 206 # Position the function in the image. 207 208 obj.location = pos 209 210 # Add header details. 211 212 image.append(obj) 213 pos += 1 214 215 # Append any default values to the image. 216 # Only do this for named functions (not lambdas). 217 218 if obj.name is not None: 219 image += obj.default_attrs 220 pos += len(obj.default_attrs) 221 222 # Append the function code to the image. 223 224 obj.code_location = pos 225 code = trans.get_code(obj) 226 image += code 227 pos += len(code) 228 229 # Remember the position of the module code. 230 231 module.code_location = pos 232 233 # Append the module top-level code to the image. 234 235 code = trans.get_module_code(final=(module is last_module)) 236 image += code 237 pos += len(code) 238 239 # Remember the generated code and the location of the first instruction. 240 241 self.code = image 242 self.code_location = self.modules["__main__"].code_location 243 return image 244 245 def get_object_table(self): 246 247 "Return a table with details of attributes for classes and modules." 248 249 if self.objtable is None: 250 t = self.objtable = micropython.table.ObjectTable() 251 for module in self.get_modules(): 252 t.add(module.full_name(), module.module_attributes()) 253 254 # Add class and instance attributes for all classes, together 255 # with descendant information. 256 257 for obj in module.all_objects: 258 if isinstance(obj, micropython.inspect.Class): 259 attributes = {obj.full_name() : obj} 260 attributes.update(obj.all_attributes()) 261 attributes.update(obj.all_descendants()) 262 t.add(obj.full_name(), attributes) 263 264 return self.objtable 265 266 def get_parameter_table(self): 267 268 "Return a table with details of parameters for functions and methods." 269 270 # Need the object table to get at class details. 271 272 objtable = self.get_object_table() 273 274 if self.paramtable is None: 275 t = self.paramtable = micropython.table.ParameterTable() 276 277 # Visit each module, getting function and method details. 278 279 for module in self.get_modules(): 280 for obj in module.all_objects: 281 if isinstance(obj, micropython.inspect.Function): 282 t.add(obj.full_name(), obj.parameters()) 283 284 # Classes are callable, too. 285 # Take details of the appropriate __init__ method to make an 286 # entry for an instantiation function for the class. 287 288 elif isinstance(obj, micropython.inspect.Class): 289 t.add(obj.full_name(), obj.get_instantiator().parameters()) 290 291 # Filter out all parameter table entries not referenced by keyword 292 # arguments. 293 294 keyword_names = set() 295 296 for module in self.get_modules(): 297 keyword_names.update(module.keyword_names) 298 299 for function_name, parameters in t.table.items(): 300 for name in parameters.keys(): 301 if name in keyword_names: 302 break 303 else: 304 del t.table[function_name] 305 306 return self.paramtable 307 308 # Import methods. 309 310 def find_in_path(self, name): 311 312 """ 313 Find the given module 'name' in the search path, returning None where no 314 such module could be found, or a 2-tuple from the 'find' method 315 otherwise. 316 """ 317 318 for d in self.path: 319 m = self.find(d, name) 320 if m: return m 321 return None 322 323 def find(self, d, name): 324 325 """ 326 In the directory 'd', find the given module 'name', where 'name' can 327 either refer to a single file module or to a package. Return None if the 328 'name' cannot be associated with either a file or a package directory, 329 or a 2-tuple from '_find_package' or '_find_module' otherwise. 330 """ 331 332 m = self._find_package(d, name) 333 if m: return m 334 m = self._find_module(d, name) 335 if m: return m 336 return None 337 338 def _find_module(self, d, name): 339 340 """ 341 In the directory 'd', find the given module 'name', returning None where 342 no suitable file exists in the directory, or a 2-tuple consisting of 343 None (indicating that no package directory is involved) and a filename 344 indicating the location of the module. 345 """ 346 347 name_py = name + os.extsep + "py" 348 filename = self._find_file(d, name_py) 349 if filename: 350 return None, filename 351 return None 352 353 def _find_package(self, d, name): 354 355 """ 356 In the directory 'd', find the given package 'name', returning None 357 where no suitable package directory exists, or a 2-tuple consisting of 358 a directory (indicating the location of the package directory itself) 359 and a filename indicating the location of the __init__.py module which 360 declares the package's top-level contents. 361 """ 362 363 filename = self._find_file(d, name) 364 if filename: 365 init_py = "__init__" + os.path.extsep + "py" 366 init_py_filename = self._find_file(filename, init_py) 367 if init_py_filename: 368 return filename, init_py_filename 369 return None 370 371 def _find_file(self, d, filename): 372 373 """ 374 Return the filename obtained when searching the directory 'd' for the 375 given 'filename', or None if no actual file exists for the filename. 376 """ 377 378 filename = os.path.join(d, filename) 379 if os.path.exists(filename): 380 return filename 381 else: 382 return None 383 384 def load(self, name, return_leaf=0): 385 386 """ 387 Load the module or package with the given 'name'. Return an object 388 referencing the loaded module or package, or None if no such module or 389 package exists. 390 """ 391 392 if self.modules.has_key(name) and self.modules[name].loaded: 393 #print "Cached (%s)" % name 394 return self.modules[name] 395 if self.verbose: 396 print "Loading", name 397 398 # Split the name into path components, and try to find the uppermost in 399 # the search path. 400 401 path = name.split(".") 402 m = self.find_in_path(path[0]) 403 if not m: 404 if self.verbose: 405 print "Not found (%s)" % path[0] 406 return None # NOTE: Import error. 407 d, filename = m 408 409 # Either acquire a reference to an already-imported module, or load the 410 # module from a file. 411 412 top = module = self.load_from_file(filename, path[0]) 413 414 # For hierarchical names, traverse each path component... 415 416 if len(path) > 1: 417 if not d: 418 if self.verbose: 419 print "No package (%s)" % filename 420 return None # NOTE: Import error (package not found). 421 else: 422 self.add_submodules(d, module) 423 424 path_so_far = path[:1] 425 for p in path[1:]: 426 path_so_far.append(p) 427 428 # Find the package or module concerned. 429 430 m = self.find(d, p) 431 if not m: 432 if self.verbose: 433 print "Not found (%s)" % p 434 return None # NOTE: Import error. 435 d, filename = m 436 module_name = ".".join(path_so_far) 437 438 # Either reference an imported module or load one from a file. 439 440 submodule = self.load_from_file(filename, module_name) 441 442 if d: 443 self.add_submodules(d, module) 444 445 # Store the submodule within its parent module. 446 447 module.set_module(p, submodule) 448 module = submodule 449 450 # Return either the deepest or the uppermost module. 451 452 if return_leaf: 453 return module 454 else: 455 return top 456 457 def load_from_file(self, name, module_name=None): 458 459 """ 460 Load the module with the given 'name' (which may be a full module path). 461 """ 462 463 if module_name is None: 464 module_name = "__main__" 465 466 module = self.add_module(module_name) 467 if not module.loaded and module not in self.loading: 468 self.loading.add(module) 469 #print "Parsing", name 470 module.parse(name) 471 #print "Done", name 472 self.loading.remove(module) 473 module.loaded = 1 474 475 # Record the module. 476 477 #print "Loaded", module_name, "with namespace", module.namespace.keys() 478 return module 479 480 def add_module(self, module_name): 481 482 """ 483 Return the module with the given 'module_name', adding a new module 484 object if one does not already exist. 485 """ 486 487 if not self.modules.has_key(module_name): 488 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 489 self.modules_ordered.append(module) 490 else: 491 module = self.modules[module_name] 492 return module 493 494 def add_submodules(self, pathname, module): 495 496 """ 497 Work around insufficient __all__ declarations and examine the directory 498 with the given 'pathname', adding submodules to the given 'module'. 499 """ 500 501 for filename in os.listdir(pathname): 502 submodule, ext = os.path.splitext(filename) 503 if ext not in ("", ".py"): 504 continue 505 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 506 507 # vim: tabstop=4 expandtab shiftwidth=4