1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.inspect 39 import micropython.table 40 import os 41 try: 42 set 43 except NameError: 44 from sets import Set as set 45 46 class Importer: 47 48 """ 49 An import machine, searching for and loading modules. In addition, this 50 class supports the generation of a program image. 51 """ 52 53 supported_optimisations = micropython.ast.Translation.supported_optimisations 54 55 def __init__(self, path=None, verbose=0): 56 57 """ 58 Initialise the importer with the given search 'path' - a list of 59 directories to search for Python modules. 60 61 The optional 'verbose' parameter causes output concerning the activities 62 of the object to be produced if set to a true value (not the default). 63 """ 64 65 self.path = path or [os.getcwd()] 66 self.verbose = verbose 67 self.modules = {} 68 self.loading = set() 69 70 # Remember the tables once generated. 71 72 self.objtable = None 73 self.paramtable = None 74 75 # Constant records. 76 77 self.constant_values = {} 78 self.constant_list = None # cache for constants 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def constants(self): 86 87 "Return a list of constants." 88 89 if self.constant_list is None: 90 self.constant_list = list(self.constant_values.values()) 91 92 return self.constant_list 93 94 def vacuum(self): 95 96 "Tidy up the modules." 97 98 for name, module in self.modules.items(): 99 if module.loaded: 100 module.vacuum() 101 else: 102 del self.modules[name] 103 104 def get_modules(self): 105 106 "Return all modules known to the importer." 107 108 return self.modules.values() 109 110 def get_image(self, with_builtins=0, optimisations=None): 111 112 "Return a dictionary mapping modules to structures." 113 114 objtable = self.get_object_table() 115 paramtable = self.get_parameter_table() 116 117 image = [] 118 119 # Append constants to the image. 120 121 for pos, const in enumerate(self.constants()): 122 const.location = pos 123 image.append(const) 124 125 for module_name, module in self.modules.items(): 126 if not with_builtins and module_name == "__builtins__": 127 continue 128 129 # Fix the attributes. 130 131 module.finalise_attributes() 132 133 pos = len(image) 134 135 # Position the module in the image and make a translation. 136 137 module.location = pos 138 trans = micropython.ast.Translation(module, self, optimisations) 139 140 # Add header details. 141 142 image.append(module) 143 pos += 1 144 145 # Append module attributes to the image. 146 147 attributes = module.module_attributes() 148 image += module.attributes_as_list() 149 pos += len(attributes.keys()) 150 151 # Append classes and functions to the image. 152 153 for obj in module.all_objects: 154 if isinstance(obj, micropython.inspect.Class): 155 156 # Fix the attributes. 157 158 obj.finalise_attributes() 159 160 # Position the class in the image. 161 162 obj.location = pos 163 164 # Add header details. 165 166 image.append(obj) 167 pos += 1 168 169 # Append class attributes to the image. 170 171 attributes = obj.class_attributes() 172 image += obj.attributes_as_list() 173 pos += len(attributes.keys()) 174 175 # Class-level code is generated separately. 176 # The code location is set within the code generation 177 # process for the module. 178 179 # NOTE: Generate module and function code here. 180 181 elif isinstance(obj, micropython.inspect.Function): 182 183 # Fix the attributes. 184 185 obj.finalise_attributes() 186 187 # Position the function in the image. 188 189 obj.location = pos 190 191 # Add header details. 192 193 image.append(obj) 194 pos += 1 195 196 # Append any default values to the image. 197 # Only do this for named functions (not lambdas). 198 199 if obj.name is not None: 200 image += obj.default_attrs 201 pos += len(obj.default_attrs) 202 203 # Append the function code to the image. 204 205 obj.code_location = pos 206 code = trans.get_code(obj) 207 image += code 208 pos += len(code) 209 210 # Remember the position of the module code. 211 212 module.code_location = pos 213 214 # Append the module top-level code to the image. 215 216 code = trans.get_module_code() 217 image += code 218 pos += len(code) 219 220 self.code = image 221 self.code_location = self.modules["__main__"].code_location 222 return image 223 224 def get_object_table(self): 225 226 "Return a table with details of attributes for classes and modules." 227 228 if self.objtable is None: 229 t = self.objtable = micropython.table.Table() 230 for module in self.get_modules(): 231 t.add(module.full_name(), module.module_attributes()) 232 for obj in module.all_objects: 233 if isinstance(obj, micropython.inspect.Class): 234 t.add(obj.full_name(), obj.all_attributes()) 235 236 return self.objtable 237 238 def get_parameter_table(self): 239 240 "Return a table with details of parameters for functions and methods." 241 242 # Need the object table to get at class details. 243 244 objtable = self.get_object_table() 245 246 if self.paramtable is None: 247 t = self.paramtable = micropython.table.Table() 248 249 # Visit each module, getting function and method details. 250 251 for module in self.get_modules(): 252 for obj in module.all_objects: 253 if isinstance(obj, micropython.inspect.Function): 254 t.add(obj.full_name(), obj.parameters()) 255 256 # Classes are callable, too. 257 # Take details of the appropriate __init__ method to make an 258 # entry for an instantiation function for the class. 259 260 elif isinstance(obj, micropython.inspect.Class): 261 t.add(obj.full_name(), obj.get_instantiator().parameters()) 262 263 # Filter out all parameter table entries not referenced by keyword 264 # arguments. 265 266 keyword_names = set() 267 268 for module in self.get_modules(): 269 keyword_names.update(module.keyword_names) 270 271 for function_name, parameters in t.table.items(): 272 for name in parameters.keys(): 273 if name in keyword_names: 274 break 275 else: 276 del t.table[function_name] 277 278 return self.paramtable 279 280 # Import methods. 281 282 def find_in_path(self, name): 283 284 """ 285 Find the given module 'name' in the search path, returning None where no 286 such module could be found, or a 2-tuple from the 'find' method 287 otherwise. 288 """ 289 290 for d in self.path: 291 m = self.find(d, name) 292 if m: return m 293 return None 294 295 def find(self, d, name): 296 297 """ 298 In the directory 'd', find the given module 'name', where 'name' can 299 either refer to a single file module or to a package. Return None if the 300 'name' cannot be associated with either a file or a package directory, 301 or a 2-tuple from '_find_package' or '_find_module' otherwise. 302 """ 303 304 m = self._find_package(d, name) 305 if m: return m 306 m = self._find_module(d, name) 307 if m: return m 308 return None 309 310 def _find_module(self, d, name): 311 312 """ 313 In the directory 'd', find the given module 'name', returning None where 314 no suitable file exists in the directory, or a 2-tuple consisting of 315 None (indicating that no package directory is involved) and a filename 316 indicating the location of the module. 317 """ 318 319 name_py = name + os.extsep + "py" 320 filename = self._find_file(d, name_py) 321 if filename: 322 return None, filename 323 return None 324 325 def _find_package(self, d, name): 326 327 """ 328 In the directory 'd', find the given package 'name', returning None 329 where no suitable package directory exists, or a 2-tuple consisting of 330 a directory (indicating the location of the package directory itself) 331 and a filename indicating the location of the __init__.py module which 332 declares the package's top-level contents. 333 """ 334 335 filename = self._find_file(d, name) 336 if filename: 337 init_py = "__init__" + os.path.extsep + "py" 338 init_py_filename = self._find_file(filename, init_py) 339 if init_py_filename: 340 return filename, init_py_filename 341 return None 342 343 def _find_file(self, d, filename): 344 345 """ 346 Return the filename obtained when searching the directory 'd' for the 347 given 'filename', or None if no actual file exists for the filename. 348 """ 349 350 filename = os.path.join(d, filename) 351 if os.path.exists(filename): 352 return filename 353 else: 354 return None 355 356 def load(self, name, return_leaf=0): 357 358 """ 359 Load the module or package with the given 'name'. Return an object 360 referencing the loaded module or package, or None if no such module or 361 package exists. 362 """ 363 364 if self.modules.has_key(name) and self.modules[name].loaded: 365 #print "Cached (%s)" % name 366 return self.modules[name] 367 if self.verbose: 368 print "Loading", name 369 370 # Split the name into path components, and try to find the uppermost in 371 # the search path. 372 373 path = name.split(".") 374 m = self.find_in_path(path[0]) 375 if not m: 376 if self.verbose: 377 print "Not found (%s)" % path[0] 378 return None # NOTE: Import error. 379 d, filename = m 380 381 # Either acquire a reference to an already-imported module, or load the 382 # module from a file. 383 384 top = module = self.load_from_file(filename, path[0]) 385 386 # For hierarchical names, traverse each path component... 387 388 if len(path) > 1: 389 if not d: 390 if self.verbose: 391 print "No package (%s)" % filename 392 return None # NOTE: Import error (package not found). 393 else: 394 self.add_submodules(d, module) 395 396 path_so_far = path[:1] 397 for p in path[1:]: 398 path_so_far.append(p) 399 400 # Find the package or module concerned. 401 402 m = self.find(d, p) 403 if not m: 404 if self.verbose: 405 print "Not found (%s)" % p 406 return None # NOTE: Import error. 407 d, filename = m 408 module_name = ".".join(path_so_far) 409 410 # Either reference an imported module or load one from a file. 411 412 submodule = self.load_from_file(filename, module_name) 413 414 if d: 415 self.add_submodules(d, module) 416 417 # Store the submodule within its parent module. 418 419 module.set_module(p, submodule) 420 module = submodule 421 422 # Return either the deepest or the uppermost module. 423 424 if return_leaf: 425 return module 426 else: 427 return top 428 429 def load_from_file(self, name, module_name=None): 430 431 """ 432 Load the module with the given 'name' (which may be a full module path). 433 """ 434 435 if module_name is None: 436 module_name = "__main__" 437 438 module = self.add_module(module_name) 439 if not module.loaded and module not in self.loading: 440 self.loading.add(module) 441 #print "Parsing", name 442 module.parse(name) 443 #print "Done", name 444 self.loading.remove(module) 445 module.loaded = 1 446 447 # Record the module. 448 449 #print "Loaded", module_name, "with namespace", module.namespace.keys() 450 return module 451 452 def add_module(self, module_name): 453 454 """ 455 Return the module with the given 'module_name', adding a new module 456 object if one does not already exist. 457 """ 458 459 if not self.modules.has_key(module_name): 460 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 461 else: 462 module = self.modules[module_name] 463 return module 464 465 def add_submodules(self, pathname, module): 466 467 """ 468 Work around insufficient __all__ declarations and examine the directory 469 with the given 'pathname', adding submodules to the given 'module'. 470 """ 471 472 for filename in os.listdir(pathname): 473 submodule, ext = os.path.splitext(filename) 474 if ext not in ("", ".py"): 475 continue 476 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 477 478 # vim: tabstop=4 expandtab shiftwidth=4