1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.inspect 39 import micropython.table 40 import os 41 try: 42 set 43 except NameError: 44 from sets import Set as set 45 46 class Importer: 47 48 """ 49 An import machine, searching for and loading modules. In addition, this 50 class supports the generation of a program image. 51 """ 52 53 supported_optimisations = micropython.ast.Translation.supported_optimisations 54 55 def __init__(self, path=None, verbose=0): 56 57 """ 58 Initialise the importer with the given search 'path' - a list of 59 directories to search for Python modules. 60 61 The optional 'verbose' parameter causes output concerning the activities 62 of the object to be produced if set to a true value (not the default). 63 """ 64 65 self.path = path or [os.getcwd()] 66 self.verbose = verbose 67 self.modules = {} 68 self.loading = set() 69 70 # Remember the tables once generated. 71 72 self.objtable = None 73 self.paramtable = None 74 75 # Constant records. 76 77 self.constant_values = {} 78 self.constant_list = None # cache for constants 79 80 # Main program information. 81 82 self.code = None 83 self.code_location = None 84 85 def constants(self): 86 87 "Return a list of constants." 88 89 if self.constant_list is None: 90 self.constant_list = list(self.constant_values.values()) 91 92 return self.constant_list 93 94 def vacuum(self): 95 96 "Tidy up the modules." 97 98 for name, module in self.modules.items(): 99 if module.loaded: 100 module.vacuum() 101 else: 102 del self.modules[name] 103 104 def get_modules(self): 105 106 "Return all modules known to the importer." 107 108 return self.modules.values() 109 110 def get_image(self, with_builtins=0, optimisations=None): 111 112 "Return a dictionary mapping modules to structures." 113 114 objtable = self.get_object_table() 115 paramtable = self.get_parameter_table() 116 117 image = [] 118 119 # Append constants to the image. 120 121 for pos, const in enumerate(self.constants()): 122 const.location = pos 123 image.append(const) 124 125 for module_name, module in self.modules.items(): 126 if not with_builtins and module_name == "__builtins__": 127 continue 128 129 # Fix the attributes. 130 131 module.finalise_attributes() 132 133 pos = len(image) 134 135 # Position the module in the image and make a translation. 136 137 module.location = pos 138 trans = micropython.ast.Translation(module, self, optimisations) 139 140 # Add header details. 141 142 image.append(module) 143 pos += 1 144 145 # Append module attributes to the image. 146 147 attributes = module.module_attributes() 148 image += module.attributes_as_list() 149 pos += len(attributes.keys()) 150 151 # Append classes and functions to the image. 152 153 for obj in module.all_objects: 154 if isinstance(obj, micropython.inspect.Class): 155 156 # Fix the attributes. 157 158 obj.finalise_attributes() 159 160 # Position the class in the image. 161 162 obj.location = pos 163 164 # Add header details. 165 166 image.append(obj) 167 pos += 1 168 169 # Append class attributes to the image. 170 171 attributes = obj.class_attributes() 172 image += obj.attributes_as_list() 173 pos += len(attributes.keys()) 174 175 # Class-level code is generated separately. 176 # The code location is set within the code generation 177 # process for the module. 178 179 # NOTE: Generate module and function code here. 180 181 elif isinstance(obj, micropython.inspect.Function): 182 183 # Fix the attributes. 184 185 obj.finalise_attributes() 186 187 # Position the function in the image. 188 189 obj.location = pos 190 191 # Add header details. 192 193 image.append(obj) 194 pos += 1 195 196 # Append any default values to the image. 197 198 image += obj.default_attrs 199 pos += len(obj.default_attrs) 200 201 # Append the function code to the image. 202 203 obj.code_location = pos 204 code = trans.get_code(obj) 205 image += code 206 pos += len(code) 207 208 # Remember the position of the module code. 209 210 module.code_location = pos 211 212 # Append the module top-level code to the image. 213 214 code = trans.get_module_code() 215 image += code 216 pos += len(code) 217 218 self.code = image 219 self.code_location = self.modules["__main__"].code_location 220 return image 221 222 def get_object_table(self): 223 224 "Return a table with details of attributes for classes and modules." 225 226 if self.objtable is None: 227 t = self.objtable = micropython.table.Table() 228 for module in self.get_modules(): 229 t.add(module.full_name(), module.module_attributes()) 230 for obj in module.all_objects: 231 if isinstance(obj, micropython.inspect.Class): 232 t.add(obj.full_name(), obj.all_attributes()) 233 234 return self.objtable 235 236 def get_parameter_table(self): 237 238 "Return a table with details of parameters for functions and methods." 239 240 # Need the object table to get at class details. 241 242 objtable = self.get_object_table() 243 244 if self.paramtable is None: 245 t = self.paramtable = micropython.table.Table() 246 247 # Visit each module, getting function and method details. 248 249 for module in self.get_modules(): 250 for obj in module.all_objects: 251 if isinstance(obj, micropython.inspect.Function): 252 t.add(obj.full_name(), obj.parameters()) 253 254 # Classes are callable, too. 255 # Take details of the appropriate __init__ method to make an 256 # entry for an instantiation function for the class. 257 258 elif isinstance(obj, micropython.inspect.Class): 259 t.add(obj.full_name(), obj.get_instantiator().parameters()) 260 261 # Filter out all parameter table entries not referenced by keyword 262 # arguments. 263 264 keyword_names = set() 265 266 for module in self.get_modules(): 267 keyword_names.update(module.keyword_names) 268 269 for function_name, parameters in t.table.items(): 270 for name in parameters.keys(): 271 if name in keyword_names: 272 break 273 else: 274 del t.table[function_name] 275 276 return self.paramtable 277 278 # Import methods. 279 280 def find_in_path(self, name): 281 282 """ 283 Find the given module 'name' in the search path, returning None where no 284 such module could be found, or a 2-tuple from the 'find' method 285 otherwise. 286 """ 287 288 for d in self.path: 289 m = self.find(d, name) 290 if m: return m 291 return None 292 293 def find(self, d, name): 294 295 """ 296 In the directory 'd', find the given module 'name', where 'name' can 297 either refer to a single file module or to a package. Return None if the 298 'name' cannot be associated with either a file or a package directory, 299 or a 2-tuple from '_find_package' or '_find_module' otherwise. 300 """ 301 302 m = self._find_package(d, name) 303 if m: return m 304 m = self._find_module(d, name) 305 if m: return m 306 return None 307 308 def _find_module(self, d, name): 309 310 """ 311 In the directory 'd', find the given module 'name', returning None where 312 no suitable file exists in the directory, or a 2-tuple consisting of 313 None (indicating that no package directory is involved) and a filename 314 indicating the location of the module. 315 """ 316 317 name_py = name + os.extsep + "py" 318 filename = self._find_file(d, name_py) 319 if filename: 320 return None, filename 321 return None 322 323 def _find_package(self, d, name): 324 325 """ 326 In the directory 'd', find the given package 'name', returning None 327 where no suitable package directory exists, or a 2-tuple consisting of 328 a directory (indicating the location of the package directory itself) 329 and a filename indicating the location of the __init__.py module which 330 declares the package's top-level contents. 331 """ 332 333 filename = self._find_file(d, name) 334 if filename: 335 init_py = "__init__" + os.path.extsep + "py" 336 init_py_filename = self._find_file(filename, init_py) 337 if init_py_filename: 338 return filename, init_py_filename 339 return None 340 341 def _find_file(self, d, filename): 342 343 """ 344 Return the filename obtained when searching the directory 'd' for the 345 given 'filename', or None if no actual file exists for the filename. 346 """ 347 348 filename = os.path.join(d, filename) 349 if os.path.exists(filename): 350 return filename 351 else: 352 return None 353 354 def load(self, name, return_leaf=0): 355 356 """ 357 Load the module or package with the given 'name'. Return an object 358 referencing the loaded module or package, or None if no such module or 359 package exists. 360 """ 361 362 if self.modules.has_key(name) and self.modules[name].loaded: 363 #print "Cached (%s)" % name 364 return self.modules[name] 365 if self.verbose: 366 print "Loading", name 367 368 # Split the name into path components, and try to find the uppermost in 369 # the search path. 370 371 path = name.split(".") 372 m = self.find_in_path(path[0]) 373 if not m: 374 if self.verbose: 375 print "Not found (%s)" % path[0] 376 return None # NOTE: Import error. 377 d, filename = m 378 379 # Either acquire a reference to an already-imported module, or load the 380 # module from a file. 381 382 top = module = self.load_from_file(filename, path[0]) 383 384 # For hierarchical names, traverse each path component... 385 386 if len(path) > 1: 387 if not d: 388 if self.verbose: 389 print "No package (%s)" % filename 390 return None # NOTE: Import error (package not found). 391 else: 392 self.add_submodules(d, module) 393 394 path_so_far = path[:1] 395 for p in path[1:]: 396 path_so_far.append(p) 397 398 # Find the package or module concerned. 399 400 m = self.find(d, p) 401 if not m: 402 if self.verbose: 403 print "Not found (%s)" % p 404 return None # NOTE: Import error. 405 d, filename = m 406 module_name = ".".join(path_so_far) 407 408 # Either reference an imported module or load one from a file. 409 410 submodule = self.load_from_file(filename, module_name) 411 412 if d: 413 self.add_submodules(d, module) 414 415 # Store the submodule within its parent module. 416 417 module.set_module(p, submodule) 418 module = submodule 419 420 # Return either the deepest or the uppermost module. 421 422 if return_leaf: 423 return module 424 else: 425 return top 426 427 def load_from_file(self, name, module_name=None): 428 429 """ 430 Load the module with the given 'name' (which may be a full module path). 431 """ 432 433 if module_name is None: 434 module_name = "__main__" 435 436 module = self.add_module(module_name) 437 if not module.loaded and module not in self.loading: 438 self.loading.add(module) 439 #print "Parsing", name 440 module.parse(name) 441 #print "Done", name 442 self.loading.remove(module) 443 module.loaded = 1 444 445 # Record the module. 446 447 #print "Loaded", module_name, "with namespace", module.namespace.keys() 448 return module 449 450 def add_module(self, module_name): 451 452 """ 453 Return the module with the given 'module_name', adding a new module 454 object if one does not already exist. 455 """ 456 457 if not self.modules.has_key(module_name): 458 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 459 else: 460 module = self.modules[module_name] 461 return module 462 463 def add_submodules(self, pathname, module): 464 465 """ 466 Work around insufficient __all__ declarations and examine the directory 467 with the given 'pathname', adding submodules to the given 'module'. 468 """ 469 470 for filename in os.listdir(pathname): 471 submodule, ext = os.path.splitext(filename) 472 if ext not in ("", ".py"): 473 continue 474 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 475 476 # vim: tabstop=4 expandtab shiftwidth=4