1 # -*- coding: iso-8859-1 -*- 2 """ 3 MoinMoin - MoinRemoteSupport library 4 5 @copyright: 2011, 2012, 2013, 2014 by Paul Boddie <paul@boddie.org.uk> 6 @license: GNU GPL (v2 or later), see COPYING.txt for details. 7 """ 8 9 from ContentTypeSupport import getContentTypeAndEncoding 10 from MoinMoin.action import cache 11 from MoinMoin import caching 12 import urllib2, time 13 14 def getCachedResource(request, url, arena, scope, max_cache_age, reader=None): 15 16 """ 17 Using the given 'request', return the resource data for the given 'url', 18 accessing a cache entry with the given 'arena' and 'scope' where the data 19 has already been downloaded. The 'max_cache_age' indicates the length in 20 seconds that a cache entry remains valid. 21 22 If the optional 'reader' object is given, it will be used to access the 23 'url' and write the downloaded data to a cache entry. Otherwise, a standard 24 URL reader will be used. 25 26 If the resource cannot be downloaded and cached, None is returned. 27 Otherwise, the form of the data is as follows: 28 29 url <newline> 30 [ content-type-header ] <newline> 31 [ other-header <newline> ]* 32 <newline> 33 content-body 34 """ 35 36 reader = reader or urlreader 37 38 # See if the URL is cached. 39 40 cache_key = cache.key(request, content=url) 41 cache_entry = caching.CacheEntry(request, arena, cache_key, scope=scope) 42 43 # If no entry exists, or if the entry is older than the specified age, 44 # create one with the response from the URL. 45 46 now = time.time() 47 mtime = cache_entry.mtime() 48 49 # NOTE: The URL could be checked and the 'If-Modified-Since' header 50 # NOTE: (see MoinMoin.action.pollsistersites) could be checked. 51 52 if not cache_entry.exists() or now - mtime >= max_cache_age: 53 54 # Access the remote data source. 55 56 cache_entry.open(mode="w") 57 58 try: 59 # Read from the source and write to the cache. 60 61 reader(url, cache_entry) 62 63 # In case of an exception, return None. 64 65 except IOError: 66 if cache_entry.exists(): 67 cache_entry.remove() 68 return None 69 70 # Open the cache entry and read it. 71 72 cache_entry.open() 73 try: 74 return cache_entry.read() 75 finally: 76 cache_entry.close() 77 78 def urlreader(url, cache_entry): 79 80 "Retrieve data from the given 'url', writing it to the 'cache_entry'." 81 82 f = urllib2.urlopen(url) 83 try: 84 cache_entry.write(url + "\n") 85 cache_entry.write((f.headers.get("content-type") or "") + "\n") 86 for key, value in f.headers.items(): 87 if key.lower() != "content-type": 88 cache_entry.write("%s: %s\n" % (key, value)) 89 cache_entry.write("\n") 90 cache_entry.write(f.read()) 91 finally: 92 cache_entry.close() 93 f.close() 94 95 def getCachedResourceMetadata(f): 96 97 "Return a metadata dictionary for the given resource file-like object 'f'." 98 99 url = f.readline() 100 content_type, encoding = getContentTypeAndEncoding(f.readline()) 101 102 metadata = {} 103 line = f.readline() 104 105 while line.strip(): 106 key, value = [v.strip() for v in line.split(":", 1)] 107 metadata[key] = value 108 line = f.readline() 109 110 return url, content_type, encoding, metadata 111 112 # vim: tabstop=4 expandtab shiftwidth=4