# HG changeset patch # User Paul Boddie # Date 1422441862 -3600 # Node ID 791ab8b6dd9d682f9cf2818f3ec9943df0ada68a # Parent 8a69d67d27ad576172fa87597bee378b65300c27# Parent 9baf3570197b280a97b61376d228f869e4dc791b Merged changes from the default branch. diff -r 8a69d67d27ad -r 791ab8b6dd9d .hgtags --- a/.hgtags Wed Jan 28 11:40:31 2015 +0100 +++ b/.hgtags Wed Jan 28 11:44:22 2015 +0100 @@ -2,3 +2,4 @@ 3d28b3c1896e7569c430662c5d4ec0b536266d56 rel-0-2 52ce79af5ae73b95addbe67b88f2f1037219246e rel-0-3 91c146800e069c86f592798b63bf4bdb4b7be271 rel-0-4 +c9b3086a024c406ab8386b9f96e04c1f76cf1c13 rel-0-4-1 diff -r 8a69d67d27ad -r 791ab8b6dd9d DateSupport.py --- a/DateSupport.py Wed Jan 28 11:40:31 2015 +0100 +++ b/DateSupport.py Wed Jan 28 11:44:22 2015 +0100 @@ -2,7 +2,7 @@ """ MoinMoin - DateSupport library (derived from EventAggregatorSupport) - @copyright: 2008, 2009, 2010, 2011, 2012, 2013 by Paul Boddie + @copyright: 2008, 2009, 2010, 2011, 2012, 2013, 2014 by Paul Boddie @license: GNU GPL (v2 or later), see COPYING.txt for details. """ @@ -58,7 +58,7 @@ timezone_iso8601_offset_str = ur'(?P(?:(?P[-+])(?P[0-9]{2}):(?P[0-9]{2})))' datetime_iso8601_regexp_str = date_regexp_str + \ ur'(?:T' + time_regexp_str + \ - ur'(?:(?PZ)|(?P' + timezone_iso8601_offset_str + '))' \ + ur'(?:(?PZ)|(?P' + timezone_iso8601_offset_str + '))?' \ ur')?' date_icalendar_regexp = re.compile(date_icalendar_regexp_str, re.UNICODE) @@ -392,7 +392,7 @@ def __str__(self): return Date.__str__(self) + self.time_string() - def time_string(self, zone_as_offset=False, time_prefix=" ", zone_prefix=" "): + def time_string(self, zone_as_offset=False, time_prefix=" ", zone_prefix=" ", zone_separator=":"): if self.has_time(): data = self.as_tuple() time_str = "%s%02d:%02d" % ((time_prefix,) + data[3:5]) @@ -402,13 +402,23 @@ if zone_as_offset: utc_offset = self.utc_offset() if utc_offset: - time_str += "%s%+03d:%02d" % ((zone_prefix,) + utc_offset) + time_str += "%s%+03d%s%02d" % (zone_prefix, utc_offset[0], zone_separator, utc_offset[1]) else: time_str += "%s%s" % (zone_prefix, data[6]) return time_str else: return "" + def as_RFC2822_datetime_string(self): + weekday = calendar.weekday(*self.data[:3]) + return "%s, %02d %s %04d %s" % ( + getDayLabel(weekday)[:3], + self.data[2], + getMonthLabel(self.data[1])[:3], + self.data[0], + self.time_string(zone_as_offset=True, time_prefix="", zone_prefix=" ", zone_separator="") + ) + def as_HTTP_datetime_string(self): weekday = calendar.weekday(*self.data[:3]) return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (( diff -r 8a69d67d27ad -r 791ab8b6dd9d ItemSupport.py --- a/ItemSupport.py Wed Jan 28 11:40:31 2015 +0100 +++ b/ItemSupport.py Wed Jan 28 11:44:22 2015 +0100 @@ -2,135 +2,60 @@ """ MoinMoin - ItemSupport library - @copyright: 2013 by Paul Boddie + @copyright: 2013, 2014 by Paul Boddie @license: GNU GPL (v2 or later), see COPYING.txt for details. """ +from MoinMoin import config +from MoinMoin.Page import Page +from MoinMoin.PageEditor import PageEditor +from MoinMoin.security import Permissions from MoinMoin.util import lock +from MoinSupport import getMetadata, getPagesForSearch import os # Content storage support. -class ItemDirectoryStore: +class SpecialPermissionsForPage(Permissions): + + "Permit saving of ACL-enabled pages." + + def __init__(self, user, pagename): + Permissions.__init__(self, user) + self.pagename = pagename - "A directory-based item store." + def admin(self, pagename): + return pagename == self.pagename + + write = admin + +class ReadPermissionsForSubpages(Permissions): + + "Permit listing of ACL-affected subpages." - def __init__(self, path, lock_dir): + def __init__(self, user, pagename): + Permissions.__init__(self, user) + self.pagename = pagename + + def read(self, pagename): + return pagename.startswith("%s/" % self.pagename) + +# Underlying storage mechanisms. - "Initialise an item store for the given 'path' and 'lock_dir'." +class GeneralItemStore: + + "Common item store functionality." - self.path = path - self.next_path = os.path.join(self.path, "next") + def __init__(self, lock_dir): + + "Initialise an item store with the given 'lock_dir' guarding access." + self.lock_dir = lock_dir self.writelock = lock.WriteLock(lock_dir) self.readlock = lock.ReadLock(lock_dir) - def mtime(self): - - "Return the last modified time of the item store directory." - - return os.path.getmtime(self.path) - - def get_next(self): - - "Return the next item number." - - next = self.read_next() - if next is None: - next = self.deduce_next() - self.write_next(next) - return next - - def get_keys(self): - - "Return the item keys." - - return [int(filename) for filename in os.listdir(self.path) if filename.isdigit()] - - def deduce_next(self): - - "Deduce the next item number from the existing item files." - - return max(self.get_keys() or [-1]) + 1 - - def read_next(self): - - "Read the next item number from a special file." - - if not os.path.exists(self.next_path): - return None - - f = open(self.next_path) - try: - try: - return int(f.read()) - except ValueError: - return None - finally: - f.close() - - def write_next(self, next): - - "Write the 'next' item number to a special file." - - f = open(self.next_path, "w") - try: - f.write(str(next)) - finally: - f.close() - - def write_item(self, item, next): - - "Write the given 'item' to a file with the given 'next' item number." - - f = open(self.get_item_path(next), "w") - try: - f.write(item) - finally: - f.close() - - def read_item(self, number): - - "Read the item with the given item 'number'." - - f = open(self.get_item_path(number)) - try: - return f.read() - finally: - f.close() - - def remove_item(self, number): - - "Remove the item with the given item 'number'." - - os.remove(self.get_item_path(number)) - - def get_item_path(self, number): - - "Get the path for the given item 'number'." - - path = os.path.abspath(os.path.join(self.path, str(number))) - basepath = os.path.join(self.path, "") - - if os.path.commonprefix([path, basepath]) != basepath: - raise OSError, path - - return path - # High-level methods. - def append(self, item): - - "Append the given 'item' to the store." - - self.writelock.acquire() - try: - next = self.get_next() - self.write_item(item, next) - self.write_next(next + 1) - finally: - self.writelock.release() - def __len__(self): """ @@ -194,13 +119,280 @@ finally: self.writelock.release() +class SequentialAccess(GeneralItemStore): + + "Support sequential access to items." + + def deduce_next(self): + + "Deduce the next item number from the existing item files." + + return max(self.get_keys() or [-1]) + 1 + + def read_next(self): + + "Read the next item number from a special file." + + if not os.path.exists(self.next_path): + return None + + f = open(self.next_path) + try: + try: + return int(f.read()) + except ValueError: + return None + finally: + f.close() + + def write_next(self, next): + + "Write the 'next' item number to a special file." + + f = open(self.next_path, "w") + try: + f.write(str(next)) + finally: + f.close() + +class DirectoryStore(GeneralItemStore): + + "A directory-based item store." + + def __init__(self, path, lock_dir): + + "Initialise an item store for the given 'path' and 'lock_dir'." + + self.path = path + self.next_path = os.path.join(self.path, "next") + self.lock_dir = lock_dir + self.writelock = lock.WriteLock(lock_dir) + self.readlock = lock.ReadLock(lock_dir) + + def mtime(self): + + "Return the last modified time of the item store directory." + + return os.path.getmtime(self.path) + + def get_keys(self): + + "Return the item keys." + + return [int(filename) for filename in os.listdir(self.path) if filename.isdigit()] + + def write_item(self, item, next): + + "Write the given 'item' to a file with the given 'next' item number." + + f = open(self.get_item_path(next), "wb") + try: + f.write(item) + finally: + f.close() + + def read_item(self, identifier): + + "Read the item with the given item 'identifier'." + + f = open(self.get_item_path(identifier), "rb") + try: + return f.read() + finally: + f.close() + + def remove_item(self, identifier): + + "Remove the item with the given item 'identifier'." + + os.remove(self.get_item_path(identifier)) + + def get_item_path(self, identifier): + + "Get the path for the given item 'identifier'." + + if isinstance(identifier, unicode): + filename = identifier.encode(config.charset) + else: + filename = identifier + + path = os.path.abspath(os.path.join(self.path, filename)) + basepath = os.path.join(self.path, "") + + if os.path.commonprefix([path, basepath]) != basepath: + raise OSError, path + + return path + +class DirectoryItemStore(DirectoryStore, SequentialAccess): + + "A directory-based item store with numeric keys." + + def get_next(self): + + "Return the next item number." + + next = self.read_next() + if next is None: + next = self.deduce_next() + self.write_next(next) + return next + + # High-level methods. + + def append(self, item): + + "Append the given 'item' to the store." + + self.writelock.acquire() + try: + next = self.get_next() + self.write_item(item, next) + self.write_next(next + 1) + finally: + self.writelock.release() + +class DirectoryNamedItemStore(DirectoryStore): + + "A directory-based item store with explicit keys." + + def __setitem__(self, name, item): + + "Using the given 'name', set the given 'item' in the store." + + self.writelock.acquire() + try: + self.write_item(item, name) + finally: + self.writelock.release() + +class SubpageItemStore(SequentialAccess): + + "A subpage-based item store." + + def __init__(self, page, lock_dir): + + "Initialise an item store for subpages under the given 'page'." + + GeneralItemStore.__init__(self, lock_dir) + self.page = page + + def mtime(self): + + "Return the last modified time of the item store." + + keys = self.get_keys() + if not keys: + page = self.page + else: + page = Page(self.page.request, self.get_item_path(max(keys))) + + return wikiutil.version2timestamp( + getMetadata(page)["last-modified"] + ) + + def get_next(self): + + "Return the next item number." + + return self.deduce_next() + + def get_keys(self): + + "Return the item keys." + + request = self.page.request + + # Collect the strict subpages of the parent page. + + leafnames = [] + parentname = self.page.page_name + + # To list pages whose ACLs may prevent access, a special policy is required. + + may = request.user.may + request.user.may = ReadPermissionsForSubpages(request.user, parentname) + + try: + for page in getPagesForSearch("title:regex:^%s/" % parentname, self.page.request): + basename, leafname = page.page_name.rsplit("/", 1) + + # Only collect numbered pages immediately below the parent. + + if basename == parentname and leafname.isdigit(): + leafnames.append(int(leafname)) + + return leafnames + + # Restore the original policy. + + finally: + request.user.may = may + + def write_item(self, item, next): + + "Write the given 'item' to a page with the given 'next' item number." + + request = self.page.request + pagename = self.get_item_path(next) + + # To add a page with an ACL, a special policy is required. + + may = request.user.may + request.user.may = SpecialPermissionsForPage(request.user, pagename) + + # Attempt to save the page, copying any ACL. + + try: + page = PageEditor(request, pagename) + page.saveText(item, 0) + + # Restore the original policy. + + finally: + request.user.may = may + + def read_item(self, number): + + "Read the item with the given item 'number'." + + page = Page(self.page.request, self.get_item_path(number)) + return page.get_raw_body() + + def remove_item(self, number): + + "Remove the item with the given item 'number'." + + page = PageEditor(self.page.request, self.get_item_path(number)) + page.deletePage() + + def get_item_path(self, number): + + "Get the path for the given item 'number'." + + return "%s/%s" % (self.page.page_name, number) + + # High-level methods. + + def append(self, item): + + "Append the given 'item' to the store." + + self.writelock.acquire() + try: + next = self.get_next() + self.write_item(item, next) + finally: + self.writelock.release() + class ItemIterator: "An iterator over items in a store." - def __init__(self, store, direction=1): + def __init__(self, store, direction=1, keys=None): self.store = store self.direction = direction + self.keys = keys self.reset() def reset(self): @@ -218,7 +410,10 @@ return self._next >= self.final def get_next(self): - next = self._next + if self.keys: + next = self.keys[self._next] + else: + next = self._next self._next += self.direction return next @@ -242,4 +437,174 @@ def __iter__(self): return self +def getDirectoryItemStoreForPage(page, item_dir, lock_dir): + + """ + A convenience function returning a directory-based store for the given + 'page', using the given 'item_dir' and 'lock_dir'. + """ + + item_dir_path = tuple(item_dir.split("/")) + lock_dir_path = tuple(lock_dir.split("/")) + return DirectoryItemStore(page.getPagePath(*item_dir_path), page.getPagePath(*lock_dir_path)) + +def getDirectoryNamedItemStoreForPage(page, item_dir, lock_dir): + + """ + A convenience function returning a directory-based store for the given + 'page', using the given 'item_dir' and 'lock_dir'. + """ + + item_dir_path = tuple(item_dir.split("/")) + lock_dir_path = tuple(lock_dir.split("/")) + return DirectoryNamedItemStore(page.getPagePath(*item_dir_path), page.getPagePath(*lock_dir_path)) + +def getSubpageItemStoreForPage(page, lock_dir): + + """ + A convenience function returning a subpage-based store for the given + 'page', using the given 'lock_dir'. + """ + + lock_dir_path = tuple(lock_dir.split("/")) + return SubpageItemStore(page, page.getPagePath(*lock_dir_path)) + +# Page-oriented item store classes. + +class ItemStoreBase: + + "Access item stores via pages, observing page access restrictions." + + def __init__(self, page, store): + self.page = page + self.store = store + + def can_write(self): + + """ + Return whether the user associated with the request can write to the + page owning this store. + """ + + user = self.page.request.user + return user and user.may.write(self.page.page_name) + + def can_read(self): + + """ + Return whether the user associated with the request can read from the + page owning this store. + """ + + user = self.page.request.user + return user and user.may.read(self.page.page_name) + + def can_delete(self): + + """ + Return whether the user associated with the request can delete the + page owning this store. + """ + + user = self.page.request.user + return user and user.may.delete(self.page.page_name) + + # Store-specific methods. + + def mtime(self): + return self.store.mtime() + + # High-level methods. + + def keys(self): + + "Return a list of keys for items in the store." + + if not self.can_read(): + return 0 + + return self.store.keys() + + def __len__(self): + + "Return the number of items in the store." + + if not self.can_read(): + return 0 + + return len(self.store) + + def __getitem__(self, number): + + "Return the item with the given 'number'." + + if not self.can_read(): + raise IndexError, number + + return self.store.__getitem__(number) + + def __delitem__(self, number): + + "Remove the item with the given 'number'." + + if not self.can_delete(): + return + + return self.store.__delitem__(number) + + def __iter__(self): + return self.store.__iter__() + + def next(self): + return self.store.next() + +class SequentialStoreBase: + + "Sequential access methods for item stores." + + def append(self, item): + + "Append the given 'item' to the store." + + if not self.can_write(): + return + + self.store.append(item) + +class NamedStoreBase: + + "Name-based access methods for item stores." + + def __setitem__(self, name, item): + + "Using the given 'name', set the given 'item' in the store." + + if not self.can_write(): + return + + self.store[name] = item + +# Convenience store classes. + +class ItemStore(ItemStoreBase, SequentialStoreBase): + + "Store items in a directory via a page." + + def __init__(self, page, item_dir="items", lock_dir=None): + ItemStoreBase.__init__(self, page, getDirectoryItemStoreForPage(page, item_dir, lock_dir or ("%s-locks" % item_dir))) + +class NamedItemStore(ItemStoreBase, NamedStoreBase): + + "Store items in a directory via a page." + + def __init__(self, page, item_dir="items", lock_dir=None): + ItemStoreBase.__init__(self, page, getDirectoryNamedItemStoreForPage(page, item_dir, lock_dir or ("%s-locks" % item_dir))) + +class ItemSubpageStore(ItemStoreBase): + + "Store items in subpages of a page." + + def __init__(self, page, lock_dir=None): + ItemStoreBase.__init__(self, page, getSubpageItemStoreForPage(page, lock_dir or "subpage-items-locks")) + # vim: tabstop=4 expandtab shiftwidth=4 diff -r 8a69d67d27ad -r 791ab8b6dd9d MoinRemoteSupport.py --- a/MoinRemoteSupport.py Wed Jan 28 11:40:31 2015 +0100 +++ b/MoinRemoteSupport.py Wed Jan 28 11:44:22 2015 +0100 @@ -2,16 +2,23 @@ """ MoinMoin - MoinRemoteSupport library - @copyright: 2011, 2012, 2013 by Paul Boddie + @copyright: 2011, 2012, 2013, 2014 by Paul Boddie @license: GNU GPL (v2 or later), see COPYING.txt for details. """ from ContentTypeSupport import getContentTypeAndEncoding from MoinMoin.action import cache -from MoinMoin import caching +from MoinMoin import caching, log +from email.parser import Parser +from email.mime.multipart import MIMEMultipart +from urllib import splithost, splitpasswd, splitport, splituser, unquote_plus +from urlparse import urlsplit import urllib2, time +import imaplib -def getCachedResource(request, url, arena, scope, max_cache_age): +logging = log.getLogger(__name__) + +def getCachedResource(request, url, arena, scope, max_cache_age, reader=None): """ Using the given 'request', return the resource data for the given 'url', @@ -19,6 +26,10 @@ has already been downloaded. The 'max_cache_age' indicates the length in seconds that a cache entry remains valid. + If the optional 'reader' object is given, it will be used to access the + 'url' and write the downloaded data to a cache entry. Otherwise, a standard + URL reader will be used. + If the resource cannot be downloaded and cached, None is returned. Otherwise, the form of the data is as follows: @@ -29,6 +40,8 @@ content-body """ + reader = reader or urlreader + # See if the URL is cached. cache_key = cache.key(request, content=url) @@ -43,32 +56,27 @@ # NOTE: The URL could be checked and the 'If-Modified-Since' header # NOTE: (see MoinMoin.action.pollsistersites) could be checked. - if not cache_entry.exists() or now - mtime >= max_cache_age: + if not cache_entry.exists() or cache_entry.size() == 0 or now - mtime >= max_cache_age: # Access the remote data source. cache_entry.open(mode="w") try: - f = urllib2.urlopen(url) try: - cache_entry.write(url + "\n") - cache_entry.write((f.headers.get("content-type") or "") + "\n") - for key, value in f.headers.items(): - if key.lower() != "content-type": - cache_entry.write("%s: %s\n" % (key, value)) - cache_entry.write("\n") - cache_entry.write(f.read()) - finally: - cache_entry.close() - f.close() + # Read from the source and write to the cache. + + reader(url, cache_entry) + + # In case of an exception, return None. - # In case of an exception, return None. + except IOError: + if cache_entry.exists(): + cache_entry.remove() + return None - except IOError: - if cache_entry.exists(): - cache_entry.remove() - return None + finally: + cache_entry.close() # Open the cache entry and read it. @@ -78,6 +86,111 @@ finally: cache_entry.close() +def urlreader(url, cache_entry): + + "Retrieve data from the given 'url', writing it to the 'cache_entry'." + + f = urllib2.urlopen(url) + try: + writeCacheHeaders(url, f.headers, cache_entry) + cache_entry.write(f.read()) + finally: + f.close() + +def imapreader(url, cache_entry): + + """ + Retrieve data associated with the given 'url' using the IMAP protocol + specifically, writing it to the 'cache_entry'. + """ + + # NOTE: Should use something like pykolab.imap_utf7.encode here. + + enc = lambda s: s.encode("utf-7") + + # The URL maps to credentials and folder details. + + scheme, netloc, path, query, fragment = urlsplit(url) + credentials, location = splituser(netloc) + username, password = map(unquote_plus, splitpasswd(credentials)) + host, port = splitport(location) + folders = map(unquote_plus, path.split("/")[1:]) + + # Connect and log in to the IMAP server. + + cls = scheme == "imaps" and imaplib.IMAP4_SSL or imaplib.IMAP4 + + if port is None: + i = cls(host) + else: + i = cls(host, int(port)) + + i.login(username, password) + + try: + # Descend to the desired folder. + + for folder in folders: + code, response = i.select(enc(folder), readonly=True) + if code != "OK": + logging.warning("Could not enter folder: %s" % folder) + raise IOError + + # Search for all messages. + # NOTE: This could also be parameterised. + + code, response = i.search(None, "(ALL)") + + if code != "OK": + logging.warning("Could not enter folder: %s" % folder) + raise IOError + + # For each result, obtain the full message, but embed it in a larger + # multipart message. + + message = MIMEMultipart() + + writeCacheHeaders(url, message, cache_entry) + + numbers = response and response[0].split(" ") or [] + + for n in numbers: + code, response = i.fetch(n, "(RFC822.PEEK)") + + if code == "OK" and response: + + # Write the message payload into the cache entry for later + # processing. + + for data in response: + try: + envelope, body = data + message.attach(Parser().parsestr(body)) + except ValueError: + pass + else: + logging.warning("Could not obtain message %d from folder %s" % (n, folder)) + + cache_entry.write(message.as_string()) + + finally: + i.logout() + del i + +def writeCacheHeaders(url, headers, cache_entry): + + """ + For the given 'url', write it and the given 'headers' to the given + 'cache_entry'. + """ + + cache_entry.write(url + "\n") + cache_entry.write((headers.get("content-type") or "") + "\n") + for key, value in headers.items(): + if key.lower() != "content-type": + cache_entry.write("%s: %s\n" % (key, value)) + cache_entry.write("\n") + def getCachedResourceMetadata(f): "Return a metadata dictionary for the given resource file-like object 'f'." diff -r 8a69d67d27ad -r 791ab8b6dd9d MoinSupport.py --- a/MoinSupport.py Wed Jan 28 11:40:31 2015 +0100 +++ b/MoinSupport.py Wed Jan 28 11:44:22 2015 +0100 @@ -2,9 +2,9 @@ """ MoinMoin - MoinSupport library (derived from EventAggregatorSupport) - @copyright: 2008, 2009, 2010, 2011, 2012, 2013 by Paul Boddie + @copyright: 2008, 2009, 2010, 2011, 2012, 2013, 2014 by Paul Boddie @copyright: 2000-2004 Juergen Hermann - 2004 by Florian Festi + 2004,2006 by Florian Festi 2006 by Mikko Virkkil 2005-2008 MoinMoin:ThomasWaldmann 2007 MoinMoin:ReimarBauer @@ -13,11 +13,11 @@ """ from DateSupport import * -from ItemSupport import ItemDirectoryStore from MoinMoin.parser import text_moin_wiki from MoinMoin.Page import Page -from MoinMoin.util import lock -from MoinMoin import config, search, wikiutil +from MoinMoin.support.python_compatibility import hash_new +from MoinMoin import caching, config, search, wikiutil +from os.path import abspath, exists, join, split from shlex import shlex import re import time @@ -36,7 +36,15 @@ except ImportError: pass -__version__ = "0.4" +# Static resource location. + +try: + from MoinMoin.web import static + htdocs = abspath(join(static.__file__, "htdocs")) +except ImportError: + htdocs = None + +__version__ = "0.5" # Extraction of shared fragments. @@ -45,7 +53,8 @@ # Extraction of headings. -heading_regexp = re.compile(r"^(?P=+)(?P.*?)(?P=level)$", re.UNICODE | re.MULTILINE) +heading_regexp_str = r"^(?P=+)(?P.*?)(?P=level)$" +heading_regexp = re.compile(heading_regexp_str, re.UNICODE | re.MULTILINE) # Category extraction from pages. @@ -65,6 +74,22 @@ ur'{{{(?P.*?)}}}' ur')', re.UNICODE) +# Access to static Moin content. + +def getStaticContentDirectory(request): + + "Use the 'request' to find the htdocs directory." + + global htdocs + + if not htdocs: + htdocs_in_data = abspath(join(split(request.cfg.data_dir)[0], "htdocs")) + if exists(htdocs_in_data): + htdocs = htdocs_in_data + return htdocs + + return htdocs + # Category discovery. def getCategoryPattern(request): @@ -191,6 +216,12 @@ else: return None +def groupHasMember(request, groupname, username): + if hasattr(request.dicts, "has_member"): + return request.dicts.has_member(groupname, username) + else: + return username in request.groups.get(groupname, []) + # Searching-related functions. def getPagesFromResults(result_pages, request): @@ -805,6 +836,64 @@ return {"timestamp" : DateTime(time.gmtime(mtime)[:6] + ("UTC",)), "comment" : comment} +# Page caching functions. + +def getPageCacheKey(page, request, with_params=False): + + """ + Return a cache key for the given 'page' using information in the 'request'. + """ + + if hasattr(page, "getCacheKey"): + return page.getCacheKey(request, with_params) + + key = getPageFormatterName(page, request) + if request.args and with_params: + args = request.args.items() + args.sort() + key_args = [] + for k, v in args: + key_args.append("%s=%s" % (k, wikiutil.url_quote(v))) + arg_str = "&".join(key_args) + key = "%s:%s" % (key, hash_new('sha1', arg_str).hexdigest()) + return key + +def enforcePageCacheLimit(page, request): + + """ + Prevent too many cache entries being stored for the given 'page', using the + 'request' to obtain cache items and configuration details. + """ + + if hasattr(page, "enforceCacheLimit"): + page.enforceCacheLimit(request) + + keys = caching.get_cache_list(request, page, 'item') + try: + cache_limit = int(getattr(request.cfg, 'page_cache_limit', "10")) + except ValueError: + cache_limit = 10 + + if len(keys) >= cache_limit: + items = [caching.CacheEntry(request, page, key, scope='item') for key in keys] + item_ages = [(item.mtime(), item) for item in items] + item_ages.sort() + for item_age, item in item_ages[:-cache_limit]: + item.remove() + +def getPageFormatterName(page, request=None): + + """ + Return a formatter name as used in the caching system for the given 'page' + or using information provided by an optional 'request'. + """ + + formatter = getattr(page, 'formatter', None) or request and getattr(request, 'formatter', None) + if not formatter: + return '' + module = formatter.__module__ + return module[module.rfind('.') + 1:] + # Page parsing and formatting of embedded content. def getOutputTypes(request, format): @@ -911,6 +1000,17 @@ buf.close() return unicode(text, "utf-8") +class RawParser: + + "A parser that just formats everything as text." + + def __init__(self, raw, request, **kw): + self.raw = raw + self.request = request + + def format(self, fmt, write=None): + (write or self.request.write)(fmt.text(self.raw)) + # Finding components for content types. def getParsersForContentType(cfg, mimetype): @@ -968,7 +1068,28 @@ # NOTE: Re-implementing support for verbatim text and linking avoidance. - return "".join([s for s in verbatim_regexp.split(text) if s is not None]) + l = [] + last = 0 + + for m in verbatim_regexp.finditer(text): + start, end = m.span() + l.append(text[last:start]) + + # Process the verbatim macro arguments. + + args = m.group("verbatim") or m.group("verbatim2") + if args: + l += [v for (n, v) in parseMacroArguments(args)] + + # Or just add the match groups. + + else: + l += [s for s in m.groups() if s] + + last = end + + l.append(text[last:]) + return "".join(l) def getEncodedWikiText(text): @@ -1069,87 +1190,4 @@ else: return title -# Content storage support. - -class ItemStore(ItemDirectoryStore): - - "A page-specific item store." - - def __init__(self, page, item_dir="items", lock_dir="item_locks"): - - "Initialise an item store for the given 'page'." - - item_dir_path = tuple(item_dir.split("/")) - lock_dir_path = tuple(lock_dir.split("/")) - ItemDirectoryStore.__init__(self, page.getPagePath(*item_dir_path), page.getPagePath(*lock_dir_path)) - self.page = page - - def can_write(self): - - """ - Return whether the user associated with the request can write to the - page owning this store. - """ - - user = self.page.request.user - return user and user.may.write(self.page.page_name) - - def can_read(self): - - """ - Return whether the user associated with the request can read from the - page owning this store. - """ - - user = self.page.request.user - return user and user.may.read(self.page.page_name) - - def can_delete(self): - - """ - Return whether the user associated with the request can delete the - page owning this store. - """ - - user = self.page.request.user - return user and user.may.delete(self.page.page_name) - - # High-level methods. - - def append(self, item): - - "Append the given 'item' to the store." - - if not self.can_write(): - return - - ItemDirectoryStore.append(self, item) - - def __len__(self): - - "Return the number of items in the store." - - if not self.can_read(): - return 0 - - return ItemDirectoryStore.__len__(self) - - def __getitem__(self, number): - - "Return the item with the given 'number'." - - if not self.can_read(): - raise IndexError, number - - return ItemDirectoryStore.__getitem__(self, number) - - def __delitem__(self, number): - - "Remove the item with the given 'number'." - - if not self.can_delete(): - return - - return ItemDirectoryStore.__delitem__(self, number) - # vim: tabstop=4 expandtab shiftwidth=4 diff -r 8a69d67d27ad -r 791ab8b6dd9d PKG-INFO --- a/PKG-INFO Wed Jan 28 11:40:31 2015 +0100 +++ b/PKG-INFO Wed Jan 28 11:44:22 2015 +0100 @@ -1,12 +1,12 @@ Metadata-Version: 1.1 Name: MoinSupport -Version: 0.4.1 +Version: 0.5 Author: Paul Boddie Author-email: paul at boddie org uk Maintainer: Paul Boddie Maintainer-email: paul at boddie org uk Home-page: http://hgweb.boddie.org.uk/MoinSupport -Download-url: http://hgweb.boddie.org.uk/MoinSupport/archive/rel-0-4-1.tar.bz2 +Download-url: http://hgweb.boddie.org.uk/MoinSupport/archive/rel-0-5.tar.bz2 Summary: Support libraries for MoinMoin extensions License: GPL (version 2 or later) Description: The MoinSupport distribution provides libraries handling datetime diff -r 8a69d67d27ad -r 791ab8b6dd9d README.txt --- a/README.txt Wed Jan 28 11:40:31 2015 +0100 +++ b/README.txt Wed Jan 28 11:44:22 2015 +0100 @@ -5,8 +5,7 @@ extensions. Some of the provided modules can be used independently of MoinMoin, such as the ContentTypeSupport, DateSupport, GeneralSupport, LocationSupport and ViewSupport modules which do not themselves import any -MoinMoin functionality. The ItemSupport module only imports file-locking -functionality from MoinMoin and could potentially be used independently. +MoinMoin functionality. Installation ------------ @@ -64,6 +63,19 @@ If time zone handling is not required, pytz need not be installed. It is, however, highly recommended that pytz be installed. +New in MoinSupport 0.5 (Changes since MoinSupport 0.4.1) +-------------------------------------------------------- + + * Moved ItemStore and related functionality into ItemSupport. + * Added support for subpage-based item stores. + * Added groupHasMember from ApproveChanges. + * Added the TokenSupport module to try and have a reliable shell-like + tokeniser. + * Added RFC 2822 datetime formatting. + * Added a "raw" parser which just formats its input as text. + * Added page-related caching functions. + * Added access to the static content location of a wiki. + New in MoinSupport 0.4.1 (Changes since MoinSupport 0.4) -------------------------------------------------------- @@ -72,6 +84,7 @@ * Fixed DateSupport to handle NonExistentTimeError. * Added macro argument quoting functions. * Fixed the quoting of text presented as an argument to the Verbatim macro. + * Fixed the extraction of "verbatim" text in getSimpleWikiText. New in MoinSupport 0.4 (Changes since MoinSupport 0.3) ------------------------------------------------------ diff -r 8a69d67d27ad -r 791ab8b6dd9d TokenSupport.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/TokenSupport.py Wed Jan 28 11:44:22 2015 +0100 @@ -0,0 +1,109 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - TokenSupport library + + @copyright: 2013 by Paul Boddie + @license: GNU GPL (v2 or later), see COPYING.txt for details. +""" + +import re + +identifier_expr = re.compile( + """(?P[^'" ]+)""" + "|" + "(?P +)" + "|" + "(?P'[^']*')" + "|" + '(?P"[^"]*")' + ) + +def getIdentifiers(s, doubling=False): + + """ + Return 's' containing space-separated quoted identifiers, parsed into + regions that hold the individual identifiers. The optional 'doubling' + argument can be used to support convenient quote doubling to reproduce + single quote characters. + + Quoting of identifiers can be done using the single-quote and double-quote + characters in order to include spaces within identifiers. For example: + + 'contains space' + -> contains space (a single identifier) + + Where one kind of quote (or apostrophe) is to be included in an identifier, + the other quoting character can be used to delimit the identifier. For + example: + + "Python's syntax" + -> Python's syntax (a single identifier) + + Where the 'doubling' argument is set to a true value, a quote character can + be doubled to include it in an identifier. For example: + + Python''s syntax + -> Python's syntax (a single identifier) + + Where a mixture of quotes is required in a single identifier, adjacent + quoted regions can be used. For example: + + "Python's "'"intuitive" syntax' + -> "Python's " (region #1) + + '"intuitive" syntax' (region #2) + -> Python's "intuitive" syntax (a single identifier) + + Where unquoted regions are adjacent to quoted regions, the regions are + combined. For example: + + "Python's "intuitive" syntax" + -> "Python's " (region #1) + + intuitive (region #2) + + " syntax" (region #3) + -> Python's intuitive syntax (a single identifier) + """ + + regions = [] + in_literal = False + + for match in identifier_expr.finditer(s): + non_literal, spaces, literal1, literal2 = match.groups() + + identifier = None + + # Spaces prevent continuation of identifier regions. + + if spaces: + in_literal = False + + # Unquoted regions contribute to the current identifier. + + if non_literal and non_literal.strip(): + identifier = non_literal.strip() + + # Quoted regions also contribute to the current identifier. + + for s in (literal1, literal2): + if s is not None: + + # Either strip the quoting or for empty regions, adopt the + # quote character. + + if not doubling or len(s) > 2: + identifier = s[1:-1] + elif doubling: + identifier = s[0] + + # Either continue or add an identifier, and indicate possible + # continuation. + + if identifier: + if in_literal: + regions[-1] += identifier + else: + regions.append(identifier) + in_literal = True + + return regions + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 8a69d67d27ad -r 791ab8b6dd9d docs/COPYING.txt --- a/docs/COPYING.txt Wed Jan 28 11:40:31 2015 +0100 +++ b/docs/COPYING.txt Wed Jan 28 11:44:22 2015 +0100 @@ -1,14 +1,14 @@ Licence Agreement ----------------- -Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013 Paul Boddie +Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013, 2014 Paul Boddie Some pieces of MoinMoin code were used in this work - typically pieces which demonstrate how to perform certain common tasks - and are thus covered by the following copyrights: Copyright (C) 2000-2004 Juergen Hermann -Copyright (C) 2004 by Florian Festi +Copyright (C) 2004, 2006 by Florian Festi Copyright (C) 2005-2008 MoinMoin:ThomasWaldmann Copyright (C) 2006 by Mikko Virkkil Copyright (C) 2007 MoinMoin:ReimarBauer diff -r 8a69d67d27ad -r 791ab8b6dd9d setup.py --- a/setup.py Wed Jan 28 11:40:31 2015 +0100 +++ b/setup.py Wed Jan 28 11:44:22 2015 +0100 @@ -8,9 +8,9 @@ author = "Paul Boddie", author_email = "paul@boddie.org.uk", url = "http://hgweb.boddie.org.uk/MoinSupport", - version = "0.4.1", + version = "0.5", py_modules = ["ContentTypeSupport", "DateSupport", "GeneralSupport", "ItemSupport", "LocationSupport", "MoinDateSupport", "MoinRemoteSupport", "MoinSupport", "RecurrenceSupport", - "ViewSupport"] + "TokenSupport", "ViewSupport"] ) diff -r 8a69d67d27ad -r 791ab8b6dd9d tests/test_tokens.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_tokens.py Wed Jan 28 11:44:22 2015 +0100 @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +from TokenSupport import getIdentifiers + +tests = [ + (1, False, """'contains space'""", ["contains space"]), + (2, False, """contains space""", ["contains", "space"]), + (1, False, '''"Python's syntax"''', ["Python's syntax"]), + (2, False, """Python''s syntax""", ["Pythons", "syntax"]), + (2, True, """Python''s syntax""", ["Python's", "syntax"]), + (1, False, '''"Python's "'"intuitive" syntax' ''', ['''Python's "intuitive" syntax''']), + (1, False, '''"Python's "intuitive" syntax" ''', ['''Python's intuitive syntax''']), + ] + +for n, doubling, s, e in tests: + l = getIdentifiers(s, doubling) + print l == e, l, "==", e, len(l) == n, len(l), "==", n, "<-", doubling, s + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r 8a69d67d27ad -r 791ab8b6dd9d tests/test_verbatim.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_verbatim.py Wed Jan 28 11:44:22 2015 +0100 @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +from MoinSupport import getSimpleWikiText + +s = u''' +Some normal text. + +Some !LinkProtected text. + +Some <> text. + +Some <> text. + +Some <> text. + +Some `monospaced` text. + +Some {{{preformatted}}} text. +''' + +print getSimpleWikiText(s) + +# vim: tabstop=4 expandtab shiftwidth=4