# HG changeset patch # User Paul Boddie # Date 1496352398 -7200 # Node ID 075f08595885a1d315e31e79ce8d40a498d3af08 # Parent a82b56e017212e25b0f68b9de6128de9903784d8 Introduced a file table abstraction for future file storage improvements. Associated specific period classes with period collections, thus simplifying certain method signatures and mechanisms. diff -r a82b56e01721 -r 075f08595885 imiptools/freebusy/__init__.py --- a/imiptools/freebusy/__init__.py Fri May 26 23:58:06 2017 +0200 +++ b/imiptools/freebusy/__init__.py Thu Jun 01 23:26:38 2017 +0200 @@ -25,6 +25,7 @@ FreeBusyCollection, \ FreeBusyGroupCollection, \ FreeBusyOffersCollection, \ - SupportAttendee, SupportExpires + SupportAttendee, SupportExpires, \ + period_from_tuple, period_to_tuple # vim: tabstop=4 expandtab shiftwidth=4 diff -r a82b56e01721 -r 075f08595885 imiptools/freebusy/common.py --- a/imiptools/freebusy/common.py Fri May 26 23:58:06 2017 +0200 +++ b/imiptools/freebusy/common.py Thu Jun 01 23:26:38 2017 +0200 @@ -30,7 +30,10 @@ "Interpret 's' using 'encoding', preserving None." if s: - return unicode(s, encoding) + if isinstance(s, unicode): + return s + else: + return unicode(s, encoding) else: return s @@ -43,6 +46,39 @@ else: return s +class period_from_tuple: + + "Convert a tuple to an instance of the given 'period_class'." + + def __init__(self, period_class): + self.period_class = period_class + def __call__(self, t): + return make_period(t, self.period_class) + +def period_to_tuple(p): + + "Convert period 'p' to a tuple for serialisation." + + return p.as_tuple(strings_only=True) + +def make_period(t, period_class): + + "Convert tuple 't' to an instance of the given 'period_class'." + + args = [] + for arg, column in zip(t, period_class.period_columns): + args.append(from_string(arg, "utf-8")) + return period_class(*args) + +def make_tuple(t, period_class): + + "Restrict tuple 't' to the columns appropriate for 'period_class'." + + args = [] + for arg, column in zip(t, period_class.period_columns): + args.append(arg) + return tuple(args) + # Period abstractions. @@ -51,6 +87,11 @@ "A free/busy record abstraction." + period_columns = [ + "start", "end", "object_uid", "transp", "object_recurrenceid", + "summary", "organiser" + ] + def __init__(self, start, end, uid=None, transp=None, recurrenceid=None, summary=None, organiser=None): @@ -61,7 +102,7 @@ """ PeriodBase.__init__(self, start, end) - self.uid = uid + self.uid = uid or None self.transp = transp or None self.recurrenceid = recurrenceid or None self.summary = summary or None @@ -143,6 +184,8 @@ "A free/busy record abstraction for an offer period." + period_columns = FreeBusyPeriod.period_columns + ["expires"] + def __init__(self, start, end, uid=None, transp=None, recurrenceid=None, summary=None, organiser=None, expires=None): @@ -180,6 +223,8 @@ "A free/busy record abstraction for a quota group period." + period_columns = FreeBusyPeriod.period_columns + ["attendee"] + def __init__(self, start, end, uid=None, transp=None, recurrenceid=None, summary=None, organiser=None, attendee=None): @@ -225,15 +270,14 @@ def __repr__(self): return "FreeBusyGroupPeriod%r" % (self.as_tuple(),) + + +# Collection abstractions. + class FreeBusyCollectionBase: "Common operations on free/busy period collections." - period_columns = [ - "start", "end", "object_uid", "transp", "object_recurrenceid", - "summary", "organiser" - ] - period_class = FreeBusyPeriod def __init__(self, mutable=True): @@ -243,6 +287,12 @@ if not self.mutable: raise TypeError, "Cannot mutate this collection." + def close(self): + + "Close the collection." + + pass + def copy(self): "Make an independent mutable copy of the collection." @@ -256,10 +306,7 @@ column details. """ - args = [] - for arg, column in zip(t, self.period_columns): - args.append(from_string(arg, "utf-8")) - return self.period_class(*args) + return make_period(t, self.period_class) def make_tuple(self, t): @@ -268,10 +315,7 @@ column details. """ - args = [] - for arg, column in zip(t, self.period_columns): - args.append(arg) - return tuple(args) + return make_tuple(t, self.period_class) # List emulation methods. @@ -284,6 +328,16 @@ # Operations. + def insert_period(self, period): + + """ + Insert the given 'period' into the collection. + + This should be implemented in subclasses. + """ + + pass + def insert_periods(self, periods): "Insert the given 'periods' into the collection." @@ -448,7 +502,6 @@ "A mix-in that supports the affected attendee in free/busy periods." - period_columns = FreeBusyCollectionBase.period_columns + ["attendee"] period_class = FreeBusyGroupPeriod def _update_freebusy(self, periods, uid, recurrenceid, attendee=None): @@ -486,7 +539,6 @@ "A mix-in that supports the expiry datetime in free/busy periods." - period_columns = FreeBusyCollectionBase.period_columns + ["expires"] period_class = FreeBusyOfferPeriod def update_freebusy(self, periods, transp, uid, recurrenceid, summary, organiser, expires=None): @@ -526,7 +578,27 @@ """ FreeBusyCollectionBase.__init__(self, mutable) - self.periods = periods or [] + + if periods is not None: + self.periods = periods + else: + self.periods = [] + + def get_filename(self): + + "Return any filename for the periods collection." + + if hasattr(self.periods, "filename"): + return self.periods.filename + else: + return None + + def close(self): + + "Close the collection." + + if hasattr(self.periods, "close"): + self.periods.close() # List emulation methods. @@ -542,6 +614,11 @@ def __getitem__(self, i): return self.periods[i] + # Dictionary emulation methods (even though this is not a mapping). + + def clear(self): + del self.periods[:] + # Operations. def insert_period(self, period): diff -r a82b56e01721 -r 075f08595885 imiptools/freebusy/database.py --- a/imiptools/freebusy/database.py Fri May 26 23:58:06 2017 +0200 +++ b/imiptools/freebusy/database.py Thu Jun 01 23:26:38 2017 +0200 @@ -90,6 +90,8 @@ self.cursor = cursor self.table_name = table_name + self.period_columns = self.period_class.period_columns + # List emulation methods. def __nonzero__(self): @@ -116,6 +118,15 @@ def __getitem__(self, i): return list(iter(self))[i] + # Dictionary emulation methods (even though this is not a mapping). + + def clear(self): + query, values = self.get_query( + "delete from %(table)s :condition" % { + "table" : self.table_name + }) + self.cursor.execute(query, values) + # Operations. def insert_period(self, period): diff -r a82b56e01721 -r 075f08595885 imiptools/stores/database/common.py --- a/imiptools/stores/database/common.py Fri May 26 23:58:06 2017 +0200 +++ b/imiptools/stores/database/common.py Thu Jun 01 23:26:38 2017 +0200 @@ -31,6 +31,12 @@ def first(l): return l[0] +def have_table(obj, collection, table_name): + + "Return whether 'obj' is a 'collection' using the given 'table_name'." + + return isinstance(obj, collection) and obj.table_name == table_name + # Store classes. class DatabaseStoreBase(DatabaseOperations): @@ -463,42 +469,42 @@ # Free/busy period access. - def get_freebusy(self, user, name=None, mutable=False, cls=None): + def get_freebusy(self, user, name=None, mutable=False, collection=None): "Get free/busy details for the given 'user'." table = name or "freebusy" - cls = cls or FreeBusyDatabaseCollection - return cls(self.cursor, table, ["store_user"], [user], mutable, self.paramstyle) + collection = collection or FreeBusyDatabaseCollection + return collection(self.cursor, table, ["store_user"], [user], mutable, self.paramstyle) - def get_freebusy_for_other(self, user, other, mutable=False, cls=None): + def get_freebusy_for_other(self, user, other, mutable=False, collection=None): "For the given 'user', get free/busy details for the 'other' user." - cls = cls or FreeBusyDatabaseCollection - return cls(self.cursor, self.freebusy_other_table, ["store_user", "other"], [user, other], mutable, self.paramstyle) + collection = collection or FreeBusyDatabaseCollection + return collection(self.cursor, self.freebusy_other_table, ["store_user", "other"], [user, other], mutable, self.paramstyle) - def set_freebusy(self, user, freebusy, name=None, cls=None): + def set_freebusy(self, user, freebusy, name=None, collection=None): "For the given 'user', set 'freebusy' details." table = name or "freebusy" - cls = cls or FreeBusyDatabaseCollection + collection = collection or FreeBusyDatabaseCollection - if not isinstance(freebusy, cls) or freebusy.table_name != table: - fbc = cls(self.cursor, table, ["store_user"], [user], True, self.paramstyle) + if not have_table(freebusy, collection, table): + fbc = collection(self.cursor, table, ["store_user"], [user], True, self.paramstyle) fbc += freebusy return True - def set_freebusy_for_other(self, user, freebusy, other, cls=None): + def set_freebusy_for_other(self, user, freebusy, other, collection=None): "For the given 'user', set 'freebusy' details for the 'other' user." - cls = cls or FreeBusyDatabaseCollection + collection = collection or FreeBusyDatabaseCollection - if not isinstance(freebusy, cls) or freebusy.table_name != self.freebusy_other_table: - fbc = cls(self.cursor, self.freebusy_other_table, ["store_user", "other"], [user, other], True, self.paramstyle) + if not have_table(freebusy, collection, self.freebusy_other_table): + fbc = collection(self.cursor, self.freebusy_other_table, ["store_user", "other"], [user, other], True, self.paramstyle) fbc += freebusy return True @@ -544,7 +550,7 @@ "For the given 'user', set 'freebusy' offers." - return self.set_freebusy(user, freebusy, "freebusy_offers", cls=FreeBusyOffersDatabaseCollection) + return self.set_freebusy(user, freebusy, "freebusy_offers", collection=FreeBusyOffersDatabaseCollection) # Requests and counter-proposals. @@ -1037,9 +1043,9 @@ # Compatibility methods. def get_freebusy_for_other(self, user, other, mutable=False): - return DatabaseStore.get_freebusy_for_other(self, user, other, mutable, cls=FreeBusyGroupDatabaseCollection) + return DatabaseStore.get_freebusy_for_other(self, user, other, mutable, collection=FreeBusyGroupDatabaseCollection) def set_freebusy_for_other(self, user, freebusy, other): - return DatabaseStore.set_freebusy_for_other(self, user, freebusy, other, cls=FreeBusyGroupDatabaseCollection) + return DatabaseStore.set_freebusy_for_other(self, user, freebusy, other, collection=FreeBusyGroupDatabaseCollection) # vim: tabstop=4 expandtab shiftwidth=4 diff -r a82b56e01721 -r 075f08595885 imiptools/stores/file.py --- a/imiptools/stores/file.py Fri May 26 23:58:06 2017 +0200 +++ b/imiptools/stores/file.py Thu Jun 01 23:26:38 2017 +0200 @@ -26,15 +26,18 @@ from imiptools.data import Object, make_calendar, parse_object, to_stream from imiptools.dates import format_datetime, get_datetime, to_timezone from imiptools.filesys import fix_permissions, FileBase -from imiptools.freebusy import FreeBusyPeriod, FreeBusyGroupPeriod, \ - FreeBusyOfferPeriod, \ - FreeBusyCollection, \ + +from imiptools.freebusy import FreeBusyCollection, \ FreeBusyGroupCollection, \ - FreeBusyOffersCollection -from imiptools.text import get_table, set_defaults + FreeBusyOffersCollection, \ + period_from_tuple, \ + period_to_tuple + +from imiptools.text import FileTable, FileTableDict, FileTableSingle, \ + have_table + from os.path import isdir, isfile, join from os import listdir, remove, rmdir -import codecs # Obtain defaults from the settings. @@ -46,7 +49,7 @@ class FileStoreBase(FileBase): - "A file store supporting user-specific locking and tabular data." + "A file store supporting user-specific locking." def acquire_lock(self, user, timeout=None): FileBase.acquire_lock(self, timeout, user) @@ -54,104 +57,6 @@ def release_lock(self, user): FileBase.release_lock(self, user) - # Utility methods. - - def _set_defaults(self, t, empty_defaults): - return set_defaults(t, empty_defaults) - - def _get_table(self, filename, empty_defaults=None, tab_separated=True): - - """ - From the file having the given 'filename', return a list of tuples - representing the file's contents. - - The 'empty_defaults' is a list of (index, value) tuples indicating the - default value where a column either does not exist or provides an empty - value. - - If 'tab_separated' is specified and is a false value, line parsing using - the imiptools.text.parse_line function will be performed instead of - splitting each line of the file using tab characters as separators. - """ - - return get_table(filename, empty_defaults, tab_separated) - - def _get_table_atomic(self, user, filename, empty_defaults=None, tab_separated=True): - - """ - From the file for the given 'user' having the given 'filename', return - a list of tuples representing the file's contents. - - The 'empty_defaults' is a list of (index, value) tuples indicating the - default value where a column either does not exist or provides an empty - value. - - If 'tab_separated' is specified and is a false value, line parsing using - the imiptools.text.parse_line function will be performed instead of - splitting each line of the file using tab characters as separators. - """ - - self.acquire_lock(user) - try: - return self._get_table(filename, empty_defaults, tab_separated) - finally: - self.release_lock(user) - - def _set_table(self, filename, items, empty_defaults=None): - - """ - Write to the file having the given 'filename' the 'items'. - - The 'empty_defaults' is a list of (index, value) tuples indicating the - default value where a column either does not exist or provides an empty - value. - """ - - f = codecs.open(filename, "wb", encoding="utf-8") - try: - for item in items: - self._set_table_item(f, item, empty_defaults) - finally: - f.close() - fix_permissions(filename) - - def _set_table_item(self, f, item, empty_defaults=None): - - "Set in table 'f' the given 'item', using any 'empty_defaults'." - - if empty_defaults: - item = self._set_defaults(list(item), empty_defaults) - f.write("\t".join(item) + "\n") - - def _set_table_atomic(self, user, filename, items, empty_defaults=None): - - """ - For the given 'user', write to the file having the given 'filename' the - 'items'. - - The 'empty_defaults' is a list of (index, value) tuples indicating the - default value where a column either does not exist or provides an empty - value. - """ - - self.acquire_lock(user) - try: - self._set_table(filename, items, empty_defaults) - finally: - self.release_lock(user) - - def _set_freebusy(self, user, freebusy, filename): - - """ - For the given 'user', convert the 'freebusy' details to a form suitable - for writing to 'filename'. - """ - - # Obtain tuples from the free/busy objects. - - self._set_table_atomic(user, filename, - map(lambda fb: freebusy.make_tuple(fb.as_tuple(strings_only=True)), list(freebusy))) - class Store(FileStoreBase, StoreBase): "A file store of tabular free/busy data and objects." @@ -438,64 +343,86 @@ """ filename = self.get_object_in_store(user, "freebusy-providers") - if not filename or not isfile(filename): + if not filename: return None # Attempt to read providers, with a declaration of the datetime # from which such providers are considered as still being active. - t = self._get_table_atomic(user, filename, [(1, None)]) - try: - dt_string = t[0][0] - except IndexError: + t = self._get_freebusy_providers_table(filename) + header = t.get_header_values() + if not header: return None - return dt_string, t[1:] + return header[0], t + + def _get_freebusy_providers_table(self, filename): + + "Return a file-based table for storing providers in 'filename'." - def _set_freebusy_providers(self, user, dt_string, t): + return FileTable(filename, + in_defaults=[(1, None)], + out_defaults=[(1, "")], + headers=1) - "Set the given provider timestamp 'dt_string' and table 't'." + def _set_freebusy_providers(self, user, dt_string, providers): + + "Set the given provider timestamp 'dt_string' and 'providers'." filename = self.get_object_in_store(user, "freebusy-providers") if not filename: return False - t.insert(0, (dt_string,)) - self._set_table_atomic(user, filename, t, [(1, "")]) + self.acquire_lock(user) + try: + if not have_table(providers, filename): + pr = self._get_freebusy_providers_table(filename) + pr.replaceall(providers) + providers = pr + providers.set_header_values([dt_string]) + providers.close() + finally: + self.release_lock(user) return True # Free/busy period access. - def get_freebusy(self, user, name=None, mutable=False, cls=None): + def get_freebusy(self, user, name=None, mutable=False): "Get free/busy details for the given 'user'." filename = self.get_object_in_store(user, name or "freebusy") - if not filename or not isfile(filename): - periods = [] - else: - cls = cls or FreeBusyPeriod - periods = map(lambda t: cls(*t), - self._get_table_atomic(user, filename)) + if not filename: + return [] - return FreeBusyCollection(periods, mutable) + return self._get_freebusy(filename, mutable, FreeBusyCollection) - def get_freebusy_for_other(self, user, other, mutable=False, cls=None, collection=None): + def get_freebusy_for_other(self, user, other, mutable=False, collection=None): "For the given 'user', get free/busy details for the 'other' user." filename = self.get_object_in_store(user, "freebusy-other", other) - if not filename or not isfile(filename): - periods = [] - else: - cls = cls or FreeBusyPeriod - periods = map(lambda t: cls(*t), - self._get_table_atomic(user, filename)) + if not filename: + return [] + + return self._get_freebusy(filename, mutable, collection or FreeBusyCollection) + + def _get_freebusy(self, filename, mutable=False, collection=None): + + """ + Return a free/busy collection for 'filename' with the given 'mutable' + condition, employing the specified 'collection' class. + """ collection = collection or FreeBusyCollection - return collection(periods, mutable) + + periods = FileTable(filename, mutable=mutable, + in_converter=period_from_tuple(collection.period_class), + out_converter=period_to_tuple) + + return collection(periods, mutable=mutable) def set_freebusy(self, user, freebusy, name=None): @@ -505,10 +432,9 @@ if not filename: return False - self._set_freebusy(user, freebusy, filename) - return True + return self._set_freebusy(user, freebusy, filename) - def set_freebusy_for_other(self, user, freebusy, other): + def set_freebusy_for_other(self, user, freebusy, other, collection=None): "For the given 'user', set 'freebusy' details for the 'other' user." @@ -516,7 +442,24 @@ if not filename: return False - self._set_freebusy(user, freebusy, filename) + return self._set_freebusy(user, freebusy, filename, collection) + + def _set_freebusy(self, user, freebusy, filename, collection=None): + + "For the given 'user', set 'freebusy' details for the given 'filename'." + + # Copy to the specified table if different from that given. + + self.acquire_lock(user) + try: + if not have_table(freebusy, filename): + fbc = self._get_freebusy(filename, True, collection) + fbc += freebusy + freebusy = fbc + freebusy.close() + finally: + self.release_lock(user) + return True def get_freebusy_others(self, user): @@ -539,7 +482,11 @@ "Get free/busy offers for the given 'user'." - offers = [] + filename = self.get_object_in_store(user, "freebusy-offers") + + if not filename: + return [] + expired = [] now = to_timezone(datetime.utcnow(), "UTC") @@ -547,39 +494,43 @@ self.acquire_lock(user) try: - l = self.get_freebusy(user, "freebusy-offers", cls=FreeBusyOfferPeriod) - for fb in l: + offers = self._get_freebusy(filename, True, FreeBusyOffersCollection) + for fb in offers: if fb.expires and get_datetime(fb.expires) <= now: - expired.append(fb) - else: - offers.append(fb) - + offers.remove(fb) if expired: - self.set_freebusy_offers(user, offers) + offers.close() finally: self.release_lock(user) - return FreeBusyOffersCollection(offers, mutable) + offers.mutable = mutable + return offers # Requests and counter-proposals. - def _get_requests(self, user, queue): + def get_requests(self, user, queue="requests"): "Get requests for the given 'user' from the given 'queue'." filename = self.get_object_in_store(user, queue) - if not filename or not isfile(filename): + if not filename: return [] - return self._get_table_atomic(user, filename, [(1, None), (2, None)]) + return FileTable(filename, + in_defaults=[(1, None), (2, None)], + out_defaults=[(1, ""), (2, "")]) - def get_requests(self, user): + def set_request(self, user, uid, recurrenceid=None, type=None): - "Get requests for the given 'user'." + """ + For the given 'user', set the queued 'uid' and 'recurrenceid', + indicating a request, along with any given 'type'. + """ - return self._get_requests(user, "requests") + requests = self.get_requests(user) + return self.set_requests(user, [(uid, recurrenceid, type)]) - def _set_requests(self, user, requests, queue): + def set_requests(self, user, requests, queue="requests"): """ For the given 'user', set the list of queued 'requests' in the given @@ -590,47 +541,20 @@ if not filename: return False - self._set_table_atomic(user, filename, requests, [(1, ""), (2, "")]) - return True - - def set_requests(self, user, requests): - - "For the given 'user', set the list of queued 'requests'." - - return self._set_requests(user, requests, "requests") - - def _set_request(self, user, request, queue): - - """ - For the given 'user', set the given 'request' in the given 'queue'. - """ - - filename = self.get_object_in_store(user, queue) - if not filename: - return False + # Copy to the specified table if different from that given. self.acquire_lock(user) try: - f = codecs.open(filename, "ab", encoding="utf-8") - try: - self._set_table_item(f, request, [(1, ""), (2, "")]) - finally: - f.close() - fix_permissions(filename) + if not have_table(requests, filename): + req = self.get_requests(user, queue) + req.replaceall(requests) + requests = req + requests.close() finally: self.release_lock(user) return True - def set_request(self, user, uid, recurrenceid=None, type=None): - - """ - For the given 'user', set the queued 'uid' and 'recurrenceid', - indicating a request, along with any given 'type'. - """ - - return self._set_request(user, (uid, recurrenceid, type), "requests") - def get_counters(self, user, uid, recurrenceid=None): """ @@ -807,10 +731,10 @@ "Return a list of delegates for 'quota'." filename = self.get_object_in_store(quota, "delegates") - if not filename or not isfile(filename): + if not filename: return [] - return [value for (value,) in self._get_table_atomic(quota, filename)] + return FileTableSingle(filename) def set_delegates(self, quota, delegates): @@ -820,7 +744,16 @@ if not filename: return False - self._set_table_atomic(quota, filename, [(value,) for value in delegates]) + self.acquire_lock(quota) + try: + if not have_table(delegates, filename): + de = self.get_delegates(quota) + de.replaceall(delegates) + delegates = de + delegates.close() + finally: + self.release_lock(quota) + return True # Groups of users sharing quotas. @@ -830,10 +763,10 @@ "Return the identity mappings for the given 'quota' as a dictionary." filename = self.get_object_in_store(quota, "groups") - if not filename or not isfile(filename): + if not filename: return {} - return dict(self._get_table_atomic(quota, filename, tab_separated=False)) + return FileTableDict(filename, tab_separated=False) def set_groups(self, quota, groups): @@ -843,7 +776,16 @@ if not filename: return False - self._set_table_atomic(quota, filename, groups.items()) + self.acquire_lock(quota) + try: + if not have_table(groups, filename): + gr = self.get_groups(quota) + gr.updateall(groups) + groups = gr + groups.close() + finally: + self.release_lock(quota) + return True def get_limits(self, quota): @@ -854,10 +796,10 @@ """ filename = self.get_object_in_store(quota, "limits") - if not filename or not isfile(filename): + if not filename: return {} - return dict(self._get_table_atomic(quota, filename, tab_separated=False)) + return FileTableDict(filename, tab_separated=False) def set_limits(self, quota, limits): @@ -870,7 +812,16 @@ if not filename: return False - self._set_table_atomic(quota, filename, limits.items()) + self.acquire_lock(quota) + try: + if not have_table(limits, filename): + li = self.get_limits(quota) + li.updateall(limits) + limits = li + limits.close() + finally: + self.release_lock(quota) + return True # Journal entry methods. @@ -896,6 +847,9 @@ # Compatibility methods. def get_freebusy_for_other(self, user, other, mutable=False): - return Store.get_freebusy_for_other(self, user, other, mutable, cls=FreeBusyGroupPeriod, collection=FreeBusyGroupCollection) + return Store.get_freebusy_for_other(self, user, other, mutable, collection=FreeBusyGroupCollection) + + def set_freebusy_for_other(self, user, entries, other): + Store.set_freebusy_for_other(self, user, entries, other, collection=FreeBusyGroupCollection) # vim: tabstop=4 expandtab shiftwidth=4 diff -r a82b56e01721 -r 075f08595885 imiptools/text.py --- a/imiptools/text.py Fri May 26 23:58:06 2017 +0200 +++ b/imiptools/text.py Thu Jun 01 23:26:38 2017 +0200 @@ -19,47 +19,305 @@ this program. If not, see . """ +from imiptools.filesys import fix_permissions +from os.path import isfile import codecs import re -# Parsing of lines to obtain functions and arguments. +def have_table(obj, filename): + + "Return whether 'obj' is a table using the given 'filename'." + + return hasattr(obj, "get_filename") and obj.get_filename() == filename + +class FileTable: + + "A file-based data table." + + def __init__(self, filename, mutable=True, + in_defaults=None, out_defaults=None, + in_converter=None, out_converter=None, + tab_separated=True, headers=False): + + """ + Open the table from the file having the given 'filename'. If 'mutable' + is given as a true value (as is the default), the table can be modified. + + The 'in_defaults' is a list of (index, value) tuples indicating the + default value where a column either does not exist or provides an empty + value. The 'out_defaults' is a corresponding list used to serialise + missing and empty values. + + The 'in_converter' is a callable accepting a tuple of values and + returning an object. The corresponding 'out_converter' accepts an object + and returns a tuple of values. + + If 'tab_separated' is specified and is a false value, line parsing using + the imiptools.text.parse_line function will be performed instead of + splitting each line of the file using tab characters as separators. + + If 'headers' is specified and is not false, the first line in the table + will provide header value information. + """ + + self.filename = filename + self.mutable = mutable + self.in_defaults = in_defaults + self.out_defaults = out_defaults + self.in_converter = in_converter + self.out_converter = out_converter + self.tab_separated = tab_separated + + # Obtain the items. In subsequent implementations, the items could be + # retrieved dynamically. + + items = [] + + if isfile(filename): + for item in get_table(filename, in_defaults, tab_separated): + if self.in_converter: + item = self.in_converter(item) + items.append(item) + + # Obtain header values and separate them from the rest of the data. + + self.table = items[headers and 1 or 0:] + self.header_values = headers and items and items[0] or [] + self.headers = headers + + def get_filename(self): + return self.filename + + def get_header_values(self): + return self.header_values + + def set_header_values(self, values): + self.header_values = values + + def close(self): -line_pattern_str = ( - r"(?:" - r"(?:'(.*?)')" # single-quoted text - r"|" - r'(?:"(.*?)")' # double-quoted text - r"|" - r"([^\s]+)" # non-whitespace characters - r")+" - r"(?:\s+|$)" # optional trailing whitespace before line end - ) + "Write any modifications and close the table." + + if self.mutable: + f = codecs.open(self.filename, "wb", encoding="utf-8") + try: + sep = self.tab_separated and "\t" or " " + + # Include any headers in the output. + + if self.headers: + self.table.insert(0, self.header_values) + + for item in self.table: + if self.out_converter: + item = self.out_converter(item) + + # Insert defaults for empty columns. + + if self.out_defaults: + item = set_defaults(list(item), self.out_defaults) + + # Separate the columns and write to the file. + + print >>f, sep.join(item) + + # Remove the headers from the items in case the table is + # accessed again. + + if self.headers: + del self.table[0] + + finally: + f.close() + fix_permissions(self.filename) + + # General collection methods. -line_pattern = re.compile(line_pattern_str) + def __nonzero__(self): + return bool(self.table) + + # List emulation methods. + + def __iadd__(self, other): + for value in other: + self.append(value) + return self + + def __iter__(self): + return iter(self.table) + + def __len__(self): + return len(self.table) -def parse_line(text): + def __delitem__(self, i): + del self.table[i] + + def __delslice__(self, start, end): + del self.table[start:end] + + def __getitem__(self, i): + return self.table[i] + + def __getslice__(self, start, end): + return self.table[start:end] + + def __setitem__(self, i, value): + self.table[i] = value + + def __setslice__(self, start, end, values): + self.table[start:end] = values + + def append(self, value): + self.table.append(value) - """ - Parse the given 'text', returning a list of words separated by whitespace in - the input, where whitespace may occur inside words if quoted using single or - double quotes. + def insert(self, i, value): + self.table.insert(i, value) + + def remove(self, value): + self.table.remove(value) + + # Dictionary emulation methods (even though this is not a mapping). + + def clear(self): + del self.table[:] + + # Additional modification methods. + + def replaceall(self, values): + self.table[:] = values + +class FileTableDict(FileTable): + + "A file-based table acting as a dictionary." + + def __init__(self, filename, mutable=True, + in_defaults=None, out_defaults=None, + in_converter=None, out_converter=None, + tab_separated=True, headers=False): + + FileTable.__init__(self, filename, mutable, in_defaults, out_defaults, + in_converter, out_converter, tab_separated, headers) + self.mapping = dict(self.table) + + def close(self): + self.table = self.mapping.items() + FileTable.close(self) + + # General collection methods. - Hello world -> ['Hello', 'world'] - Hello ' world' -> ['Hello', ' world'] - Hello' 'world -> ["'Hello'", "'world'] - """ + def __nonzero__(self): + return bool(self.mapping) + + # List emulation methods. + + def __iter__(self): + return iter(self.mapping) + + def __len__(self): + return len(self.mapping) + + def append(self, value): + key, value = value + self.mapping[key] = value + + def insert(self, i, value): + self.append(value) + + def remove(self, value): + key, value = value + del self.mapping[key] + + # Unimplemented methods. + + def __delslice__(self, start, end): + raise NotImplementedError, "__delslice__" + + def __getslice__(self, start, end): + raise NotImplementedError, "__getslice__" + + def __setslice__(self, start, end, values): + raise NotImplementedError, "__setslice__" + + # Dictionary emulation methods. + + def clear(self): + self.mapping.clear() - parts = [] + def get(self, i, default=None): + return self.mapping.get(i, default) + + def keys(self): + return self.mapping.keys() + + def items(self): + return self.mapping.items() + + def update(self, other): + self.mapping.update(other) + + def values(self): + return self.mapping.values() + + def __delitem__(self, i): + del self.mapping[i] - # Match the components of each part. + def __getitem__(self, i): + return self.mapping[i] + + def __setitem__(self, i, value): + if self.mutable: + self.mapping[i] = value + + # Additional modification methods. - for match in line_pattern.finditer(text): + def replaceall(self, values): + self.mapping = {} + self.mapping.update(dict(values)) + + def updateall(self, mapping): + self.mapping = {} + self.mapping.update(mapping) + +def first(t): + return t[0] - # Combine the components by traversing the matching groups. +def tuplevalue(v): + return (v,) + +class FileTableSingle(FileTable): + + "A file-based table providing single value items." + + def __iter__(self): + return iter(self[:]) + + def __getitem__(self, i): + return self.table[i][0] + + def __getslice__(self, start, end): + return map(first, self.table[start:end]) + + def __setitem__(self, i, value): + self.table[i] = [(value,)] - parts.append(reduce(lambda a, b: (a or "") + (b or ""), match.groups())) + def __setslice__(self, start, end, values): + self.table[start:end] = map(tuplevalue, values) + + def append(self, value): + self.table.append((value,)) + + def insert(self, i, value): + self.table.insert(i, (value,)) - return parts + def remove(self, value): + self.table.remove((value,)) + + # Additional modification methods. + + def replaceall(self, values): + self.table[:] = map(tuplevalue, values) + + # Parsing of tabular files. @@ -129,4 +387,45 @@ return l + + +# Parsing of lines to obtain functions and arguments. + +line_pattern_str = ( + r"(?:" + r"(?:'(.*?)')" # single-quoted text + r"|" + r'(?:"(.*?)")' # double-quoted text + r"|" + r"([^\s]+)" # non-whitespace characters + r")+" + r"(?:\s+|$)" # optional trailing whitespace before line end + ) + +line_pattern = re.compile(line_pattern_str) + +def parse_line(text): + + """ + Parse the given 'text', returning a list of words separated by whitespace in + the input, where whitespace may occur inside words if quoted using single or + double quotes. + + Hello world -> ['Hello', 'world'] + Hello ' world' -> ['Hello', ' world'] + Hello' 'world -> ["'Hello'", "'world'] + """ + + parts = [] + + # Match the components of each part. + + for match in line_pattern.finditer(text): + + # Combine the components by traversing the matching groups. + + parts.append(reduce(lambda a, b: (a or "") + (b or ""), match.groups())) + + return parts + # vim: tabstop=4 expandtab shiftwidth=4 diff -r a82b56e01721 -r 075f08595885 tests/internal/file_store.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/internal/file_store.py Thu Jun 01 23:26:38 2017 +0200 @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +""" +Test file-based storage. + +Copyright (C) 2017 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +from imiptools.dates import get_datetime +from imiptools.freebusy.common import * +from imiptools.text import FileTable, FileTableDict, FileTableSingle +from imiptools.stores.file import Store + +# Test free/busy collection. + +fb = FreeBusyCollection(FileTable("testfb.tmp", + in_converter=period_from_tuple(FreeBusyPeriod), + out_converter=period_to_tuple)) +fb.clear() + +start = get_datetime("20170530T210700Z") +end = get_datetime("20170530T210900Z") +p1 = FreeBusyPeriod(start, end) +fb.insert_period(p1) + +start = get_datetime("20170530T205600Z") +end = get_datetime("20170530T205800Z") +p2 = FreeBusyPeriod(start, end) +fb.insert_period(p2) + +print list(fb) +fb.close() + +fb = FreeBusyCollection(FileTable("testfb.tmp", + in_converter=period_from_tuple(FreeBusyPeriod), + out_converter=period_to_tuple)) +print list(fb) + +print "----" + + + +# Test single value table. + +values = FileTableSingle("testsv.tmp") +values.clear() + +values.append("Hello") +values.insert(0, "world") + +print list(values) +values.close() + +values = FileTableSingle("testsv.tmp") +print list(values) + +print "----" + + + +# Test dictionary table. + +limits = FileTableDict("testdt.tmp") +limits.clear() + +limits["mailto:paul.boddie@example.com"] = "PT1H" + +print list(limits) +limits.close() + +limits = FileTableDict("testdt.tmp") +print list(limits) + +print "----" + + + +# Test store. + +s = Store("store.tmp") + +fb = s.get_freebusy("mailto:paul.boddie@example.com") +try: + fb.insert_period(p1) +except TypeError: + print "Free/busy collection not mutable, as expected." + +fb = s.get_freebusy("mailto:paul.boddie@example.com", mutable=True) +fb.insert_period(p1) +fb.insert_period(p2) +s.set_freebusy("mailto:paul.boddie@example.com", fb) +s.set_freebusy("mailto:harvey.horse@example.com", fb) + +print list(fb) + +s = Store("store.tmp") + +fb = s.get_freebusy("mailto:paul.boddie@example.com") +print list(fb) +fb = s.get_freebusy("mailto:harvey.horse@example.com") +print list(fb) + + + +# Test store. + +s = Store("store.tmp") + +req = s.get_requests("mailto:paul.boddie@example.com") +req.clear() + +req.append(("uid1@example.com", None, None)) +req.append(("uid2@example.com", None, None)) +req.append(("uid2@example.com", "20170531T140100Z", None)) +req.append(("uid2@example.com", "20170531T140900Z", "COUNTER")) +s.set_requests("mailto:paul.boddie@example.com", req) +s.set_requests("mailto:harvey.horse@example.com", req) + +print list(req) + +s = Store("store.tmp") + +req = s.get_requests("mailto:paul.boddie@example.com") +print list(req) +req = s.get_requests("mailto:harvey.horse@example.com") +print list(req) + + + +# Test store. + +s = Store("store.tmp") + +fb = s.get_freebusy_for_other("mailto:paul.boddie@example.com", "mailto:harvey.horse@example.com", mutable=True) +fb.clear() +fb.insert_period(p1) +fb.insert_period(p2) +s.set_freebusy_for_other("mailto:paul.boddie@example.com", fb, "mailto:harvey.horse@example.com") + +print list(fb) + +s = Store("store.tmp") + +fb = s.get_freebusy_for_other("mailto:paul.boddie@example.com", "mailto:harvey.horse@example.com") +print list(fb) + +# vim: tabstop=4 expandtab shiftwidth=4