# HG changeset patch # User Paul Boddie # Date 1461273657 -7200 # Node ID 649c0b1cc0fd722c5abd39a099645d30ef444abb # Parent e7e62c2e1598040e09b30c6e91becc97d470ab9a Added a store-copying tool together with additional store methods to support the complete copying of one store into another. Updated the get_all_events methods to be able to return either active or cancelled event and recurrence details. Tidied various store method return types. diff -r e7e62c2e1598 -r 649c0b1cc0fd imiptools/stores/common.py --- a/imiptools/stores/common.py Thu Apr 21 23:17:35 2016 +0200 +++ b/imiptools/stores/common.py Thu Apr 21 23:20:57 2016 +0200 @@ -35,26 +35,57 @@ # Event and event metadata access. + def get_all_events(self, user, dirname=None): + + """ + Return a set of (uid, recurrenceid) tuples for all events. Unless + 'dirname' is specified, only active events are returned; otherwise, + events from the given 'dirname' are returned. + """ + + cancelled = self.get_cancelled_events(user) + active = self.get_events(user) + + if dirname == "cancellations": + uids = cancelled + else: + uids = active + + if not uids: + return set() + + all_events = set() + + # Add all qualifying parent events to the result set. + + for uid in uids: + all_events.add((uid, None)) + + # Process all parent events regardless of status to find those in the + # category/directory of interest. + + for uid in active + cancelled: + + if dirname == "cancellations": + recurrenceids = self.get_cancelled_recurrences(user, uid) + else: + recurrenceids = self.get_active_recurrences(user, uid) + + all_events.update([(uid, recurrenceid) for recurrenceid in recurrenceids]) + + return all_events + def get_events(self, user): "Return a list of event identifiers." pass - def get_all_events(self, user): - - "Return a set of (uid, recurrenceid) tuples for all events." + def get_cancelled_events(self, user): - uids = self.get_events(user) - if not uids: - return set() + "Return a list of event identifiers for cancelled events." - all_events = set() - for uid in uids: - all_events.add((uid, None)) - all_events.update([(uid, recurrenceid) for recurrenceid in self.get_recurrences(user, uid)]) - - return all_events + pass def get_event(self, user, uid, recurrenceid=None, dirname=None): @@ -301,13 +332,20 @@ def get_freebusy_for_update(self, user, name=None): - "Get free/busy details for the given 'user'." + """ + Get free/busy details for the given 'user' that may be updated, + potentially affecting the stored information directly. + """ return self.get_freebusy(user, name, True) def get_freebusy_for_other_for_update(self, user, other): - "For the given 'user', get free/busy details for the 'other' user." + """ + For the given 'user', get free/busy details for the 'other' user + that may be updated, potentially affecting the stored information + directly. + """ return self.get_freebusy_for_other(user, other, True) @@ -323,6 +361,15 @@ pass + def get_freebusy_others(self, user): + + """ + For the given 'user', return a list of other users for whom free/busy + information is retained. + """ + + pass + # Tentative free/busy periods related to countering. def get_freebusy_offers(self, user, mutable=False): @@ -333,7 +380,10 @@ def get_freebusy_offers_for_update(self, user): - "Get free/busy offers for the given 'user'." + """ + Get free/busy offers for the given 'user' that may be updated, + potentially affecting the stored information directly. + """ return self.get_freebusy_offers(user, True) @@ -552,6 +602,14 @@ pass + def set_group(self, quota, store_user, user_group): + + """ + For the given 'quota', set a mapping from 'store_user' to 'user_group'. + """ + + pass + def get_limits(self, quota): """ @@ -572,6 +630,15 @@ # Free/busy period access for users within quota groups. + def get_freebusy_users(self, quota): + + """ + Return a list of users whose free/busy details are retained for the + given 'quota'. + """ + + pass + def get_freebusy(self, quota, user, mutable=False): "Get free/busy details for the given 'quota' and 'user'." @@ -580,7 +647,10 @@ def get_freebusy_for_update(self, quota, user): - "Get free/busy details for the given 'quota' and 'user'." + """ + Get free/busy details for the given 'quota' and 'user' that may be + updated, potentially affecting the stored information directly. + """ return self.get_freebusy(quota, user, True) @@ -605,7 +675,8 @@ """ Return a list of journal entries for the given 'quota' for the indicated - 'group'. + 'group' that may be updated, potentially affecting the stored + information directly. """ return self.get_entries(quota, group, True) diff -r e7e62c2e1598 -r 649c0b1cc0fd imiptools/stores/database/common.py --- a/imiptools/stores/database/common.py Thu Apr 21 23:17:35 2016 +0200 +++ b/imiptools/stores/database/common.py Thu Apr 21 23:20:57 2016 +0200 @@ -58,12 +58,36 @@ # Event and event metadata access. - def get_events(self, user): + def get_all_events(self, user, dirname=None): + + """ + Return a set of (uid, recurrenceid) tuples for all events. Unless + 'dirname' is specified, only active events are returned; otherwise, + events from the given 'dirname' are returned. + """ + + columns, values = self.get_event_table_filters(dirname) + + columns += ["store_user"] + values += [user] + + query, values = self.get_query( + "select object_uid, null as object_recurrenceid from objects :condition " + "union all " + "select object_uid, object_recurrenceid from recurrences :condition", + columns, values) + + self.cursor.execute(query, values) + return self.cursor.fetchall() + + def get_events(self, user, dirname=None): "Return a list of event identifiers." - columns = ["store_user", "status"] - values = [user, "active"] + columns, values = self.get_event_table_filters(dirname) + + columns += ["store_user"] + values += [user] query, values = self.get_query( "select object_uid from objects :condition", @@ -72,36 +96,11 @@ self.cursor.execute(query, values) return [r[0] for r in self.cursor.fetchall()] - def get_all_events(self, user): - - "Return a set of (uid, recurrenceid) tuples for all events." - - query, values = self.get_query( - "select object_uid, null as object_recurrenceid from objects :condition " - "union all " - "select object_uid, object_recurrenceid from recurrences :condition", - ["store_user"], [user]) - - self.cursor.execute(query, values) - return self.cursor.fetchall() - - def get_event_table(self, recurrenceid=None, dirname=None): + def get_cancelled_events(self, user): - "Get the table providing events for any specified 'dirname'." - - if recurrenceid: - return self.get_recurrence_table(dirname) - else: - return self.get_complete_event_table(dirname) + "Return a list of event identifiers for cancelled events." - def get_event_table_filters(self, dirname=None): - - "Get filter details for any specified 'dirname'." - - if dirname == "cancellations": - return ["status"], ["cancelled"] - else: - return ["status"], ["active"] + return self.get_events(user, "cancellations") def get_event(self, user, uid, recurrenceid=None, dirname=None): @@ -131,15 +130,6 @@ result = self.cursor.fetchone() return result and parse_string(result[0], "utf-8") - def get_complete_event_table(self, dirname=None): - - "Get the table providing events for any specified 'dirname'." - - if dirname == "counters": - return "countered_objects" - else: - return "objects" - def get_complete_event(self, user, uid): "Get the event for the given 'user' with the given 'uid'." @@ -231,15 +221,6 @@ self.cursor.execute(query, values) return [t[0] for t in self.cursor.fetchall() or []] - def get_recurrence_table(self, dirname=None): - - "Get the table providing recurrences for any specified 'dirname'." - - if dirname == "counters": - return "countered_recurrences" - else: - return "recurrences" - def get_recurrence(self, user, uid, recurrenceid): """ @@ -320,6 +301,44 @@ self.cursor.execute(query, values) return True + # Event table computation. + + def get_event_table(self, recurrenceid=None, dirname=None): + + "Get the table providing events for any specified 'dirname'." + + if recurrenceid: + return self.get_recurrence_table(dirname) + else: + return self.get_complete_event_table(dirname) + + def get_event_table_filters(self, dirname=None): + + "Get filter details for any specified 'dirname'." + + if dirname == "cancellations": + return ["status"], ["cancelled"] + else: + return ["status"], ["active"] + + def get_complete_event_table(self, dirname=None): + + "Get the table providing events for any specified 'dirname'." + + if dirname == "counters": + return "countered_objects" + else: + return "objects" + + def get_recurrence_table(self, dirname=None): + + "Get the table providing recurrences for any specified 'dirname'." + + if dirname == "counters": + return "countered_recurrences" + else: + return "recurrences" + # Free/busy period providers, upon extension of the free/busy records. def _get_freebusy_providers(self, user): @@ -444,6 +463,23 @@ return True + def get_freebusy_others(self, user): + + """ + For the given 'user', return a list of other users for whom free/busy + information is retained. + """ + + columns = ["store_user"] + values = [user] + + query, values = self.get_query( + "select distinct other from freebusy_others :condition", + columns, values) + + self.cursor.execute(query, values) + return [r[0] for r in self.cursor.fetchall()] + # Tentative free/busy periods related to countering. def get_freebusy_offers(self, user, mutable=False): @@ -812,6 +848,36 @@ self.cursor.execute(query, values) return dict(self.cursor.fetchall()) + def set_group(self, quota, store_user, user_group): + + """ + For the given 'quota', set a mapping from 'store_user' to 'user_group'. + """ + + columns = ["quota", "store_user"] + values = [quota, store_user] + setcolumns = ["user_group"] + setvalues = [user_group] + + query, values = self.get_query( + "update user_groups :set :condition", + columns, values, setcolumns, setvalues) + + self.cursor.execute(query, values) + + if self.cursor.rowcount > 0: + return True + + columns = ["quota", "store_user", "user_group"] + values = [quota, store_user, user_group] + + query, values = self.get_query( + "insert into user_groups (:columns) values (:values)", + columns, values) + + self.cursor.execute(query, values) + return True + def get_limits(self, quota): """ @@ -862,6 +928,23 @@ # Free/busy period access for users within quota groups. + def get_freebusy_users(self, quota): + + """ + Return a list of users whose free/busy details are retained for the + given 'quota'. + """ + + columns = ["quota"] + values = [quota] + + query, values = self.get_query( + "select distinct store_user from user_freebusy :condition", + columns, values) + + self.cursor.execute(query) + return [r[0] for r in self.cursor.fetchall()] + def get_freebusy(self, quota, user, mutable=False): "Get free/busy details for the given 'quota' and 'user'." diff -r e7e62c2e1598 -r 649c0b1cc0fd imiptools/stores/file.py --- a/imiptools/stores/file.py Thu Apr 21 23:17:35 2016 +0200 +++ b/imiptools/stores/file.py Thu Apr 21 23:20:57 2016 +0200 @@ -232,25 +232,19 @@ filename = self.get_object_in_store(user, "objects") if not filename or not isdir(filename): - return None + return [] return [name for name in listdir(filename) if isfile(join(filename, name))] - def get_event_filename(self, user, uid, recurrenceid=None, dirname=None, username=None): + def get_cancelled_events(self, user): - """ - Get the filename providing the event for the given 'user' with the given - 'uid'. If the optional 'recurrenceid' is specified, a specific instance - or occurrence of an event is returned. + "Return a list of event identifiers for cancelled events." - Where 'dirname' is specified, the given directory name is used as the - base of the location within which any filename will reside. - """ + filename = self.get_object_in_store(user, "cancellations", "objects") + if not filename or not isdir(filename): + return [] - if recurrenceid: - return self.get_recurrence_filename(user, uid, recurrenceid, dirname, username) - else: - return self.get_complete_event_filename(user, uid, dirname, username) + return [name for name in listdir(filename) if isfile(join(filename, name))] def get_event(self, user, uid, recurrenceid=None, dirname=None): @@ -266,21 +260,6 @@ return filename and self._get_object(user, filename) - def get_complete_event_filename(self, user, uid, dirname=None, username=None): - - """ - Get the filename providing the event for the given 'user' with the given - 'uid'. - - Where 'dirname' is specified, the given directory name is used as the - base of the location within which any filename will reside. - - Where 'username' is specified, the event details will reside in a file - bearing that name within a directory having 'uid' as its name. - """ - - return self.get_object_in_store(user, dirname, "objects", uid, username) - def get_complete_event(self, user, uid): "Get the event for the given 'user' with the given 'uid'." @@ -346,21 +325,6 @@ return [name for name in listdir(filename) if isfile(join(filename, name))] - def get_recurrence_filename(self, user, uid, recurrenceid, dirname=None, username=None): - - """ - For the event of the given 'user' with the given 'uid', return the - filename providing the recurrence with the given 'recurrenceid'. - - Where 'dirname' is specified, the given directory name is used as the - base of the location within which any filename will reside. - - Where 'username' is specified, the event details will reside in a file - bearing that name within a directory having 'uid' as its name. - """ - - return self.get_object_in_store(user, dirname, "recurrences", uid, recurrenceid, username) - def get_recurrence(self, user, uid, recurrenceid): """ @@ -410,6 +374,54 @@ return True + # Event filename computation. + + def get_event_filename(self, user, uid, recurrenceid=None, dirname=None, username=None): + + """ + Get the filename providing the event for the given 'user' with the given + 'uid'. If the optional 'recurrenceid' is specified, a specific instance + or occurrence of an event is returned. + + Where 'dirname' is specified, the given directory name is used as the + base of the location within which any filename will reside. + """ + + if recurrenceid: + return self.get_recurrence_filename(user, uid, recurrenceid, dirname, username) + else: + return self.get_complete_event_filename(user, uid, dirname, username) + + def get_recurrence_filename(self, user, uid, recurrenceid, dirname=None, username=None): + + """ + For the event of the given 'user' with the given 'uid', return the + filename providing the recurrence with the given 'recurrenceid'. + + Where 'dirname' is specified, the given directory name is used as the + base of the location within which any filename will reside. + + Where 'username' is specified, the event details will reside in a file + bearing that name within a directory having 'uid' as its name. + """ + + return self.get_object_in_store(user, dirname, "recurrences", uid, recurrenceid, username) + + def get_complete_event_filename(self, user, uid, dirname=None, username=None): + + """ + Get the filename providing the event for the given 'user' with the given + 'uid'. + + Where 'dirname' is specified, the given directory name is used as the + base of the location within which any filename will reside. + + Where 'username' is specified, the event details will reside in a file + bearing that name within a directory having 'uid' as its name. + """ + + return self.get_object_in_store(user, dirname, "objects", uid, username) + # Free/busy period providers, upon extension of the free/busy records. def _get_freebusy_providers(self, user): @@ -502,6 +514,20 @@ map(lambda fb: fb.as_tuple(strings_only=True), freebusy.periods)) return True + def get_freebusy_others(self, user): + + """ + For the given 'user', return a list of other users for whom free/busy + information is retained. + """ + + filename = self.get_object_in_store(user, "freebusy-other") + + if not filename or not isdir(filename): + return [] + + return listdir(filename) + # Tentative free/busy periods related to countering. def get_freebusy_offers(self, user, mutable=False): @@ -538,7 +564,7 @@ filename = self.get_object_in_store(user, queue) if not filename or not isfile(filename): - return None + return [] return self._get_table_atomic(user, filename, [(1, None), (2, None)]) @@ -609,7 +635,7 @@ filename = self.get_event_filename(user, uid, recurrenceid, "counters") if not filename or not isdir(filename): - return False + return [] return [name for name in listdir(filename) if isfile(join(filename, name))] @@ -622,7 +648,7 @@ filename = self.get_event_filename(user, uid, recurrenceid, "counters", other) if not filename or not isfile(filename): - return False + return None return self._get_object(user, filename) @@ -797,6 +823,22 @@ return dict(self._get_table_atomic(quota, filename, tab_separated=False)) + def set_group(self, quota, store_user, user_group): + + """ + For the given 'quota', set a mapping from 'store_user' to 'user_group'. + """ + + filename = self.get_object_in_store(quota, "groups") + if not filename: + return False + + groups = self.get_groups(quota) or {} + groups[store_user] = user_group + + self._set_table_atomic(quota, filename, groups.items()) + return True + def get_limits(self, quota): """ @@ -806,7 +848,7 @@ filename = self.get_object_in_store(quota, "limits") if not filename or not isfile(filename): - return None + return {} return dict(self._get_table_atomic(quota, filename, tab_separated=False)) @@ -819,7 +861,7 @@ filename = self.get_object_in_store(quota, "limits") if not filename: - return None + return False limits = self.get_limits(quota) or {} limits[group] = limit @@ -829,6 +871,19 @@ # Free/busy period access for users within quota groups. + def get_freebusy_users(self, quota): + + """ + Return a list of users whose free/busy details are retained for the + given 'quota'. + """ + + filename = self.get_object_in_store(quota, "freebusy") + if not filename or not isdir(filename): + return [] + + return listdir(filename) + def get_freebusy(self, quota, user, mutable=False): "Get free/busy details for the given 'quota' and 'user'." diff -r e7e62c2e1598 -r 649c0b1cc0fd tools/copy_store.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/copy_store.py Thu Apr 21 23:20:57 2016 +0200 @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +""" +Copy store information into another store. + +Copyright (C) 2014, 2015, 2016 Paul Boddie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +from os.path import abspath, split +import sys + +# Find the modules. + +try: + import imiptools +except ImportError: + parent = abspath(split(split(__file__)[0])[0]) + if split(parent)[1] == "imip-agent": + sys.path.append(parent) + +from imiptools import config +from imiptools.data import Object +from imiptools.stores import get_store, get_publisher, get_journal + +def copy_store(from_store, from_journal, to_store, to_journal): + + """ + Copy stored information from the specified 'from_store' and 'from_journal' + to the specified 'to_store' and 'to_journal' respectively. + """ + + # For each user... + + for user in from_store.get_users(): + + # Copy requests. + + to_store.set_requests(user, from_store.get_requests(user)) + + # Copy events, both active and cancellations. + + for dirname in (None, "cancellations"): + + # Get event, recurrence information. + + for uid, recurrenceid in from_store.get_all_events(user, dirname=dirname): + d = from_store.get_event(user, uid, recurrenceid, dirname=dirname) + if d: + to_store.set_event(user, uid, recurrenceid, Object(d).to_node()) + if dirname == "cancellations": + to_store.cancel_event(user, uid, recurrenceid) + else: + print >>sys.stderr, "Event for %s with UID %s and RECURRENCE-ID %s not found in %s" % ( + (user, uid, recurrenceid or "null", dirname or "active events")) + + # Copy counter-proposals. + + if dirname is None: + for other in from_store.get_counters(user, uid, recurrenceid): + d = from_store.get_counter(user, other, uid, recurrenceid) + if d: + to_store.set_counter(user, other, Object(d).to_node(), uid, recurrenceid) + else: + print >>sys.stderr, "Counter-proposal for %s with UID %s and RECURRENCE-ID %s not found in %s" % ( + (user, uid, recurrenceid or "null", dirname or "active events")) + + # Copy free/busy information for the user. + + to_store.set_freebusy(user, from_store.get_freebusy(user)) + + # Copy free/busy information for other users. + + for other in from_store.get_freebusy_others(user): + to_store.set_freebusy_for_other(user, from_store.get_freebusy_for_other(user, other), other) + + # Copy free/busy offers. + + to_store.set_freebusy_offers(user, from_store.get_freebusy_offers(user)) + + # For each quota group... + + for quota in from_journal.get_quotas(): + + # Copy quota limits. + + for user_group, limit in from_journal.get_limits(quota).items(): + to_journal.set_limit(quota, user_group, limit) + + # Copy group mappings. + + for store_user, user_group in from_journal.get_groups(quota).items(): + to_journal.set_group(quota, store_user, user_group) + + # Copy journal details. + + for group in from_journal.get_quota_users(quota): + to_journal.set_entries(quota, group, from_journal.get_entries(quota, group)) + + # Copy individual free/busy details. + + for store_user in from_journal.get_freebusy_users(quota): + to_journal.set_freebusy(store_user, from_journal.get_freebusy(store_user)) + +# Main program. + +if __name__ == "__main__": + + # Interpret the command line arguments. + + from_store_args = [] + to_store_args = [] + l = ignored = [] + + for arg in sys.argv[1:]: + if arg in ("-t", "--to"): + l = to_store_args + elif arg in ("-f", "--from"): + l = from_store_args + else: + l.append(arg) + + if len(from_store_args) not in (0, 3) or len(to_store_args) != 3: + print >>sys.stderr, """\ +Usage: %s \\ + [ ( -f | --from ) ] \\ + ( -t | --to ) + +Need details of a destination store indicated by the -t or --to option. +In addition, details of a source store may be indicated by the -f or --from +option; otherwise, the currently-configured store is used. +""" % split(sys.argv[0])[1] + sys.exit(1) + + # Override defaults if indicated. + + getvalue = lambda value, pos=0, default=None: value and value[pos] or default + + from_store_type = getvalue(from_store_args, 0, config.STORE_TYPE) + from_store_dir = getvalue(from_store_args, 1) + from_journal_dir = getvalue(from_store_args, 2) + + to_store_type, to_store_dir, to_journal_dir = to_store_args + + # Obtain store-related objects. + + from_store = get_store(from_store_type, from_store_dir) + from_journal = get_journal(from_store_type, from_journal_dir) + + to_store = get_store(to_store_type, to_store_dir) + to_journal = get_journal(to_store_type, to_journal_dir) + + # Process the store. + + copy_store(from_store, from_journal, to_store, to_journal) + +# vim: tabstop=4 expandtab shiftwidth=4 diff -r e7e62c2e1598 -r 649c0b1cc0fd tools/install.sh --- a/tools/install.sh Thu Apr 21 23:17:35 2016 +0200 +++ b/tools/install.sh Thu Apr 21 23:20:57 2016 +0200 @@ -103,7 +103,7 @@ # Tools -TOOLS="fix.sh init.sh init_user.sh make_freebusy.py set_quota_limit.py update_quotas.py update_scheduling_modules.py" +TOOLS="copy_store.py fix.sh init.sh init_user.sh make_freebusy.py set_quota_limit.py update_quotas.py update_scheduling_modules.py" if [ ! -e "$INSTALL_DIR/tools" ]; then mkdir -p "$INSTALL_DIR/tools"