#!/usr/bin/env python

"""

Confluence XML dump conversion to a MoinMoin-compatible representation.

Copyright (C) 2012, 2013 Paul Boddie <paul@boddie.org.uk>

This software is free software; you can redistribute it and/or

modify it under the terms of the GNU General Public License as

published by the Free Software Foundation; either version 2 of

the License, or (at your option) any later version.

This software is distributed in the hope that it will be useful,

but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

GNU General Public License for more details.

You should have received a copy of the GNU General Public

License along with this library; see the file LICENCE.txt

If not, write to the Free Software Foundation, Inc.,

51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA

"""

from os import listdir, mkdir, makedirs

from os.path import exists, extsep, join, splitext

from zipfile import ZipFile

from cStringIO import StringIO

import codecs

import xmlread

import parser

import sys

MAX_TITLE_LENGTH = 120

class ConfluenceHandler:

    "Handle content from a Confluence Wiki dump."

    def __init__(self, space, no_translate=False):

        self.content = {}

        self.elements = []

        self.space = space

        self.no_translate = no_translate

    def handle_object(self, name, elements, attributes, all_text, text):

        "Handle objects according to type."

        objecttype = attributes[-1]["class"]

        # Any identifier is stored as the object's textual content.

        identifier = text.strip()

        # The content is a dictionary mapping names to properties and

        # collections.

        content = self.content

        pages_dir = join(self.space, "pages")

        versions_dir = join(self.space, "versions")

        # Handle particular types.

        if objecttype in ("Page", "Comment", "BlogPost"):

            # Handle pages and revisions, adding revisions to the page manifest.

            # The original version is used as a unifying identifier for all the

            # different revisions (each of which being defined by a Page

            # element). Although "original" implies the first identifier used,

            # it actually appears to be the latest and will have the highest

            # version number.

            if content.has_key("originalVersion"):

                pageid = content["originalVersion"]

            else:

                pageid = identifier

            versionfile = join(versions_dir, identifier)

            # Note page metadata, not necessarily in the correct order.

            # For comments, the title will need to be rewritten, since they

            # should be defined in terms of their owner page.

            mkdirs(join(pages_dir, pageid))

            title = content["title"]

            # Limit the title to a "safe" number of characters in order to avoid

            # filesystem issues.

            title = title[:MAX_TITLE_LENGTH]

            if title:

                title = "%s/%s" % (self.space, title)

                write(join(pages_dir, pageid, "pagetitle"), title)

            # See sort_manifest for access to this data.

            append(join(pages_dir, pageid, "manifest"),

                "%s|AddRevision|%s|%s|%s|%s\n" % (

                    content["version"],

                    versionfile,

                    title, # comment titles will incorporate the comment's position

                    content["lastModifierName"],

                    content["versionComment"]

                ))

            # Add information to parent pages for child page lists.

            if content.has_key("parent"):

                parentid = content["parent"]

                mkdirs(join(pages_dir, parentid))

                append(join(pages_dir, parentid, "children"), title + "\n")

            # Add creation details for comments to the owner page.

            # Since comments can be versioned, the date of the original version

            # is used, and only this "original" version has the owner property.

            if objecttype == "Comment" and content.has_key("owner"):

                ownerid = content["owner"]

                mkdirs(join(pages_dir, ownerid))

                append(join(pages_dir, ownerid, "comments"), "%s|%s\n" % (content["creationDate"], pageid))

            # Some metadata is not particularly relevant. For example,

            # ancestors, children, parent are navigation-related.

            # Other metadata could be added to the page content itself.

            # For example, labelling could be converted to categories.

        # Handle revisions.

        elif objecttype == "BodyContent":

            body = content["body"]

            if not body:

                body = "## Empty page."

            # NOTE: Very simple technique employed for guessing the format.

            if no_translate:

                fn = write

            elif body.startswith("<"):

                fn = xmltranslate

            else:

                fn = translate

            try:

                fn(join(versions_dir, content["content"]), body)

            except:

                print >>sys.stderr, "Error parsing..."

                print >>sys.stderr, body

                raise

        self.content = {}

    def handle_property(self, name, elements, attributes, all_text, text):

        "Record properties in the current content dictionary."

        self.content[attributes[-1]["name"]] = text.strip()

    def handle_id(self, name, elements, attributes, all_text, text):

        "Promote identifiers to the parent element's text."

        all_text[-2].append(text)

    def handle_collection(self, name, elements, attributes, all_text, text):

        "Record collections in the current content dictionary."

        self.content[attributes[-1]["name"]] = self.elements

        self.elements = []

    def handle_element(self, name, elements, attributes, all_text, text):

        "Add elements to the current collection."

        self.elements.append((attributes[-1]["class"], text.strip()))

def mkdirs(name):

    "Make the directory with the given 'name' at any depth."

    try:

        makedirs(name)

    except OSError:

        pass

def append(filename, s):

    "Append to the file with the given 'filename' the string 's'."

    write(filename, s, True)

def write(filename, s, append=False):

    """

    Write to the file with the given 'filename' the string 's'. If the optional

    'append' parameter is set to a true value, 's' will be appended to the file.

    """

    f = codecs.open(filename, append and "a" or "w", encoding="utf-8")

    try:

        f.write(s)

    finally:

        f.close()

def read(filename):

    """

    Read from the file with the given 'filename', returning a string containing

    its contents.

    """

    f = codecs.open(filename, encoding="utf-8")

    try:

        return f.read()

    finally:

        f.close()

def translate(filename, body, fn=None):

    """

    Write to the file with the given 'filename' a translation of the given

    'body'.

    """

    fn = fn or parser.parse

    out = codecs.open(filename, "w", encoding="utf-8")

    try:

        fn(body, out)

    finally:

        out.close()

def xmltranslate(filename, body):

    translate(filename, body, parser.xmlparse)

def sort_comments(pages_dir, pageid):

    """

    Where 'pageid' has comments associated with it, sort them chronologically

    and label the comment pages with the owner page's title and comment's

    position in the chronological sequence. Such labelling is done by writing

    a "pagetitle" file in each comment page's directory.

    """

    comments = join(pages_dir, pageid, "comments")

    if not exists(comments):

        return

    title = read(join(pages_dir, pageid, "pagetitle"))

    details = [line.split("|") for line in read(comments).split("\n") if line]

    details.sort()

    # Write the sorted comments list for testing purposes.

    write(comments, "\n".join(["|".join(x) for x in details]))

    # Define comments as subpages by setting their titles using this

    # page's name/title and their position in the comments collection.

    for position, (_lastmodified, commentid) in enumerate(details):

        # In the page directory for each comment, write the page title in a

        # special file for later processing.

        write(join(pages_dir, commentid, "pagetitle"), "%s/%04d" % (title, position))

def sort_manifest(pages_dir, pageid, output=None, no_translate=False):

    """

    Using the given 'pageid', locate the manifest for the page and any page

    title information written to a "pagetitle" file.

    Then sort the manifest according to revision so that it will be added to

    MoinMoin in the correct order.

    If a "pagetitle" file exists, the title column in the manifest will be

    augmented with the contents of that file. This is typically done for

    comments.

    If a "children" file exists, the pages in that file will be added as a list

    to the end of each revision's content.

    If 'output' is given, the manifest details will be appended to the file

    having that filename instead of being rewritten to the original manifest

    file.

    """

    manifest = join(pages_dir, pageid, "manifest")

    pagetitle = join(pages_dir, pageid, "pagetitle")

    children = join(pages_dir, pageid, "children")

    comments = join(pages_dir, pageid, "comments")

    if exists(pagetitle):

        title = read(pagetitle)

    else:

        title = None

    f = codecs.open(manifest, "r", encoding="utf-8")

    try:

        lines = [x.split("|") for x in f.readlines()]

        lines.sort(cmp=lambda x, y: cmp(int(x[0]), int(y[0])))

        # Reconstruct the lines, optionally changing the titles.

        result = []

        for line in lines:

            version, _addrevision, filename, old_title, username, comment = line

            # Replace title information with the information already present.

            if title is not None:

                new_title = title

            else:

                new_title = old_title

            # The version is omitted now that the manifest is ordered.

            line = _addrevision, filename, new_title, username, comment

            result.append("|".join(line))

            # Add child page information to the content.

            if exists(children) and not no_translate:

                child_pages = []

                child_page_names = [x for x in read(children).split("\n") if x]

                child_page_names.sort()

                for child_page_name in child_page_names:

                    child_pages.append(" * [[%s]]" % child_page_name)

                append(filename, child_page_section % "\n".join(child_pages))

            # Add comments to the content.

            if exists(comments) and title and not no_translate:

                append(filename, comment_section % title)

    finally:

        f.close()

    s = "".join(result)

    if output is None:

        write(manifest, s)

    else:

        append(output, s)

# Template for child page information.

child_page_section = """

----

%s

"""

# Template for comments.

comment_section = """

----

<<Include("^%s/")>>

"""

# Main program.

if __name__ == "__main__":

    try:

        filename = sys.argv[1]

        is_zipfile = splitext(filename)[-1] == extsep + "zip"

        space = sys.argv[2]

    except IndexError:

        print >>sys.stderr, "Please specify an XML file containing Wiki data and a workspace name."

        print >>sys.stderr, "For example: com_entities.xml COM"

        sys.exit(1)

    no_translate = "--no-translate" in sys.argv

    if exists(space):

        print >>sys.stderr, "Directory exists for space %s. Please choose another or remove its contents." % space

        sys.exit(1)

    package_zip = space + extsep + "zip"

    if exists(package_zip):

        print >>sys.stderr, "Page package exists. Please remove or rename it:", package_zip

        sys.exit(1)

    mkdir(space)

    mkdirs(join(space, "pages"))

    mkdirs(join(space, "versions"))

    p = xmlread.ConfigurableParser()

    handler = ConfluenceHandler(space, no_translate)

    # Register handlers in the parser for different elements.

    p["object"] = handler.handle_object

    p["property"] = handler.handle_property

    p["id"] = handler.handle_id

    p["collection"] = handler.handle_collection

    p["element"] = handler.handle_element

    # Open the XML dump.

    f = open(filename)

    if is_zipfile:

        zf = ZipFile(f)

        ff = StringIO(zf.read("entities.xml"))

    else:

        ff = f

    # Parse the data.

    try:

        p.parse(ff)

    finally:

        f.close()

    # Tidy up the import manifests, sorting each of them by revision and

    # finalising them.

    pages_dir = join(space, "pages")

    for pageid in listdir(pages_dir):

        sort_comments(pages_dir, pageid)

    output_manifest = join(space, "MOIN_PACKAGE")

    append(output_manifest, "MoinMoinPackage|1\n")

    for pageid in listdir(pages_dir):

        sort_manifest(pages_dir, pageid, output_manifest, no_translate)

    # Write the page package.

    page_package = ZipFile(package_zip, "w")

    try:

        # Include the page revisions.

        versions_dir = join(space, "versions")

        for versionid in listdir(versions_dir):

            page_package.write(join(versions_dir, versionid))

        # Include only the top-level manifest.

        page_package.write(output_manifest, "MOIN_PACKAGE")

    finally:

        page_package.close()

# vim: tabstop=4 expandtab shiftwidth=4