#!/usr/bin/env python  """ Confluence Wiki syntax parsing.  Copyright (C) 2012, 2013 Paul Boddie <paul@boddie.org.uk>  This software is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.  This software is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.  You should have received a copy of the GNU General Public License along with this library; see the file LICENCE.txt If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA  --------  The basic procedure is as follows:   1. Wiki pages are first split up into regions.  2. Then, within these regions, the text is split into blocks.     1. First, lists are identified.     2. Additionally, other block-like elements are identified.  3. Each block is then split into regions. """  from common import * import re import sys import codecs import operator  # Section extraction.  sections_regexp_str = r"(?<!{){(?P<type>[^-_*+{}\n:]+)(?P<options>:[^}\n]+)?}" \                       r"|" \                       r"^(?P<rowstart>[|]{1,2})" \                       r"|" \                       r"(?P<rowend>[|]{1,2}(\n|$))" \                       r"|" \                       r"^(?P<listitem>\s*[*#-]+\s+.*?([^|](\n|$)|(?=[|](\n|$))))"  sections_regexp = re.compile(sections_regexp_str, re.MULTILINE)  def get_regions(s):      """     Return a list of regions from 's'. Each region is specified using a tuple of     the form (type, text).     """      last = 0     regions = [""]     depth = 0     had_row = False     had_item = False      for match in sections_regexp.finditer(s):         start, end = match.span()         is_start = match.group("options") or match.group("rowstart")         is_section = is_section_marker(match.group("type"))         is_row = match.group("rowstart") or match.group("rowend")         is_item = match.group("listitem")          # The start of a region is either indicated by a marker with options or         # by a marker where no region is currently active.          if is_start or not depth:              # Where no region is active, add the text since the last match as a             # "null" region.              if not depth:                 regions[-1] += s[last:start]                  # A new region is maintained as a string.                  if is_section:                     regions.append(s[start:end])                  # A new row may either continue a table region or start a new                 # table region.                  elif is_row:                     if had_row and last == start:                         regions[-2] += regions[-1] + s[start:end]                         regions.pop()                     else:                         regions.append(s[start:end])                  # A list item may either continue a list region or start a new                 # list region.                  elif is_item:                      # If continuing a list, merge the list regions and start a                     # new potentally separate region.                      if had_item and last == start:                         regions[-2] += regions[-1] + s[start:end]                         regions[-1] = ""                      # If not continuing a list, make a region for a new list and                     # start a new potentally separate region.                      else:                         regions.append(s[start:end])                         regions.append("")                  # Certain markers may be standalone macros.                  else:                     regions[-1] += s[start:end]              # Where a region is active, add the text since the last match as             # well as the text in this match to the region.              else:                 regions[-1] += s[last:end]              if is_section or is_row:                 depth += 1          # The end of a region is indicated by a marker with no options or the         # end of a row.          else:             # Where no region is active, the text since the last match plus the             # marker are added to the current "null" region.              if not depth:                  # Add to the string portion of the "null" region.                  regions[-1] += s[last:end]              # Where a region is active, the end marker and preceding text is             # either incorporated into the current region if more than one             # region is active, or the preceding text is incorporated into the             # current region and the details of the region are then obtained.              else:                 if depth > 1 or (not is_section and not is_row):                     regions[-1] += s[last:end]                  # Terminate the active region, interpreting its contents.                  else:                     regions[-1] += s[last:end]                     regions.append("")                  if is_section or is_row:                     depth -= 1          had_row = is_row         had_item = is_item         last = end      # Where a region is still active, terminate it.      regions[-1] += s[last:]      return [get_section_details(s) for s in regions if s]  def is_section_marker(sectiontype):     return sectiontypes.has_key(sectiontype) or sectiontype == "color"  # Section inspection.  section_regexp_str = r"{(?P<sectiontype>[^\n:]*?)(?::(?P<options>.*?))?}(?P<section>.*){(?P=sectiontype)}" section_regexp = re.compile(section_regexp_str, re.DOTALL | re.MULTILINE)  def get_section_details(s):      "Return the details of a section 's' in the form (type, text)."      match = section_regexp.match(s)     if match:         return (match.group("sectiontype"), match.group("options")), match.group("section")     else:         return None, s  # Heading, table and list extraction.  list_regexp_str = r"^\s*(?P<listtype>[*#-])[*#-]*\s+.*(\n\s*(?P=listtype).*?)*(?:\n|$)" table_regexp_str = r"^((?P<celltype>[|]{1,2})((.|\n(?!\n))+?(?P=celltype))+(\n|$))+" blocktext_regexp_str = r"^(?P<type>h\d|bq)\.\s+(?P<text>.*)$"  blockelement_regexp = re.compile(     "(" + list_regexp_str + ")"     "|"     "(" + table_regexp_str + ")"     "|"     "(" + blocktext_regexp_str + ")",     re.MULTILINE     )  def get_block_elements(s):      """     Extract headings, tables and lists from the given string 's'.     """      last = 0     blocks = []     for match in blockelement_regexp.finditer(s):         start, end = match.span()         matchtype = match.group("listtype") and "list" or match.group("celltype") and "table" or match.group("type")         blocks.append((None, s[last:start]))         blocks.append((matchtype, match.group("text") or s[start:end]))         last = end     blocks.append((None, s[last:]))     return blocks  # Block extraction.  block_regexp_str = r"^(?:\s*\n)+" block_regexp = re.compile(block_regexp_str, re.MULTILINE)  def get_basic_blocks(s):      """     Return blocks from the given string 's' by splitting the text on blank lines     and eliminating those lines.     """      return [b for b in block_regexp.split(s) if b.strip()]  # Block inspection.  def get_blocks(s):      """     Return blocks from the given string 's', inspecting the basic blocks and     generating additional block-level text where appropriate.     """      blocks = []      for blocktype, blocktext in get_block_elements(s):          # Collect heading, list and table blocks.          if blocktype is not None:             blocks.append((blocktype, blocktext))          # Attempt to find new subblocks in other regions.          else:             for block in get_basic_blocks(blocktext):                 blocks.append((None, block))      return blocks  # List item inspection.  listitem_regexp_str = r"^(?P<marker> *[-*#]+)\s+(?P<text>.*)$" listitem_regexp = re.compile(listitem_regexp_str, re.MULTILINE)  def get_list_items(text):      "Return a list of (marker, text) tuples for the given list 'text'."      items = []      for match in listitem_regexp.finditer(text):         items.append((match.group("marker"), match.group("text")))      return items  # Content inspection.  monospace_regexp_str = r"{{(?P<monotext>.*?)}}" link_regexp_str      = r"(?<!\\)[[](?P<linktext>.*?)]" image_regexp_str     = r"!(?P<imagetext>\w.*?)!" macro_regexp_str     = r"{(?P<macro>.*?)(?::(?P<options>.*?))?}"  # Word-dependent patterns. # Here, the unbracketed markers must test for the absence of surrounding word # characters.  italic_regexp_str    = r"(?:(?<!\w)_|\{_\})(?P<italictext>.*?)(?:_(?!\w)|\{_\})" bold_regexp_str      = r"(?:(?<!\w)\*|\{\*\})(?P<boldtext>.*?)(?:\*(?!\w)|\{\*\})" del_regexp_str       = r"(?:(?<!\w)-|\{-\})(?P<deltext>.*?)(?:-(?!\w)|\{-\})" underline_regexp_str = r"(?:(?<!\w)\+|\{\+\})(?P<underlinetext>.*?)(?:\+(?!\w)|\{\+\})" sub_regexp_str       = r"(?:(?<!\w)~|\{~\})(?P<subtext>.*?)(?:~(?!\w)|\{~\})"  content_regexp_str = (     "(" + monospace_regexp_str + ")"     "|"     "(" + link_regexp_str + ")"     "|"     "(" + image_regexp_str + ")"     "|"     "(" + macro_regexp_str + ")"     "|"     "(" + italic_regexp_str + ")"     "|"     "(" + bold_regexp_str + ")"     "|"     "(" + del_regexp_str + ")"     "|"     "(" + underline_regexp_str + ")"     "|"     "(" + sub_regexp_str + ")"     )  # Table row inspection.  cellsep_regexp_str = r"(?P<celltype>[|]{1,2})"  table_content_regexp_str = (     content_regexp_str +     "|"     "(" + cellsep_regexp_str + ")"     )  content_regexp = re.compile(content_regexp_str) table_content_regexp = re.compile(table_content_regexp_str)  def get_table_rows(text):      "Return a list of (cellsep, columns) tuples for the given table 'text'."      rows = []      for row in text.split("|\n"):         if not row:             break          row += "|"         cellsep = None         columns = [""]         last = 0         for match in table_content_regexp.finditer(row):             start, end = match.span()             columns[-1] += row[last:start]              if match.group("celltype"):                 if cellsep is None:                     cellsep = match.group("celltype")                 columns.append("")             else:                 columns[-1] += match.group()              last = end          columns[-1] += row[last:]          if cellsep:             rows.append((cellsep, columns[1:-1]))      return rows  # Notation conversion.  notation_mapping = [     (r"\!", "!"),     (r"\-", "-"),     (r"\\""\n", "<<BR>>"),     (r"\\ ", "<<BR>>"),     (r"\~", "~"),     (r"\[", "<<Verbatim([)>>"),     (r"\]", "<<Verbatim(])>>"),     (r"\*", "*"),     ]  preformatted_notation_mapping = [     (r"\!", "!"),     (r"\-", "-"),     (r"\\""\n", "\n"),     (r"\\ ", "\n"),     (r"\~", "~"),     ]  # Translation helpers.  markers = {     "*" : "*",     "#" : "1.",     "-" : "*",     }  cellseps = {     "|" : "\n|| ",     "||" : "\n|| ",     }  cellextra = {     "|" : "",     "||" : "'''",     }  sectiontypes = {     "code"      : "",     "excerpt"   : "#!wiki",     "noformat"  : "",     "quote"     : "",     "info"      : "#!wiki important",     "note"      : "#!wiki caution",     "tip"       : "#!wiki tip",     "warning"   : "#!wiki warning",     }  preformatted_sectiontypes = (None, "noformat")  macroargs = {     "color"     : "col",     }  macrotypes = {     "anchor"    : "<<Anchor(%(args)s)>>",     "color"     : "<<Color2(%(content)s, %(args)s)>>",     "toc"       : "<<TableOfContents>>",     }  class ConfluenceParser:      "A parser for Confluence markup."      def __init__(self):         self.max_level = self.level = 0         self.in_heading = False         self.held_anchors = []         self.macro = None         self.sections = []      def translate_marker(self, marker):          "Translate the given 'marker' to a suitable Moin representation."          return " " * len(marker) + markers[marker[-1]]      def translate_cellsep(self, cellsep):          "Translate the given 'cellsep' to a suitable Moin representation."          return cellseps[cellsep]      def translate_cell(self, cellsep, text):          "Using 'cellsep', translate the cell 'text'."          return cellextra[cellsep] + self.parse_text(text).strip() + cellextra[cellsep]      def translate_content_match(self, match):          "Translate the content described by the given 'match', returning a string."          if match.group("monotext"):             self.enter_section(); self.leave_section()             return "{{{%s}}}" % match.group("monotext")          elif match.group("linktext"):             parts = match.group("linktext").split("|")              # NOTE: Proper detection of external links required.              if len(parts) == 1:                 label, target, title = None, parts[0], None             elif len(parts) == 2:                 (label, target), title = parts, None             else:                 label, target, title = parts              target = target.strip()              # Look for namespace links and rewrite them.              if target.find(":") != -1:                 prefix = ""                 space, rest = target.split(":", 1)                 if space not in URL_SCHEMES:                     rest = get_page_title(rest)                     target = "%s/%s" % (space, rest)              # Detect anchors.              elif target.startswith("#"):                 prefix = ""              # Detect attachments.              elif target.startswith("^"):                 prefix = "attachment:"              # Link to other pages within a space.              else:                 prefix = "../"                  # Make the link tidier by making a target if none was given.                  if not label:                     label = target                  target = get_page_title(target)              if not label and not title:                 return "[[%s%s]]" % (prefix, target)             elif not title:                 return "[[%s%s|%s]]" % (prefix, target, label)             else:                 return "[[%s%s|%s|title=%s]]" % (prefix, target, label, title)          elif match.group("imagetext"):             parts = match.group("imagetext").split("|")              # NOTE: Proper detection of external links required.              if parts[0].startswith("http"):                 prefix = ""             else:                 prefix = "attachment:"              # NOTE: Proper options conversion required.              if len(parts) == 1:                 return "{{%s%s}}" % (prefix, parts[0])             else:                 return "{{%s%s|%s}}" % (prefix, parts[0], parts[1])          elif match.group("macro"):             macro_name = match.group("macro")             if macrotypes.has_key(macro_name):                 argname = macroargs.get(macro_name)                 result = macrotypes[macro_name] % {                     "args" : quote_macro_argument((argname and ("%s=" % argname) or "") + (match.group("options") or ""))                     }                 if not self.forbids_macros():                     return result                 if macro_name == "anchor":                     self.held_anchors.append(result)             return ""          elif match.group("italictext"):             return "''%s''" % self.translate_content(match.group("italictext"))          elif match.group("boldtext"):             return "'''%s'''" % self.translate_content(match.group("boldtext"))          elif match.group("deltext"):             return "--(%s)--" % self.translate_content(match.group("deltext"))          elif match.group("underlinetext"):             return "__%s__" % self.translate_content(match.group("underlinetext"))          elif match.group("subtext"):             return ",,%s,," % self.translate_content(match.group("subtext"))          else:             return self.translate_text(match.group())      def translate_text(self, s, preformatted=False):          "Translate the plain text string 's', converting notation."          for before, after in preformatted and preformatted_notation_mapping or notation_mapping:             s = s.replace(before, after)         return s      def translate_content(self, text):          """         Return a translation of the given 'text'. If the optional 'sectiontype' is         specified, the translation may be modified to a form appropriate to the         section being translated.         """          parts = []         preformatted = self.is_preformatted()          last = 0         for match in content_regexp.finditer(text):             start, end = match.span()             parts.append(self.translate_text(text[last:start], preformatted))              # Handle unformatted sections.              if self.sections and self.sections[-1] in ("code", "noformat"):                 parts.append(match.group())             else:                 parts.append(self.translate_content_match(match))              last = end          parts.append(self.translate_text(text[last:], preformatted))         return "".join(parts)      def is_preformatted(self):         return reduce(operator.or_, [x in preformatted_sectiontypes for x in self.sections], False)      def translate_block(self, blocktype, blocktext):          "Translate the block with the given 'blocktype' and 'blocktext'."          if blocktype in headings:             self.in_heading = True             self.held_anchors = []          parts = []          # Translate headings and blockquotes.          if blocktypes.has_key(blocktype):             text = self.parse_text(blocktext)             for anchor in self.held_anchors:                 parts.append(anchor)             parts.append(blocktypes[blocktype] % text)          # Translate list items.          elif blocktype == "list":             for listmarker, listitem in get_list_items(blocktext):                 parts.append("%s %s" % (self.translate_marker(listmarker), self.parse_text(listitem)))          # Translate table items.          elif blocktype == "table":              # Enter the table.              self.enter_section("table")              table_parts = []             first = True              for cellsep, columns in get_table_rows(blocktext):                 if not first:                     table_parts.append("==")                 else:                     first = False                 moinsep = self.translate_cellsep(cellsep)                 table_parts.append(moinsep.join([self.translate_cell(cellsep, column) for column in columns]))              # Nest the section appropriately.              opening, closing = self.nest_section()              parts.append("%s#!table" % opening)             parts += table_parts             parts.append(closing)              # Leave the table.              self.leave_section()          # Handle anonymous blocks.          else:             parts.append(self.parse_text(blocktext))          if blocktype in headings:             self.in_heading = False          return "\n".join(parts)      def translate_section(self, sectiontype, options, text):          """         Translate the section with the given 'sectiontype', 'options' and         'text'.         """          parts = []          # Enter the section.          self.enter_section(sectiontype)          # Sections can contain other sections.          if sectiontype == "noformat":             section_content = self.translate_content(text.strip("\n"))         else:             section_content = self.parse_text(text.strip())          # Nest the section appropriately.          opening, closing = self.nest_section()         mointype = sectiontypes.get(sectiontype)          parts.append("%s%s\n" % (opening, mointype or ""))         parts.append(section_content)         parts.append("\n%s\n" % closing)          # Leave the section.          self.leave_section()          return parts      def enter_section(self, sectiontype=None):         self.level += 1         self.max_level = max(self.level, self.max_level)         self.sections.append(sectiontype)      def leave_section(self):         self.level -= 1         if not self.level:             self.max_level = 0         self.sections.pop()      def nest_section(self):         level = 3 + self.max_level - self.level         opening = "{" * level         closing = "}" * level         return opening, closing      # General parsing.      def parse_text(self, s, top=False):          "Parse the content in the string 's', returning the translation."          parts = []          # Control spacing between blocks and other blocks or sections.          preceded_by_block = False          for type, text in get_regions(s):              # Handle list, heading, blockquote or anonymous blocks.              if type is None:                  # Where the region is the same as the provided text, return                 # immediately. This is the base case of the recursive parsing                 # process.                  if text == s and not top:                     return self.translate_content(text)                  # Otherwise, obtain and translate the blocks.                  if preceded_by_block:                     parts.append("\n")                  first = True                 for blocktype, blocktext in get_blocks(text):                     if not first:                         parts.append("\n")                     else:                         first = False                     parts.append("%s" % self.translate_block(blocktype, blocktext))                  if not first:                     preceded_by_block = True              # Handle sections.              else:                 sectiontype, options = type                  # Direct translations of sections.                  if sectiontypes.has_key(sectiontype):                     if preceded_by_block:                         parts.append("\n")                      parts += self.translate_section(sectiontype, options, text)                     preceded_by_block = True                  # Translations of macros acting as sections.                  elif macrotypes.has_key(sectiontype):                      # Prevent the production of macros in places they would                     # produce illegal Moin syntax.                      if not self.forbids_macros():                         self.macro = sectiontype                         argname = macroargs.get(sectiontype)                         parts.append(macrotypes[sectiontype] % {                             "content"   : quote_macro_argument(self.parse_text(text)),                             "args"      : quote_macro_argument((argname and ("%s=" % argname) or "") + options)                             })                         self.macro = None                      # Include the contents of section-based macros where the                     # macros themselves are not allowed.                      else:                         parts.append(self.translate_content(text))                      preceded_by_block = False                  # Unrecognised sections.                  else:                     parts += self.translate_section(sectiontype, None, text)                     preceded_by_block = False          return "".join(parts)      def forbids_macros(self):         return self.in_heading or self.macro  def parse(s, out):      "Parse the content in the string 's', writing a translation to 'out'."      parser = ConfluenceParser()     out.write(parser.parse_text(s, top=True))  if __name__ == "__main__":     s = codecs.getreader("utf-8")(sys.stdin).read()     out = codecs.getwriter("utf-8")(sys.stdout)     parse(s, out)  # vim: tabstop=4 expandtab shiftwidth=4