1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/moinformat/parsers/common.py Wed Dec 13 00:50:09 2017 +0100
1.3 @@ -0,0 +1,328 @@
1.4 +#!/usr/bin/env python
1.5 +
1.6 +"""
1.7 +Moin wiki parsing functionality.
1.8 +
1.9 +Copyright (C) 2017 Paul Boddie <paul@boddie.org.uk>
1.10 +
1.11 +This program is free software; you can redistribute it and/or modify it under
1.12 +the terms of the GNU General Public License as published by the Free Software
1.13 +Foundation; either version 3 of the License, or (at your option) any later
1.14 +version.
1.15 +
1.16 +This program is distributed in the hope that it will be useful, but WITHOUT
1.17 +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1.18 +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
1.19 +details.
1.20 +
1.21 +You should have received a copy of the GNU General Public License along with
1.22 +this program. If not, see <http://www.gnu.org/licenses/>.
1.23 +"""
1.24 +
1.25 +from moinformat.tree import Block, Region, Text
1.26 +import re
1.27 +
1.28 +# Pattern management.
1.29 +
1.30 +ws_excl_nl = r"[ \f\r\t\v]"
1.31 +
1.32 +def get_patterns(syntax):
1.33 +
1.34 + """
1.35 + Define patterns for the regular expressions in the 'syntax' mapping. In each
1.36 + pattern, replace \N with a pattern for matching whitespace excluding
1.37 + newlines.
1.38 + """
1.39 +
1.40 + patterns = {}
1.41 + for name, value in syntax.items():
1.42 + value = value.replace(r"\N", ws_excl_nl)
1.43 + patterns[name] = re.compile(value, re.UNICODE | re.MULTILINE)
1.44 + return patterns
1.45 +
1.46 +def get_subset(d, keys):
1.47 +
1.48 + "Return a subset of 'd' having the given 'keys'."
1.49 +
1.50 + subset = {}
1.51 + for key in keys:
1.52 + subset[key] = d[key]
1.53 + return subset
1.54 +
1.55 +
1.56 +
1.57 +# Tokenising functions.
1.58 +
1.59 +class TokenStream:
1.60 +
1.61 + "A stream of tokens taken from a string."
1.62 +
1.63 + def __init__(self, s, pos=0):
1.64 + self.s = s
1.65 + self.pos = pos
1.66 + self.match = None
1.67 + self.matching = None
1.68 +
1.69 + def rewind(self, length):
1.70 +
1.71 + "Rewind in the string by 'length'."
1.72 +
1.73 + self.pos -= min(length, self.pos)
1.74 +
1.75 + def read_until(self, patterns, remaining=True):
1.76 +
1.77 + """
1.78 + Find the first match for the given 'patterns'. Return the text preceding
1.79 + any match, the remaining text if no match was found, or None if no match
1.80 + was found and 'remaining' is given as a false value.
1.81 + """
1.82 +
1.83 + first = None
1.84 + self.matching = None
1.85 +
1.86 + # Find the first matching pattern.
1.87 +
1.88 + for pattern_name, pattern in patterns.items():
1.89 + match = pattern.search(self.s, self.pos)
1.90 + if match:
1.91 + start, end = match.span()
1.92 + if self.matching is None or start < first:
1.93 + first = start
1.94 + self.matching = pattern_name
1.95 + self.match = match
1.96 +
1.97 + if self.matching is None:
1.98 + if remaining:
1.99 + return self.s[self.pos:]
1.100 + else:
1.101 + return None
1.102 + else:
1.103 + return self.s[self.pos:first]
1.104 +
1.105 + def read_match(self, group=1):
1.106 +
1.107 + """
1.108 + Return the matched text, updating the position in the stream. If 'group'
1.109 + is specified, the indicated group in a match will be returned.
1.110 + Typically, group 1 should contain all pertinent data, but groups defined
1.111 + within group 1 can provide sections of the data.
1.112 + """
1.113 +
1.114 + if self.match:
1.115 + _start, self.pos = self.match.span()
1.116 + try:
1.117 + return self.match.group(group)
1.118 + except IndexError:
1.119 + return ""
1.120 + else:
1.121 + self.pos = len(self.s)
1.122 + return None
1.123 +
1.124 +
1.125 +
1.126 +# Utility functions.
1.127 +
1.128 +def new_block(region):
1.129 +
1.130 + "Start a new block in 'region'."
1.131 +
1.132 + region.add(Block([]))
1.133 +
1.134 +
1.135 +
1.136 +# Parser abstractions.
1.137 +
1.138 +class ParserBase:
1.139 +
1.140 + "Common parsing methods."
1.141 +
1.142 + region_pattern_names = None
1.143 +
1.144 + def __init__(self, formats=None):
1.145 +
1.146 + """
1.147 + Initialise the parser with any given 'formats' mapping from region type
1.148 + names to parser objects.
1.149 + """
1.150 +
1.151 + self.formats = formats
1.152 +
1.153 + def get_parser(self, format_type):
1.154 +
1.155 + """
1.156 + Return a parser for 'format_type' or None if no suitable parser is found.
1.157 + """
1.158 +
1.159 + if not self.formats:
1.160 + return None
1.161 +
1.162 + cls = self.formats.get(format_type)
1.163 + if cls:
1.164 + return cls(self.formats)
1.165 + else:
1.166 + return None
1.167 +
1.168 + def get_patterns(self, pattern_names):
1.169 +
1.170 + "Return a mapping of the given 'pattern_names' to patterns."
1.171 +
1.172 + return get_subset(self.patterns, pattern_names)
1.173 +
1.174 + def get_items(self, s, pos=0):
1.175 +
1.176 + "Return a sequence of token items for 's' and 'pos'."
1.177 +
1.178 + return TokenStream(s, pos)
1.179 +
1.180 + def set_region(self, items, region):
1.181 +
1.182 + "Set the 'items' used to populate the given 'region'."
1.183 +
1.184 + self.items = items
1.185 + self.region = region
1.186 +
1.187 + def read_until(self, pattern_names, remaining=True):
1.188 +
1.189 + """
1.190 + Read the next portion of input, matching using 'pattern_names'. Return
1.191 + the text preceding any match, the remaining text if no match was found,
1.192 + or None if no match was found and 'remaining' is given as a false value.
1.193 + """
1.194 +
1.195 + return self.items.read_until(self.get_patterns(pattern_names))
1.196 +
1.197 + def read_match(self, group=1):
1.198 +
1.199 + """
1.200 + Return the group of the matching pattern with the given 'group' number.
1.201 + """
1.202 +
1.203 + return self.items.read_match(group)
1.204 +
1.205 + def read_matching(self):
1.206 +
1.207 + "Return the name of the matching pattern."
1.208 +
1.209 + return self.items.matching
1.210 +
1.211 + # Parser methods invoked from other objects.
1.212 +
1.213 + def parse(self, s):
1.214 +
1.215 + """
1.216 + Parse page text 's'. Pages consist of regions delimited by markers.
1.217 + """
1.218 +
1.219 + self.items = self.get_items(s)
1.220 + self.region = self.parse_region()
1.221 + return self.region
1.222 +
1.223 + def parse_region_content(self, items, region):
1.224 +
1.225 + "Parse the data provided by 'items' to populate a 'region'."
1.226 +
1.227 + self.set_region(items, region)
1.228 +
1.229 + # Define a block to hold text and start parsing.
1.230 +
1.231 + new_block(region)
1.232 +
1.233 + if self.region_pattern_names:
1.234 + self.parse_region_details(region, self.region_pattern_names)
1.235 +
1.236 + # Top-level parser handler methods.
1.237 +
1.238 + def parse_region(self, level=0, indent=0):
1.239 +
1.240 + """
1.241 + Parse the data to populate a region with the given 'level' at the given
1.242 + 'indent'.
1.243 + """
1.244 +
1.245 + region = Region([], level, indent)
1.246 +
1.247 + # Parse section headers, then parse according to region type.
1.248 +
1.249 + self.parse_region_header(region)
1.250 + self.parse_region_type(region)
1.251 +
1.252 + return region
1.253 +
1.254 + def parse_region_type(self, region):
1.255 +
1.256 + """
1.257 + Use configured parsers to parse 'region' based on its type.
1.258 + """
1.259 +
1.260 + # Find an appropriate parser given the type.
1.261 +
1.262 + parser = self.get_parser(region.type)
1.263 +
1.264 + if parser:
1.265 + parser.parse_region_content(self.items, region)
1.266 +
1.267 + # Otherwise, treat the section as opaque.
1.268 +
1.269 + else:
1.270 + self.parse_region_opaque(region)
1.271 +
1.272 + def parse_region_header(self, region):
1.273 +
1.274 + """
1.275 + Parse the region header, setting it on the 'region' object.
1.276 + """
1.277 +
1.278 + if self.read_until(["header"], False) == "": # None means no header
1.279 + region.type = self.read_match()
1.280 +
1.281 + def parse_region_opaque(self, region):
1.282 +
1.283 + "Parse the data to populate an opaque 'region'."
1.284 +
1.285 + region.transparent = False
1.286 + self.parse_region_details(region, ["regionend"])
1.287 +
1.288 + # Parsing utilities.
1.289 +
1.290 + def parse_region_details(self, region, pattern_names):
1.291 +
1.292 + "Search 'region' using the 'pattern_names'."
1.293 +
1.294 + try:
1.295 + while True:
1.296 +
1.297 + # Obtain text before any marker or the end of the input.
1.298 +
1.299 + preceding = self.read_until(pattern_names)
1.300 + if preceding:
1.301 + region.append_inline(Text(preceding))
1.302 +
1.303 + # End of input.
1.304 +
1.305 + if not self.read_matching():
1.306 + break
1.307 +
1.308 + # Obtain any feature.
1.309 +
1.310 + feature = self.read_match()
1.311 + handler = self.handlers.get(self.read_matching())
1.312 +
1.313 + # Handle each feature or add text to the region.
1.314 +
1.315 + if handler:
1.316 + handler(self, region)
1.317 + else:
1.318 + region.append_inline(Text(feature))
1.319 +
1.320 + except StopIteration:
1.321 + pass
1.322 +
1.323 + region.normalise()
1.324 +
1.325 + def end_region(self, region):
1.326 +
1.327 + "End the parsing of 'region', breaking out of the parsing loop."
1.328 +
1.329 + raise StopIteration
1.330 +
1.331 +# vim: tabstop=4 expandtab shiftwidth=4