1.1 --- a/README.txt Mon Nov 11 13:42:55 2013 +0100
1.2 +++ b/README.txt Fri Dec 20 23:25:44 2013 +0100
1.3 @@ -69,6 +69,8 @@
1.4 * Moved ItemStore and related functionality into ItemSupport.
1.5 * Added support for subpage-based item stores.
1.6 * Added groupHasMember from ApproveChanges.
1.7 + * Added the TokenSupport module to try and have a reliable shell-like
1.8 + tokeniser.
1.9
1.10 New in MoinSupport 0.4.1 (Changes since MoinSupport 0.4)
1.11 --------------------------------------------------------
2.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
2.2 +++ b/TokenSupport.py Fri Dec 20 23:25:44 2013 +0100
2.3 @@ -0,0 +1,109 @@
2.4 +# -*- coding: iso-8859-1 -*-
2.5 +"""
2.6 + MoinMoin - TokenSupport library
2.7 +
2.8 + @copyright: 2013 by Paul Boddie <paul@boddie.org.uk>
2.9 + @license: GNU GPL (v2 or later), see COPYING.txt for details.
2.10 +"""
2.11 +
2.12 +import re
2.13 +
2.14 +identifier_expr = re.compile(
2.15 + """(?P<non_literal>[^'" ]+)"""
2.16 + "|"
2.17 + "(?P<spaces> +)"
2.18 + "|"
2.19 + "(?P<literal1>'[^']*')"
2.20 + "|"
2.21 + '(?P<literal2>"[^"]*")'
2.22 + )
2.23 +
2.24 +def getIdentifiers(s, doubling=False):
2.25 +
2.26 + """
2.27 + Return 's' containing space-separated quoted identifiers, parsed into
2.28 + regions that hold the individual identifiers. The optional 'doubling'
2.29 + argument can be used to support convenient quote doubling to reproduce
2.30 + single quote characters.
2.31 +
2.32 + Quoting of identifiers can be done using the single-quote and double-quote
2.33 + characters in order to include spaces within identifiers. For example:
2.34 +
2.35 + 'contains space'
2.36 + -> contains space (a single identifier)
2.37 +
2.38 + Where one kind of quote (or apostrophe) is to be included in an identifier,
2.39 + the other quoting character can be used to delimit the identifier. For
2.40 + example:
2.41 +
2.42 + "Python's syntax"
2.43 + -> Python's syntax (a single identifier)
2.44 +
2.45 + Where the 'doubling' argument is set to a true value, a quote character can
2.46 + be doubled to include it in an identifier. For example:
2.47 +
2.48 + Python''s syntax
2.49 + -> Python's syntax (a single identifier)
2.50 +
2.51 + Where a mixture of quotes is required in a single identifier, adjacent
2.52 + quoted regions can be used. For example:
2.53 +
2.54 + "Python's "'"intuitive" syntax'
2.55 + -> "Python's " (region #1)
2.56 + + '"intuitive" syntax' (region #2)
2.57 + -> Python's "intuitive" syntax (a single identifier)
2.58 +
2.59 + Where unquoted regions are adjacent to quoted regions, the regions are
2.60 + combined. For example:
2.61 +
2.62 + "Python's "intuitive" syntax"
2.63 + -> "Python's " (region #1)
2.64 + + intuitive (region #2)
2.65 + + " syntax" (region #3)
2.66 + -> Python's intuitive syntax (a single identifier)
2.67 + """
2.68 +
2.69 + regions = []
2.70 + in_literal = False
2.71 +
2.72 + for match in identifier_expr.finditer(s):
2.73 + non_literal, spaces, literal1, literal2 = match.groups()
2.74 +
2.75 + identifier = None
2.76 +
2.77 + # Spaces prevent continuation of identifier regions.
2.78 +
2.79 + if spaces:
2.80 + in_literal = False
2.81 +
2.82 + # Unquoted regions contribute to the current identifier.
2.83 +
2.84 + if non_literal and non_literal.strip():
2.85 + identifier = non_literal.strip()
2.86 +
2.87 + # Quoted regions also contribute to the current identifier.
2.88 +
2.89 + for s in (literal1, literal2):
2.90 + if s is not None:
2.91 +
2.92 + # Either strip the quoting or for empty regions, adopt the
2.93 + # quote character.
2.94 +
2.95 + if not doubling or len(s) > 2:
2.96 + identifier = s[1:-1]
2.97 + elif doubling:
2.98 + identifier = s[0]
2.99 +
2.100 + # Either continue or add an identifier, and indicate possible
2.101 + # continuation.
2.102 +
2.103 + if identifier:
2.104 + if in_literal:
2.105 + regions[-1] += identifier
2.106 + else:
2.107 + regions.append(identifier)
2.108 + in_literal = True
2.109 +
2.110 + return regions
2.111 +
2.112 +# vim: tabstop=4 expandtab shiftwidth=4
3.1 --- a/setup.py Mon Nov 11 13:42:55 2013 +0100
3.2 +++ b/setup.py Fri Dec 20 23:25:44 2013 +0100
3.3 @@ -11,5 +11,6 @@
3.4 version = "0.5",
3.5 py_modules = ["ContentTypeSupport", "DateSupport", "GeneralSupport",
3.6 "ItemSupport", "LocationSupport", "MoinDateSupport",
3.7 - "MoinRemoteSupport", "MoinSupport", "ViewSupport"]
3.8 + "MoinRemoteSupport", "MoinSupport", "TokenSupport",
3.9 + "ViewSupport"]
3.10 )
4.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
4.2 +++ b/tests/test_tokens.py Fri Dec 20 23:25:44 2013 +0100
4.3 @@ -0,0 +1,19 @@
4.4 +#!/usr/bin/env python
4.5 +
4.6 +from TokenSupport import getIdentifiers
4.7 +
4.8 +tests = [
4.9 + (1, False, """'contains space'""", ["contains space"]),
4.10 + (2, False, """contains space""", ["contains", "space"]),
4.11 + (1, False, '''"Python's syntax"''', ["Python's syntax"]),
4.12 + (2, False, """Python''s syntax""", ["Pythons", "syntax"]),
4.13 + (2, True, """Python''s syntax""", ["Python's", "syntax"]),
4.14 + (1, False, '''"Python's "'"intuitive" syntax' ''', ['''Python's "intuitive" syntax''']),
4.15 + (1, False, '''"Python's "intuitive" syntax" ''', ['''Python's intuitive syntax''']),
4.16 + ]
4.17 +
4.18 +for n, doubling, s, e in tests:
4.19 + l = getIdentifiers(s, doubling)
4.20 + print l == e, l, "==", e, len(l) == n, len(l), "==", n, "<-", doubling, s
4.21 +
4.22 +# vim: tabstop=4 expandtab shiftwidth=4