diff options
Diffstat (limited to 'source-builder/sb/markdown/inlinepatterns.py')
| -rw-r--r-- | source-builder/sb/markdown/inlinepatterns.py | 534 |
1 files changed, 534 insertions, 0 deletions
diff --git a/source-builder/sb/markdown/inlinepatterns.py b/source-builder/sb/markdown/inlinepatterns.py new file mode 100644 index 0000000..37c9afa --- /dev/null +++ b/source-builder/sb/markdown/inlinepatterns.py @@ -0,0 +1,534 @@ +""" +INLINE PATTERNS +============================================================================= + +Inline patterns such as *emphasis* are handled by means of auxiliary +objects, one per pattern. Pattern objects must be instances of classes +that extend markdown.Pattern. Each pattern object uses a single regular +expression and needs support the following methods: + + pattern.getCompiledRegExp() # returns a regular expression + + pattern.handleMatch(m) # takes a match object and returns + # an ElementTree element or just plain text + +All of python markdown's built-in patterns subclass from Pattern, +but you can add additional patterns that don't. + +Also note that all the regular expressions used by inline must +capture the whole block. For this reason, they all start with +'^(.*)' and end with '(.*)!'. In case with built-in expression +Pattern takes care of adding the "^(.*)" and "(.*)!". + +Finally, the order in which regular expressions are applied is very +important - e.g. if we first replace http://.../ links with <a> tags +and _then_ try to replace inline html, we would end up with a mess. +So, we apply the expressions in the following order: + +* escape and backticks have to go before everything else, so + that we can preempt any markdown patterns by escaping them. + +* then we handle auto-links (must be done before inline html) + +* then we handle inline HTML. At this point we will simply + replace all inline HTML strings with a placeholder and add + the actual HTML to a hash. + +* then inline images (must be done before links) + +* then bracketed links, first regular then reference-style + +* finally we apply strong and emphasis +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from . import util +from . import odict +import re +try: # pragma: no cover + from urllib.parse import urlparse, urlunparse +except ImportError: # pragma: no cover + from urlparse import urlparse, urlunparse +try: # pragma: no cover + from html import entities +except ImportError: # pragma: no cover + import htmlentitydefs as entities + + +def build_inlinepatterns(md_instance, **kwargs): + """ Build the default set of inline patterns for Markdown. """ + inlinePatterns = odict.OrderedDict() + inlinePatterns["backtick"] = BacktickPattern(BACKTICK_RE) + inlinePatterns["escape"] = EscapePattern(ESCAPE_RE, md_instance) + inlinePatterns["reference"] = ReferencePattern(REFERENCE_RE, md_instance) + inlinePatterns["link"] = LinkPattern(LINK_RE, md_instance) + inlinePatterns["image_link"] = ImagePattern(IMAGE_LINK_RE, md_instance) + inlinePatterns["image_reference"] = ImageReferencePattern( + IMAGE_REFERENCE_RE, md_instance + ) + inlinePatterns["short_reference"] = ReferencePattern( + SHORT_REF_RE, md_instance + ) + inlinePatterns["autolink"] = AutolinkPattern(AUTOLINK_RE, md_instance) + inlinePatterns["automail"] = AutomailPattern(AUTOMAIL_RE, md_instance) + inlinePatterns["linebreak"] = SubstituteTagPattern(LINE_BREAK_RE, 'br') + if md_instance.safeMode != 'escape': + inlinePatterns["html"] = HtmlPattern(HTML_RE, md_instance) + inlinePatterns["entity"] = HtmlPattern(ENTITY_RE, md_instance) + inlinePatterns["not_strong"] = SimpleTextPattern(NOT_STRONG_RE) + inlinePatterns["em_strong"] = DoubleTagPattern(EM_STRONG_RE, 'strong,em') + inlinePatterns["strong_em"] = DoubleTagPattern(STRONG_EM_RE, 'em,strong') + inlinePatterns["strong"] = SimpleTagPattern(STRONG_RE, 'strong') + inlinePatterns["emphasis"] = SimpleTagPattern(EMPHASIS_RE, 'em') + if md_instance.smart_emphasis: + inlinePatterns["emphasis2"] = SimpleTagPattern(SMART_EMPHASIS_RE, 'em') + else: + inlinePatterns["emphasis2"] = SimpleTagPattern(EMPHASIS_2_RE, 'em') + return inlinePatterns + + +""" +The actual regular expressions for patterns +----------------------------------------------------------------------------- +""" + +NOBRACKET = r'[^\]\[]*' +BRK = ( + r'\[(' + + (NOBRACKET + r'(\[')*6 + + (NOBRACKET + r'\])*')*6 + + NOBRACKET + r')\]' +) +NOIMG = r'(?<!\!)' + +# `e=f()` or ``e=f("`")`` +BACKTICK_RE = r'(?:(?<!\\)((?:\\{2})+)(?=`+)|(?<!\\)(`+)(.+?)(?<!`)\3(?!`))' + +# \< +ESCAPE_RE = r'\\(.)' + +# *emphasis* +EMPHASIS_RE = r'(\*)([^\*]+)\2' + +# **strong** +STRONG_RE = r'(\*{2}|_{2})(.+?)\2' + +# ***strongem*** or ***em*strong** +EM_STRONG_RE = r'(\*|_)\2{2}(.+?)\2(.*?)\2{2}' + +# ***strong**em* +STRONG_EM_RE = r'(\*|_)\2{2}(.+?)\2{2}(.*?)\2' + +# _smart_emphasis_ +SMART_EMPHASIS_RE = r'(?<!\w)(_)(?!_)(.+?)(?<!_)\2(?!\w)' + +# _emphasis_ +EMPHASIS_2_RE = r'(_)(.+?)\2' + +# [text](url) or [text](<url>) or [text](url "title") +LINK_RE = NOIMG + BRK + \ + r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*?)\12\s*)?\)''' + +#  or  +IMAGE_LINK_RE = r'\!' + BRK + r'\s*\(\s*(<.*?>|([^"\)\s]+\s*"[^"]*"|[^\)\s]*))\s*\)' + +# [Google][3] +REFERENCE_RE = NOIMG + BRK + r'\s?\[([^\]]*)\]' + +# [Google] +SHORT_REF_RE = NOIMG + r'\[([^\]]+)\]' + +# ![alt text][2] +IMAGE_REFERENCE_RE = r'\!' + BRK + '\s?\[([^\]]*)\]' + +# stand-alone * or _ +NOT_STRONG_RE = r'((^| )(\*|_)( |$))' + +# <http://www.123.com> +AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^>]*)>' + +# <me@example.com> +AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>' + +# <...> +HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)' + +# & +ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)' + +# two spaces at end of line +LINE_BREAK_RE = r' \n' + + +def dequote(string): + """Remove quotes from around a string.""" + if ((string.startswith('"') and string.endswith('"')) or + (string.startswith("'") and string.endswith("'"))): + return string[1:-1] + else: + return string + + +ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123} + + +def handleAttributes(text, parent): + """Set values of an element based on attribute definitions ({@id=123}).""" + def attributeCallback(match): + parent.set(match.group(1), match.group(2).replace('\n', ' ')) + return ATTR_RE.sub(attributeCallback, text) + + +""" +The pattern classes +----------------------------------------------------------------------------- +""" + + +class Pattern(object): + """Base class that inline patterns subclass. """ + + def __init__(self, pattern, markdown_instance=None): + """ + Create an instant of an inline pattern. + + Keyword arguments: + + * pattern: A regular expression that matches a pattern + + """ + self.pattern = pattern + self.compiled_re = re.compile("^(.*?)%s(.*)$" % pattern, + re.DOTALL | re.UNICODE) + + # Api for Markdown to pass safe_mode into instance + self.safe_mode = False + if markdown_instance: + self.markdown = markdown_instance + + def getCompiledRegExp(self): + """ Return a compiled regular expression. """ + return self.compiled_re + + def handleMatch(self, m): + """Return a ElementTree element from the given match. + + Subclasses should override this method. + + Keyword arguments: + + * m: A re match object containing a match of the pattern. + + """ + pass # pragma: no cover + + def type(self): + """ Return class name, to define pattern type """ + return self.__class__.__name__ + + def unescape(self, text): + """ Return unescaped text given text with an inline placeholder. """ + try: + stash = self.markdown.treeprocessors['inline'].stashed_nodes + except KeyError: # pragma: no cover + return text + + def itertext(el): # pragma: no cover + ' Reimplement Element.itertext for older python versions ' + tag = el.tag + if not isinstance(tag, util.string_type) and tag is not None: + return + if el.text: + yield el.text + for e in el: + for s in itertext(e): + yield s + if e.tail: + yield e.tail + + def get_stash(m): + id = m.group(1) + if id in stash: + value = stash.get(id) + if isinstance(value, util.string_type): + return value + else: + # An etree Element - return text content only + return ''.join(itertext(value)) + return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text) + + +class SimpleTextPattern(Pattern): + """ Return a simple text of group(2) of a Pattern. """ + def handleMatch(self, m): + return m.group(2) + + +class EscapePattern(Pattern): + """ Return an escaped character. """ + + def handleMatch(self, m): + char = m.group(2) + if char in self.markdown.ESCAPED_CHARS: + return '%s%s%s' % (util.STX, ord(char), util.ETX) + else: + return None + + +class SimpleTagPattern(Pattern): + """ + Return element of type `tag` with a text attribute of group(3) + of a Pattern. + + """ + def __init__(self, pattern, tag): + Pattern.__init__(self, pattern) + self.tag = tag + + def handleMatch(self, m): + el = util.etree.Element(self.tag) + el.text = m.group(3) + return el + + +class SubstituteTagPattern(SimpleTagPattern): + """ Return an element of type `tag` with no children. """ + def handleMatch(self, m): + return util.etree.Element(self.tag) + + +class BacktickPattern(Pattern): + """ Return a `<code>` element containing the matching text. """ + def __init__(self, pattern): + Pattern.__init__(self, pattern) + self.ESCAPED_BSLASH = '%s%s%s' % (util.STX, ord('\\'), util.ETX) + self.tag = 'code' + + def handleMatch(self, m): + if m.group(4): + el = util.etree.Element(self.tag) + el.text = util.AtomicString(m.group(4).strip()) + return el + else: + return m.group(2).replace('\\\\', self.ESCAPED_BSLASH) + + +class DoubleTagPattern(SimpleTagPattern): + """Return a ElementTree element nested in tag2 nested in tag1. + + Useful for strong emphasis etc. + + """ + def handleMatch(self, m): + tag1, tag2 = self.tag.split(",") + el1 = util.etree.Element(tag1) + el2 = util.etree.SubElement(el1, tag2) + el2.text = m.group(3) + if len(m.groups()) == 5: + el2.tail = m.group(4) + return el1 + + +class HtmlPattern(Pattern): + """ Store raw inline html and return a placeholder. """ + def handleMatch(self, m): + rawhtml = self.unescape(m.group(2)) + place_holder = self.markdown.htmlStash.store(rawhtml) + return place_holder + + def unescape(self, text): + """ Return unescaped text given text with an inline placeholder. """ + try: + stash = self.markdown.treeprocessors['inline'].stashed_nodes + except KeyError: # pragma: no cover + return text + + def get_stash(m): + id = m.group(1) + value = stash.get(id) + if value is not None: + try: + return self.markdown.serializer(value) + except: + return '\%s' % value + + return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text) + + +class LinkPattern(Pattern): + """ Return a link element from the given match. """ + def handleMatch(self, m): + el = util.etree.Element("a") + el.text = m.group(2) + title = m.group(13) + href = m.group(9) + + if href: + if href[0] == "<": + href = href[1:-1] + el.set("href", self.sanitize_url(self.unescape(href.strip()))) + else: + el.set("href", "") + + if title: + title = dequote(self.unescape(title)) + el.set("title", title) + return el + + def sanitize_url(self, url): + """ + Sanitize a url against xss attacks in "safe_mode". + + Rather than specifically blacklisting `javascript:alert("XSS")` and all + its aliases (see <http://ha.ckers.org/xss.html>), we whitelist known + safe url formats. Most urls contain a network location, however some + are known not to (i.e.: mailto links). Script urls do not contain a + location. Additionally, for `javascript:...`, the scheme would be + "javascript" but some aliases will appear to `urlparse()` to have no + scheme. On top of that relative links (i.e.: "foo/bar.html") have no + scheme. Therefore we must check "path", "parameters", "query" and + "fragment" for any literal colons. We don't check "scheme" for colons + because it *should* never have any and "netloc" must allow the form: + `username:password@host:port`. + + """ + if not self.markdown.safeMode: + # Return immediately bipassing parsing. + return url + + try: + scheme, netloc, path, params, query, fragment = url = urlparse(url) + except ValueError: # pragma: no cover + # Bad url - so bad it couldn't be parsed. + return '' + + locless_schemes = ['', 'mailto', 'news'] + allowed_schemes = locless_schemes + ['http', 'https', 'ftp', 'ftps'] + if scheme not in allowed_schemes: + # Not a known (allowed) scheme. Not safe. + return '' + + if netloc == '' and scheme not in locless_schemes: # pragma: no cover + # This should not happen. Treat as suspect. + return '' + + for part in url[2:]: + if ":" in part: + # A colon in "path", "parameters", "query" + # or "fragment" is suspect. + return '' + + # Url passes all tests. Return url as-is. + return urlunparse(url) + + +class ImagePattern(LinkPattern): + """ Return a img element from the given match. """ + def handleMatch(self, m): + el = util.etree.Element("img") + src_parts = m.group(9).split() + if src_parts: + src = src_parts[0] + if src[0] == "<" and src[-1] == ">": + src = src[1:-1] + el.set('src', self.sanitize_url(self.unescape(src))) + else: + el.set('src', "") + if len(src_parts) > 1: + el.set('title', dequote(self.unescape(" ".join(src_parts[1:])))) + + if self.markdown.enable_attributes: + truealt = handleAttributes(m.group(2), el) + else: + truealt = m.group(2) + + el.set('alt', self.unescape(truealt)) + return el + + +class ReferencePattern(LinkPattern): + """ Match to a stored reference and return link element. """ + + NEWLINE_CLEANUP_RE = re.compile(r'[ ]?\n', re.MULTILINE) + + def handleMatch(self, m): + try: + id = m.group(9).lower() + except IndexError: + id = None + if not id: + # if we got something like "[Google][]" or "[Goggle]" + # we'll use "google" as the id + id = m.group(2).lower() + + # Clean up linebreaks in id + id = self.NEWLINE_CLEANUP_RE.sub(' ', id) + if id not in self.markdown.references: # ignore undefined refs + return None + href, title = self.markdown.references[id] + + text = m.group(2) + return self.makeTag(href, title, text) + + def makeTag(self, href, title, text): + el = util.etree.Element('a') + + el.set('href', self.sanitize_url(href)) + if title: + el.set('title', title) + + el.text = text + return el + + +class ImageReferencePattern(ReferencePattern): + """ Match to a stored reference and return img element. """ + def makeTag(self, href, title, text): + el = util.etree.Element("img") + el.set("src", self.sanitize_url(href)) + if title: + el.set("title", title) + + if self.markdown.enable_attributes: + text = handleAttributes(text, el) + + el.set("alt", self.unescape(text)) + return el + + +class AutolinkPattern(Pattern): + """ Return a link Element given an autolink (`<http://example/com>`). """ + def handleMatch(self, m): + el = util.etree.Element("a") + el.set('href', self.unescape(m.group(2))) + el.text = util.AtomicString(m.group(2)) + return el + + +class AutomailPattern(Pattern): + """ + Return a mailto link Element given an automail link (`<foo@example.com>`). + """ + def handleMatch(self, m): + el = util.etree.Element('a') + email = self.unescape(m.group(2)) + if email.startswith("mailto:"): + email = email[len("mailto:"):] + + def codepoint2name(code): + """Return entity definition by code, or the code if not defined.""" + entity = entities.codepoint2name.get(code) + if entity: + return "%s%s;" % (util.AMP_SUBSTITUTE, entity) + else: + return "%s#%d;" % (util.AMP_SUBSTITUTE, code) + + letters = [codepoint2name(ord(letter)) for letter in email] + el.text = util.AtomicString(''.join(letters)) + + mailto = "mailto:" + email + mailto = "".join([util.AMP_SUBSTITUTE + '#%d;' % + ord(letter) for letter in mailto]) + el.set('href', mailto) + return el |