JezK
Edit File: lexer.py
# mako/lexer.py # Copyright 2006-2022 the Mako authors and contributors <see AUTHORS file> # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """provides the Lexer class for parsing template strings into parse trees.""" import codecs import re from mako import exceptions from mako import parsetree from mako.pygen import adjust_whitespace _regexp_cache = {} class Lexer: def __init__( self, text, filename=None, input_encoding=None, preprocessor=None ): self.text = text self.filename = filename self.template = parsetree.TemplateNode(self.filename) self.matched_lineno = 1 self.matched_charpos = 0 self.lineno = 1 self.match_position = 0 self.tag = [] self.control_line = [] self.ternary_stack = [] self.encoding = input_encoding if preprocessor is None: self.preprocessor = [] elif not hasattr(preprocessor, "__iter__"): self.preprocessor = [preprocessor] else: self.preprocessor = preprocessor @property def exception_kwargs(self): return { "source": self.text, "lineno": self.matched_lineno, "pos": self.matched_charpos, "filename": self.filename, } def match(self, regexp, flags=None): """compile the given regexp, cache the reg, and call match_reg().""" try: reg = _regexp_cache[(regexp, flags)] except KeyError: reg = re.compile(regexp, flags) if flags else re.compile(regexp) _regexp_cache[(regexp, flags)] = reg return self.match_reg(reg) def match_reg(self, reg): """match the given regular expression object to the current text position. if a match occurs, update the current text and line position. """ mp = self.match_position match = reg.match(self.text, self.match_position) if match: (start, end) = match.span() self.match_position = end + 1 if end == start else end self.matched_lineno = self.lineno cp = mp - 1 if cp >= 0 and cp < self.textlength: cp = self.text[: cp + 1].rfind("\n") self.matched_charpos = mp - cp self.lineno += self.text[mp : self.match_position].count("\n") return match def parse_until_text(self, watch_nesting, *text): startpos = self.match_position text_re = r"|".join(text) brace_level = 0 paren_level = 0 bracket_level = 0 while True: match = self.match(r"#.*\n") if match: continue match = self.match( r"(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1", re.S ) if match: continue match = self.match(r"(%s)" % text_re) if match and not ( watch_nesting and (brace_level > 0 or paren_level > 0 or bracket_level > 0) ): return ( self.text[ startpos : self.match_position - len(match.group(1)) ], match.group(1), ) elif not match: match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) if match: brace_level += match.group(1).count("{") brace_level -= match.group(1).count("}") paren_level += match.group(1).count("(") paren_level -= match.group(1).count(")") bracket_level += match.group(1).count("[") bracket_level -= match.group(1).count("]") continue raise exceptions.SyntaxException( "Expected: %s" % ",".join(text), **self.exception_kwargs ) def append_node(self, nodecls, *args, **kwargs): kwargs.setdefault("source", self.text) kwargs.setdefault("lineno", self.matched_lineno) kwargs.setdefault("pos", self.matched_charpos) kwargs["filename"] = self.filename node = nodecls(*args, **kwargs) if len(self.tag): self.tag[-1].nodes.append(node) else: self.template.nodes.append(node) # build a set of child nodes for the control line # (used for loop variable detection) # also build a set of child nodes on ternary control lines # (used for determining if a pass needs to be auto-inserted if self.control_line: control_frame = self.control_line[-1] control_frame.nodes.append(node) if ( not ( isinstance(node, parsetree.ControlLine) and control_frame.is_ternary(node.keyword) ) and self.ternary_stack and self.ternary_stack[-1] ): self.ternary_stack[-1][-1].nodes.append(node) if isinstance(node, parsetree.Tag): if len(self.tag): node.parent = self.tag[-1] self.tag.append(node) elif isinstance(node, parsetree.ControlLine): if node.isend: self.control_line.pop() self.ternary_stack.pop() elif node.is_primary: self.control_line.append(node) self.ternary_stack.append([]) elif self.control_line and self.control_line[-1].is_ternary( node.keyword ): self.ternary_stack[-1].append(node) elif self.control_line and not self.control_line[-1].is_ternary( node.keyword ): raise exceptions.SyntaxException( "Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), **self.exception_kwargs, ) _coding_re = re.compile(r"#.*coding[:=]\s*([-\w.]+).*\r?\n") def decode_raw_stream(self, text, decode_raw, known_encoding, filename): """given string/unicode or bytes/string, determine encoding from magic encoding comment, return body as unicode or raw if decode_raw=False """ if isinstance(text, str): m = self._coding_re.match(text) encoding = m and m.group(1) or known_encoding or "utf-8" return encoding, text if text.startswith(codecs.BOM_UTF8): text = text[len(codecs.BOM_UTF8) :] parsed_encoding = "utf-8" m = self._coding_re.match(text.decode("utf-8", "ignore")) if m is not None and m.group(1) != "utf-8": raise exceptions.CompileException( "Found utf-8 BOM in file, with conflicting " "magic encoding comment of '%s'" % m.group(1), text.decode("utf-8", "ignore"), 0, 0, filename, ) else: m = self._coding_re.match(text.decode("utf-8", "ignore")) parsed_encoding = m.group(1) if m else known_encoding or "utf-8" if decode_raw: try: text = text.decode(parsed_encoding) except UnicodeDecodeError: raise exceptions.CompileException( "Unicode decode operation of encoding '%s' failed" % parsed_encoding, text.decode("utf-8", "ignore"), 0, 0, filename, ) return parsed_encoding, text def parse(self): self.encoding, self.text = self.decode_raw_stream( self.text, True, self.encoding, self.filename ) for preproc in self.preprocessor: self.text = preproc(self.text) # push the match marker past the # encoding comment. self.match_reg(self._coding_re) self.textlength = len(self.text) while True: if self.match_position > self.textlength: break if self.match_end(): break if self.match_expression(): continue if self.match_control_line(): continue if self.match_comment(): continue if self.match_tag_start(): continue if self.match_tag_end(): continue if self.match_python_block(): continue if self.match_text(): continue if self.match_position > self.textlength: break # TODO: no coverage here raise exceptions.MakoException("assertion failed") if len(self.tag): raise exceptions.SyntaxException( "Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs, ) if len(self.control_line): raise exceptions.SyntaxException( "Unterminated control keyword: '%s'" % self.control_line[-1].keyword, self.text, self.control_line[-1].lineno, self.control_line[-1].pos, self.filename, ) return self.template def match_tag_start(self): reg = r""" \<% # opening tag ([\w\.\:]+) # keyword ((?:\s+\w+|\s*=\s*|"[^"]*?"|'[^']*?'|\s*,\s*)*) # attrname, = \ # sign, string expression # comma is for backwards compat # identified in #366 \s* # more whitespace (/)?> # closing """ match = self.match( reg, re.I | re.S | re.X, ) if not match: return False keyword, attr, isend = match.groups() self.keyword = keyword attributes = {} if attr: for att in re.findall( r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr ): key, val1, val2 = att text = val1 or val2 text = text.replace("\r\n", "\n") attributes[key] = text self.append_node(parsetree.Tag, keyword, attributes) if isend: self.tag.pop() elif keyword == "text": match = self.match(r"(.*?)(?=\</%text>)", re.S) if not match: raise exceptions.SyntaxException( "Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs, ) self.append_node(parsetree.Text, match.group(1)) return self.match_tag_end() return True def match_tag_end(self): match = self.match(r"\</%[\t ]*([^\t ]+?)[\t ]*>") if match: if not len(self.tag): raise exceptions.SyntaxException( "Closing tag without opening tag: </%%%s>" % match.group(1), **self.exception_kwargs, ) elif self.tag[-1].keyword != match.group(1): raise exceptions.SyntaxException( "Closing tag </%%%s> does not match tag: <%%%s>" % (match.group(1), self.tag[-1].keyword), **self.exception_kwargs, ) self.tag.pop() return True else: return False def match_end(self): match = self.match(r"\Z", re.S) if not match: return False string = match.group() if string: return string else: return True def match_text(self): match = self.match( r""" (.*?) # anything, followed by: ( (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based # comment preceded by a # consumed newline and whitespace | (?=\${) # an expression | (?=</?[%&]) # a substitution or block or call start or end # - don't consume | (\\\r?\n) # an escaped newline - throw away | \Z # end of string )""", re.X | re.S, ) if match: text = match.group(1) if text: self.append_node(parsetree.Text, text) return True else: return False def match_python_block(self): match = self.match(r"<%(!)?") if match: line, pos = self.matched_lineno, self.matched_charpos text, end = self.parse_until_text(False, r"%>") # the trailing newline helps # compiler.parse() not complain about indentation text = adjust_whitespace(text) + "\n" self.append_node( parsetree.Code, text, match.group(1) == "!", lineno=line, pos=pos, ) return True else: return False def match_expression(self): match = self.match(r"\${") if not match: return False line, pos = self.matched_lineno, self.matched_charpos text, end = self.parse_until_text(True, r"\|", r"}") if end == "|": escapes, end = self.parse_until_text(True, r"}") else: escapes = "" text = text.replace("\r\n", "\n") self.append_node( parsetree.Expression, text, escapes.strip(), lineno=line, pos=pos, ) return True def match_control_line(self): match = self.match( r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\\r?\n)|[^\r\n])*)" r"(?:\r?\n|\Z)", re.M, ) if not match: return False operator = match.group(1) text = match.group(2) if operator == "%": m2 = re.match(r"(end)?(\w+)\s*(.*)", text) if not m2: raise exceptions.SyntaxException( "Invalid control line: '%s'" % text, **self.exception_kwargs, ) isend, keyword = m2.group(1, 2) isend = isend is not None if isend: if not len(self.control_line): raise exceptions.SyntaxException( "No starting keyword '%s' for '%s'" % (keyword, text), **self.exception_kwargs, ) elif self.control_line[-1].keyword != keyword: raise exceptions.SyntaxException( "Keyword '%s' doesn't match keyword '%s'" % (text, self.control_line[-1].keyword), **self.exception_kwargs, ) self.append_node(parsetree.ControlLine, keyword, isend, text) else: self.append_node(parsetree.Comment, text) return True def match_comment(self): """matches the multiline version of a comment""" match = self.match(r"<%doc>(.*?)</%doc>", re.S) if match: self.append_node(parsetree.Comment, match.group(1)) return True else: return False