JezK
Edit File: token.py
""" pygments.token ~~~~~~~~~~~~~~ Basic token types and the standard tokens. :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ class _TokenType(tuple): parent = None def split(self): buf = [] node = self while node is not None: buf.append(node) node = node.parent buf.reverse() return buf def __init__(self, *args): # no need to call super.__init__ self.subtypes = set() def __contains__(self, val): return self is val or ( type(val) is self.__class__ and val[:len(self)] == self ) def __getattr__(self, val): if not val or not val[0].isupper(): return tuple.__getattribute__(self, val) new = _TokenType(self + (val,)) setattr(self, val, new) self.subtypes.add(new) new.parent = self return new def __repr__(self): return 'Token' + (self and '.' or '') + '.'.join(self) def __copy__(self): # These instances are supposed to be singletons return self def __deepcopy__(self, memo): # These instances are supposed to be singletons return self Token = _TokenType() # Special token types Text = Token.Text Whitespace = Text.Whitespace Escape = Token.Escape Error = Token.Error # Text that doesn't belong to this lexer (e.g. HTML in PHP) Other = Token.Other # Common token types for source code Keyword = Token.Keyword Name = Token.Name Literal = Token.Literal String = Literal.String Number = Literal.Number Punctuation = Token.Punctuation Operator = Token.Operator Comment = Token.Comment # Generic types for non-source code Generic = Token.Generic # String and some others are not direct children of Token. # alias them: Token.Token = Token Token.String = String Token.Number = Number def is_token_subtype(ttype, other): """ Return True if ``ttype`` is a subtype of ``other``. exists for backwards compatibility. use ``ttype in other`` now. """ return ttype in other def string_to_tokentype(s): """ Convert a string into a token type:: >>> string_to_token('String.Double') Token.Literal.String.Double >>> string_to_token('Token.Literal.Number') Token.Literal.Number >>> string_to_token('') Token Tokens that are already tokens are returned unchanged: >>> string_to_token(String) Token.Literal.String """ if isinstance(s, _TokenType): return s if not s: return Token node = Token for item in s.split('.'): node = getattr(node, item) return node # Map standard token types to short names, used in CSS class naming. # If you add a new item, please be sure to run this file to perform # a consistency check for duplicate values. STANDARD_TYPES = { Token: '', Text: '', Whitespace: 'w', Escape: 'esc', Error: 'err', Other: 'x', Keyword: 'k', Keyword.Constant: 'kc', Keyword.Declaration: 'kd', Keyword.Namespace: 'kn', Keyword.Pseudo: 'kp', Keyword.Reserved: 'kr', Keyword.Type: 'kt', Name: 'n', Name.Attribute: 'na', Name.Builtin: 'nb', Name.Builtin.Pseudo: 'bp', Name.Class: 'nc', Name.Constant: 'no', Name.Decorator: 'nd', Name.Entity: 'ni', Name.Exception: 'ne', Name.Function: 'nf', Name.Function.Magic: 'fm', Name.Property: 'py', Name.Label: 'nl', Name.Namespace: 'nn', Name.Other: 'nx', Name.Tag: 'nt', Name.Variable: 'nv', Name.Variable.Class: 'vc', Name.Variable.Global: 'vg', Name.Variable.Instance: 'vi', Name.Variable.Magic: 'vm', Literal: 'l', Literal.Date: 'ld', String: 's', String.Affix: 'sa', String.Backtick: 'sb', String.Char: 'sc', String.Delimiter: 'dl', String.Doc: 'sd', String.Double: 's2', String.Escape: 'se', String.Heredoc: 'sh', String.Interpol: 'si', String.Other: 'sx', String.Regex: 'sr', String.Single: 's1', String.Symbol: 'ss', Number: 'm', Number.Bin: 'mb', Number.Float: 'mf', Number.Hex: 'mh', Number.Integer: 'mi', Number.Integer.Long: 'il', Number.Oct: 'mo', Operator: 'o', Operator.Word: 'ow', Punctuation: 'p', Punctuation.Marker: 'pm', Comment: 'c', Comment.Hashbang: 'ch', Comment.Multiline: 'cm', Comment.Preproc: 'cp', Comment.PreprocFile: 'cpf', Comment.Single: 'c1', Comment.Special: 'cs', Generic: 'g', Generic.Deleted: 'gd', Generic.Emph: 'ge', Generic.Error: 'gr', Generic.Heading: 'gh', Generic.Inserted: 'gi', Generic.Output: 'go', Generic.Prompt: 'gp', Generic.Strong: 'gs', Generic.Subheading: 'gu', Generic.Traceback: 'gt', }