| """ |
| pygments.lexers.python |
| ~~~~~~~~~~~~~~~~~~~~~~ |
| |
| Lexers for Python and related languages. |
| |
| :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. |
| :license: BSD, see LICENSE for details. |
| """ |
| |
| import re |
| import keyword |
| |
| from pip._vendor.pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ |
| default, words, combined, do_insertions, this |
| from pip._vendor.pygments.util import get_bool_opt, shebang_matches |
| from pip._vendor.pygments.token import Text, Comment, Operator, Keyword, Name, String, \ |
| Number, Punctuation, Generic, Other, Error |
| from pip._vendor.pygments import unistring as uni |
| |
| __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', |
| 'Python2Lexer', 'Python2TracebackLexer', |
| 'CythonLexer', 'DgLexer', 'NumPyLexer'] |
| |
| line_re = re.compile('.*?\n') |
| |
| |
| class PythonLexer(RegexLexer): |
| """ |
| For Python source code (version 3.x). |
| |
| .. versionadded:: 0.10 |
| |
| .. versionchanged:: 2.5 |
| This is now the default ``PythonLexer``. It is still available as the |
| alias ``Python3Lexer``. |
| """ |
| |
| name = 'Python' |
| url = 'http://www.python.org' |
| aliases = ['python', 'py', 'sage', 'python3', 'py3'] |
| filenames = [ |
| '*.py', |
| '*.pyw', |
| # Jython |
| '*.jy', |
| # Sage |
| '*.sage', |
| # SCons |
| '*.sc', |
| 'SConstruct', |
| 'SConscript', |
| # Skylark/Starlark (used by Bazel, Buck, and Pants) |
| '*.bzl', |
| 'BUCK', |
| 'BUILD', |
| 'BUILD.bazel', |
| 'WORKSPACE', |
| # Twisted Application infrastructure |
| '*.tac', |
| ] |
| mimetypes = ['text/x-python', 'application/x-python', |
| 'text/x-python3', 'application/x-python3'] |
| |
| uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue) |
| |
| def innerstring_rules(ttype): |
| return [ |
| # the old style '%s' % (...) string formatting (still valid in Py3) |
| (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' |
| '[hlL]?[E-GXc-giorsaux%]', String.Interpol), |
| # the new style '{}'.format(...) string formatting |
| (r'\{' |
| r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name |
| r'(\![sra])?' # conversion |
| r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' |
| r'\}', String.Interpol), |
| |
| # backslashes, quotes and formatting signs must be parsed one at a time |
| (r'[^\\\'"%{\n]+', ttype), |
| (r'[\'"\\]', ttype), |
| # unhandled string formatting sign |
| (r'%|(\{{1,2})', ttype) |
| # newlines are an error (use "nl" state) |
| ] |
| |
| def fstring_rules(ttype): |
| return [ |
| # Assuming that a '}' is the closing brace after format specifier. |
| # Sadly, this means that we won't detect syntax error. But it's |
| # more important to parse correct syntax correctly, than to |
| # highlight invalid syntax. |
| (r'\}', String.Interpol), |
| (r'\{', String.Interpol, 'expr-inside-fstring'), |
| # backslashes, quotes and formatting signs must be parsed one at a time |
| (r'[^\\\'"{}\n]+', ttype), |
| (r'[\'"\\]', ttype), |
| # newlines are an error (use "nl" state) |
| ] |
| |
| tokens = { |
| 'root': [ |
| (r'\n', Text), |
| (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', |
| bygroups(Text, String.Affix, String.Doc)), |
| (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", |
| bygroups(Text, String.Affix, String.Doc)), |
| (r'\A#!.+$', Comment.Hashbang), |
| (r'#.*$', Comment.Single), |
| (r'\\\n', Text), |
| (r'\\', Text), |
| include('keywords'), |
| include('soft-keywords'), |
| (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), |
| (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), |
| (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), |
| 'fromimport'), |
| (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), |
| 'import'), |
| include('expr'), |
| ], |
| 'expr': [ |
| # raw f-strings |
| ('(?i)(rf|fr)(""")', |
| bygroups(String.Affix, String.Double), |
| combined('rfstringescape', 'tdqf')), |
| ("(?i)(rf|fr)(''')", |
| bygroups(String.Affix, String.Single), |
| combined('rfstringescape', 'tsqf')), |
| ('(?i)(rf|fr)(")', |
| bygroups(String.Affix, String.Double), |
| combined('rfstringescape', 'dqf')), |
| ("(?i)(rf|fr)(')", |
| bygroups(String.Affix, String.Single), |
| combined('rfstringescape', 'sqf')), |
| # non-raw f-strings |
| ('([fF])(""")', bygroups(String.Affix, String.Double), |
| combined('fstringescape', 'tdqf')), |
| ("([fF])(''')", bygroups(String.Affix, String.Single), |
| combined('fstringescape', 'tsqf')), |
| ('([fF])(")', bygroups(String.Affix, String.Double), |
| combined('fstringescape', 'dqf')), |
| ("([fF])(')", bygroups(String.Affix, String.Single), |
| combined('fstringescape', 'sqf')), |
| # raw bytes and strings |
| ('(?i)(rb|br|r)(""")', |
| bygroups(String.Affix, String.Double), 'tdqs'), |
| ("(?i)(rb|br|r)(''')", |
| bygroups(String.Affix, String.Single), 'tsqs'), |
| ('(?i)(rb|br|r)(")', |
| bygroups(String.Affix, String.Double), 'dqs'), |
| ("(?i)(rb|br|r)(')", |
| bygroups(String.Affix, String.Single), 'sqs'), |
| # non-raw strings |
| ('([uU]?)(""")', bygroups(String.Affix, String.Double), |
| combined('stringescape', 'tdqs')), |
| ("([uU]?)(''')", bygroups(String.Affix, String.Single), |
| combined('stringescape', 'tsqs')), |
| ('([uU]?)(")', bygroups(String.Affix, String.Double), |
| combined('stringescape', 'dqs')), |
| ("([uU]?)(')", bygroups(String.Affix, String.Single), |
| combined('stringescape', 'sqs')), |
| # non-raw bytes |
| ('([bB])(""")', bygroups(String.Affix, String.Double), |
| combined('bytesescape', 'tdqs')), |
| ("([bB])(''')", bygroups(String.Affix, String.Single), |
| combined('bytesescape', 'tsqs')), |
| ('([bB])(")', bygroups(String.Affix, String.Double), |
| combined('bytesescape', 'dqs')), |
| ("([bB])(')", bygroups(String.Affix, String.Single), |
| combined('bytesescape', 'sqs')), |
| |
| (r'[^\S\n]+', Text), |
| include('numbers'), |
| (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator), |
| (r'[]{}:(),;[]', Punctuation), |
| (r'(in|is|and|or|not)\b', Operator.Word), |
| include('expr-keywords'), |
| include('builtins'), |
| include('magicfuncs'), |
| include('magicvars'), |
| include('name'), |
| ], |
| 'expr-inside-fstring': [ |
| (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), |
| # without format specifier |
| (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) |
| r'(\![sraf])?' # conversion |
| r'\}', String.Interpol, '#pop'), |
| # with format specifier |
| # we'll catch the remaining '}' in the outer scope |
| (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) |
| r'(\![sraf])?' # conversion |
| r':', String.Interpol, '#pop'), |
| (r'\s+', Text), # allow new lines |
| include('expr'), |
| ], |
| 'expr-inside-fstring-inner': [ |
| (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), |
| (r'[])}]', Punctuation, '#pop'), |
| (r'\s+', Text), # allow new lines |
| include('expr'), |
| ], |
| 'expr-keywords': [ |
| # Based on https://docs.python.org/3/reference/expressions.html |
| (words(( |
| 'async for', 'await', 'else', 'for', 'if', 'lambda', |
| 'yield', 'yield from'), suffix=r'\b'), |
| Keyword), |
| (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), |
| ], |
| 'keywords': [ |
| (words(( |
| 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif', |
| 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', |
| 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', |
| 'yield from', 'as', 'with'), suffix=r'\b'), |
| Keyword), |
| (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), |
| ], |
| 'soft-keywords': [ |
| # `match`, `case` and `_` soft keywords |
| (r'(^[ \t]*)' # at beginning of line + possible indentation |
| r'(match|case)\b' # a possible keyword |
| r'(?![ \t]*(?:' # not followed by... |
| r'[:,;=^&|@~)\]}]|(?:' + # characters and keywords that mean this isn't |
| r'|'.join(keyword.kwlist) + r')\b))', # pattern matching |
| bygroups(Text, Keyword), 'soft-keywords-inner'), |
| ], |
| 'soft-keywords-inner': [ |
| # optional `_` keyword |
| (r'(\s+)([^\n_]*)(_\b)', bygroups(Text, using(this), Keyword)), |
| default('#pop') |
| ], |
| 'builtins': [ |
| (words(( |
| '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', |
| 'breakpoint', 'bytes', 'chr', 'classmethod', 'compile', 'complex', |
| 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'filter', |
| 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', |
| 'hash', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', |
| 'iter', 'len', 'list', 'locals', 'map', 'max', 'memoryview', |
| 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', |
| 'property', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', |
| 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', |
| 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'), |
| Name.Builtin), |
| (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo), |
| (words(( |
| 'ArithmeticError', 'AssertionError', 'AttributeError', |
| 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning', |
| 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', |
| 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', |
| 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', |
| 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', |
| 'NotImplementedError', 'OSError', 'OverflowError', |
| 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning', |
| 'RuntimeError', 'RuntimeWarning', 'StopIteration', |
| 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', |
| 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', |
| 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', |
| 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', |
| 'Warning', 'WindowsError', 'ZeroDivisionError', |
| # new builtin exceptions from PEP 3151 |
| 'BlockingIOError', 'ChildProcessError', 'ConnectionError', |
| 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError', |
| 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError', |
| 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError', |
| 'PermissionError', 'ProcessLookupError', 'TimeoutError', |
| # others new in Python 3 |
| 'StopAsyncIteration', 'ModuleNotFoundError', 'RecursionError', |
| 'EncodingWarning'), |
| prefix=r'(?<!\.)', suffix=r'\b'), |
| Name.Exception), |
| ], |
| 'magicfuncs': [ |
| (words(( |
| '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__', |
| '__and__', '__anext__', '__await__', '__bool__', '__bytes__', |
| '__call__', '__complex__', '__contains__', '__del__', '__delattr__', |
| '__delete__', '__delitem__', '__dir__', '__divmod__', '__enter__', |
| '__eq__', '__exit__', '__float__', '__floordiv__', '__format__', |
| '__ge__', '__get__', '__getattr__', '__getattribute__', |
| '__getitem__', '__gt__', '__hash__', '__iadd__', '__iand__', |
| '__ifloordiv__', '__ilshift__', '__imatmul__', '__imod__', |
| '__imul__', '__index__', '__init__', '__instancecheck__', |
| '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__', |
| '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__', |
| '__len__', '__length_hint__', '__lshift__', '__lt__', '__matmul__', |
| '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', |
| '__new__', '__next__', '__or__', '__pos__', '__pow__', |
| '__prepare__', '__radd__', '__rand__', '__rdivmod__', '__repr__', |
| '__reversed__', '__rfloordiv__', '__rlshift__', '__rmatmul__', |
| '__rmod__', '__rmul__', '__ror__', '__round__', '__rpow__', |
| '__rrshift__', '__rshift__', '__rsub__', '__rtruediv__', |
| '__rxor__', '__set__', '__setattr__', '__setitem__', '__str__', |
| '__sub__', '__subclasscheck__', '__truediv__', |
| '__xor__'), suffix=r'\b'), |
| Name.Function.Magic), |
| ], |
| 'magicvars': [ |
| (words(( |
| '__annotations__', '__bases__', '__class__', '__closure__', |
| '__code__', '__defaults__', '__dict__', '__doc__', '__file__', |
| '__func__', '__globals__', '__kwdefaults__', '__module__', |
| '__mro__', '__name__', '__objclass__', '__qualname__', |
| '__self__', '__slots__', '__weakref__'), suffix=r'\b'), |
| Name.Variable.Magic), |
| ], |
| 'numbers': [ |
| (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)' |
| r'([eE][+-]?\d(?:_?\d)*)?', Number.Float), |
| (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float), |
| (r'0[oO](?:_?[0-7])+', Number.Oct), |
| (r'0[bB](?:_?[01])+', Number.Bin), |
| (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), |
| (r'\d(?:_?\d)*', Number.Integer), |
| ], |
| 'name': [ |
| (r'@' + uni_name, Name.Decorator), |
| (r'@', Operator), # new matrix multiplication operator |
| (uni_name, Name), |
| ], |
| 'funcname': [ |
| include('magicfuncs'), |
| (uni_name, Name.Function, '#pop'), |
| default('#pop'), |
| ], |
| 'classname': [ |
| (uni_name, Name.Class, '#pop'), |
| ], |
| 'import': [ |
| (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), |
| (r'\.', Name.Namespace), |
| (uni_name, Name.Namespace), |
| (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), |
| default('#pop') # all else: go back |
| ], |
| 'fromimport': [ |
| (r'(\s+)(import)\b', bygroups(Text, Keyword.Namespace), '#pop'), |
| (r'\.', Name.Namespace), |
| # if None occurs here, it's "raise x from None", since None can |
| # never be a module name |
| (r'None\b', Name.Builtin.Pseudo, '#pop'), |
| (uni_name, Name.Namespace), |
| default('#pop'), |
| ], |
| 'rfstringescape': [ |
| (r'\{\{', String.Escape), |
| (r'\}\}', String.Escape), |
| ], |
| 'fstringescape': [ |
| include('rfstringescape'), |
| include('stringescape'), |
| ], |
| 'bytesescape': [ |
| (r'\\([\\abfnrtv"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) |
| ], |
| 'stringescape': [ |
| (r'\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})', String.Escape), |
| include('bytesescape') |
| ], |
| 'fstrings-single': fstring_rules(String.Single), |
| 'fstrings-double': fstring_rules(String.Double), |
| 'strings-single': innerstring_rules(String.Single), |
| 'strings-double': innerstring_rules(String.Double), |
| 'dqf': [ |
| (r'"', String.Double, '#pop'), |
| (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings |
| include('fstrings-double') |
| ], |
| 'sqf': [ |
| (r"'", String.Single, '#pop'), |
| (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings |
| include('fstrings-single') |
| ], |
| 'dqs': [ |
| (r'"', String.Double, '#pop'), |
| (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings |
| include('strings-double') |
| ], |
| 'sqs': [ |
| (r"'", String.Single, '#pop'), |
| (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings |
| include('strings-single') |
| ], |
| 'tdqf': [ |
| (r'"""', String.Double, '#pop'), |
| include('fstrings-double'), |
| (r'\n', String.Double) |
| ], |
| 'tsqf': [ |
| (r"'''", String.Single, '#pop'), |
| include('fstrings-single'), |
| (r'\n', String.Single) |
| ], |
| 'tdqs': [ |
| (r'"""', String.Double, '#pop'), |
| include('strings-double'), |
| (r'\n', String.Double) |
| ], |
| 'tsqs': [ |
| (r"'''", String.Single, '#pop'), |
| include('strings-single'), |
| (r'\n', String.Single) |
| ], |
| } |
| |
| def analyse_text(text): |
| return shebang_matches(text, r'pythonw?(3(\.\d)?)?') or \ |
| 'import ' in text[:1000] |
| |
| |
| Python3Lexer = PythonLexer |
| |
| |
| class Python2Lexer(RegexLexer): |
| """ |
| For Python 2.x source code. |
| |
| .. versionchanged:: 2.5 |
| This class has been renamed from ``PythonLexer``. ``PythonLexer`` now |
| refers to the Python 3 variant. File name patterns like ``*.py`` have |
| been moved to Python 3 as well. |
| """ |
| |
| name = 'Python 2.x' |
| url = 'http://www.python.org' |
| aliases = ['python2', 'py2'] |
| filenames = [] # now taken over by PythonLexer (3.x) |
| mimetypes = ['text/x-python2', 'application/x-python2'] |
| |
| def innerstring_rules(ttype): |
| return [ |
| # the old style '%s' % (...) string formatting |
| (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' |
| '[hlL]?[E-GXc-giorsux%]', String.Interpol), |
| # backslashes, quotes and formatting signs must be parsed one at a time |
| (r'[^\\\'"%\n]+', ttype), |
| (r'[\'"\\]', ttype), |
| # unhandled string formatting sign |
| (r'%', ttype), |
| # newlines are an error (use "nl" state) |
| ] |
| |
| tokens = { |
| 'root': [ |
| (r'\n', Text), |
| (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', |
| bygroups(Text, String.Affix, String.Doc)), |
| (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", |
| bygroups(Text, String.Affix, String.Doc)), |
| (r'[^\S\n]+', Text), |
| (r'\A#!.+$', Comment.Hashbang), |
| (r'#.*$', Comment.Single), |
| (r'[]{}:(),;[]', Punctuation), |
| (r'\\\n', Text), |
| (r'\\', Text), |
| (r'(in|is|and|or|not)\b', Operator.Word), |
| (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator), |
| include('keywords'), |
| (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), |
| (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), |
| (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), |
| 'fromimport'), |
| (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), |
| 'import'), |
| include('builtins'), |
| include('magicfuncs'), |
| include('magicvars'), |
| include('backtick'), |
| ('([rR]|[uUbB][rR]|[rR][uUbB])(""")', |
| bygroups(String.Affix, String.Double), 'tdqs'), |
| ("([rR]|[uUbB][rR]|[rR][uUbB])(''')", |
| bygroups(String.Affix, String.Single), 'tsqs'), |
| ('([rR]|[uUbB][rR]|[rR][uUbB])(")', |
| bygroups(String.Affix, String.Double), 'dqs'), |
| ("([rR]|[uUbB][rR]|[rR][uUbB])(')", |
| bygroups(String.Affix, String.Single), 'sqs'), |
| ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), |
| combined('stringescape', 'tdqs')), |
| ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), |
| combined('stringescape', 'tsqs')), |
| ('([uUbB]?)(")', bygroups(String.Affix, String.Double), |
| combined('stringescape', 'dqs')), |
| ("([uUbB]?)(')", bygroups(String.Affix, String.Single), |
| combined('stringescape', 'sqs')), |
| include('name'), |
| include('numbers'), |
| ], |
| 'keywords': [ |
| (words(( |
| 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except', |
| 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass', |
| 'print', 'raise', 'return', 'try', 'while', 'yield', |
| 'yield from', 'as', 'with'), suffix=r'\b'), |
| Keyword), |
| ], |
| 'builtins': [ |
| (words(( |
| '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', |
| 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod', |
| 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod', |
| 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float', |
| 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id', |
| 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len', |
| 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object', |
| 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce', |
| 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', |
| 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', |
| 'unichr', 'unicode', 'vars', 'xrange', 'zip'), |
| prefix=r'(?<!\.)', suffix=r'\b'), |
| Name.Builtin), |
| (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls' |
| r')\b', Name.Builtin.Pseudo), |
| (words(( |
| 'ArithmeticError', 'AssertionError', 'AttributeError', |
| 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError', |
| 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', |
| 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', |
| 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', |
| 'MemoryError', 'NameError', |
| 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning', |
| 'PendingDeprecationWarning', 'ReferenceError', |
| 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration', |
| 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', |
| 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', |
| 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', |
| 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning', |
| 'WindowsError', 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'), |
| Name.Exception), |
| ], |
| 'magicfuncs': [ |
| (words(( |
| '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__', |
| '__complex__', '__contains__', '__del__', '__delattr__', '__delete__', |
| '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__', |
| '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__', |
| '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__', |
| '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__', |
| '__ilshift__', '__imod__', '__imul__', '__index__', '__init__', |
| '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__', |
| '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__', |
| '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__', |
| '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__', |
| '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__', |
| '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__', |
| '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__', |
| '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__', |
| '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__', |
| '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__', |
| '__unicode__', '__xor__'), suffix=r'\b'), |
| Name.Function.Magic), |
| ], |
| 'magicvars': [ |
| (words(( |
| '__bases__', '__class__', '__closure__', '__code__', '__defaults__', |
| '__dict__', '__doc__', '__file__', '__func__', '__globals__', |
| '__metaclass__', '__module__', '__mro__', '__name__', '__self__', |
| '__slots__', '__weakref__'), |
| suffix=r'\b'), |
| Name.Variable.Magic), |
| ], |
| 'numbers': [ |
| (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), |
| (r'\d+[eE][+-]?[0-9]+j?', Number.Float), |
| (r'0[0-7]+j?', Number.Oct), |
| (r'0[bB][01]+', Number.Bin), |
| (r'0[xX][a-fA-F0-9]+', Number.Hex), |
| (r'\d+L', Number.Integer.Long), |
| (r'\d+j?', Number.Integer) |
| ], |
| 'backtick': [ |
| ('`.*?`', String.Backtick), |
| ], |
| 'name': [ |
| (r'@[\w.]+', Name.Decorator), |
| (r'[a-zA-Z_]\w*', Name), |
| ], |
| 'funcname': [ |
| include('magicfuncs'), |
| (r'[a-zA-Z_]\w*', Name.Function, '#pop'), |
| default('#pop'), |
| ], |
| 'classname': [ |
| (r'[a-zA-Z_]\w*', Name.Class, '#pop') |
| ], |
| 'import': [ |
| (r'(?:[ \t]|\\\n)+', Text), |
| (r'as\b', Keyword.Namespace), |
| (r',', Operator), |
| (r'[a-zA-Z_][\w.]*', Name.Namespace), |
| default('#pop') # all else: go back |
| ], |
| 'fromimport': [ |
| (r'(?:[ \t]|\\\n)+', Text), |
| (r'import\b', Keyword.Namespace, '#pop'), |
| # if None occurs here, it's "raise x from None", since None can |
| # never be a module name |
| (r'None\b', Name.Builtin.Pseudo, '#pop'), |
| # sadly, in "raise x from y" y will be highlighted as namespace too |
| (r'[a-zA-Z_.][\w.]*', Name.Namespace), |
| # anything else here also means "raise x from y" and is therefore |
| # not an error |
| default('#pop'), |
| ], |
| 'stringescape': [ |
| (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' |
| r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) |
| ], |
| 'strings-single': innerstring_rules(String.Single), |
| 'strings-double': innerstring_rules(String.Double), |
| 'dqs': [ |
| (r'"', String.Double, '#pop'), |
| (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings |
| include('strings-double') |
| ], |
| 'sqs': [ |
| (r"'", String.Single, '#pop'), |
| (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings |
| include('strings-single') |
| ], |
| 'tdqs': [ |
| (r'"""', String.Double, '#pop'), |
| include('strings-double'), |
| (r'\n', String.Double) |
| ], |
| 'tsqs': [ |
| (r"'''", String.Single, '#pop'), |
| include('strings-single'), |
| (r'\n', String.Single) |
| ], |
| } |
| |
| def analyse_text(text): |
| return shebang_matches(text, r'pythonw?2(\.\d)?') |
| |
| |
| class PythonConsoleLexer(Lexer): |
| """ |
| For Python console output or doctests, such as: |
| |
| .. sourcecode:: pycon |
| |
| >>> a = 'foo' |
| >>> print a |
| foo |
| >>> 1 / 0 |
| Traceback (most recent call last): |
| File "<stdin>", line 1, in <module> |
| ZeroDivisionError: integer division or modulo by zero |
| |
| Additional options: |
| |
| `python3` |
| Use Python 3 lexer for code. Default is ``True``. |
| |
| .. versionadded:: 1.0 |
| .. versionchanged:: 2.5 |
| Now defaults to ``True``. |
| """ |
| name = 'Python console session' |
| aliases = ['pycon'] |
| mimetypes = ['text/x-python-doctest'] |
| |
| def __init__(self, **options): |
| self.python3 = get_bool_opt(options, 'python3', True) |
| Lexer.__init__(self, **options) |
| |
| def get_tokens_unprocessed(self, text): |
| if self.python3: |
| pylexer = PythonLexer(**self.options) |
| tblexer = PythonTracebackLexer(**self.options) |
| else: |
| pylexer = Python2Lexer(**self.options) |
| tblexer = Python2TracebackLexer(**self.options) |
| |
| curcode = '' |
| insertions = [] |
| curtb = '' |
| tbindex = 0 |
| tb = 0 |
| for match in line_re.finditer(text): |
| line = match.group() |
| if line.startswith('>>> ') or line.startswith('... '): |
| tb = 0 |
| insertions.append((len(curcode), |
| [(0, Generic.Prompt, line[:4])])) |
| curcode += line[4:] |
| elif line.rstrip() == '...' and not tb: |
| # only a new >>> prompt can end an exception block |
| # otherwise an ellipsis in place of the traceback frames |
| # will be mishandled |
| insertions.append((len(curcode), |
| [(0, Generic.Prompt, '...')])) |
| curcode += line[3:] |
| else: |
| if curcode: |
| yield from do_insertions( |
| insertions, pylexer.get_tokens_unprocessed(curcode)) |
| curcode = '' |
| insertions = [] |
| if (line.startswith('Traceback (most recent call last):') or |
| re.match(' File "[^"]+", line \\d+\\n$', line)): |
| tb = 1 |
| curtb = line |
| tbindex = match.start() |
| elif line == 'KeyboardInterrupt\n': |
| yield match.start(), Name.Class, line |
| elif tb: |
| curtb += line |
| if not (line.startswith(' ') or line.strip() == '...'): |
| tb = 0 |
| for i, t, v in tblexer.get_tokens_unprocessed(curtb): |
| yield tbindex+i, t, v |
| curtb = '' |
| else: |
| yield match.start(), Generic.Output, line |
| if curcode: |
| yield from do_insertions(insertions, |
| pylexer.get_tokens_unprocessed(curcode)) |
| if curtb: |
| for i, t, v in tblexer.get_tokens_unprocessed(curtb): |
| yield tbindex+i, t, v |
| |
| |
| class PythonTracebackLexer(RegexLexer): |
| """ |
| For Python 3.x tracebacks, with support for chained exceptions. |
| |
| .. versionadded:: 1.0 |
| |
| .. versionchanged:: 2.5 |
| This is now the default ``PythonTracebackLexer``. It is still available |
| as the alias ``Python3TracebackLexer``. |
| """ |
| |
| name = 'Python Traceback' |
| aliases = ['pytb', 'py3tb'] |
| filenames = ['*.pytb', '*.py3tb'] |
| mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback'] |
| |
| tokens = { |
| 'root': [ |
| (r'\n', Text), |
| (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'), |
| (r'^During handling of the above exception, another ' |
| r'exception occurred:\n\n', Generic.Traceback), |
| (r'^The above exception was the direct cause of the ' |
| r'following exception:\n\n', Generic.Traceback), |
| (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), |
| (r'^.*\n', Other), |
| ], |
| 'intb': [ |
| (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', |
| bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)), |
| (r'^( File )("[^"]+")(, line )(\d+)(\n)', |
| bygroups(Text, Name.Builtin, Text, Number, Text)), |
| (r'^( )(.+)(\n)', |
| bygroups(Text, using(PythonLexer), Text), 'markers'), |
| (r'^([ \t]*)(\.\.\.)(\n)', |
| bygroups(Text, Comment, Text)), # for doctests... |
| (r'^([^:]+)(: )(.+)(\n)', |
| bygroups(Generic.Error, Text, Name, Text), '#pop'), |
| (r'^([a-zA-Z_][\w.]*)(:?\n)', |
| bygroups(Generic.Error, Text), '#pop') |
| ], |
| 'markers': [ |
| # Either `PEP 657 <https://www.python.org/dev/peps/pep-0657/>` |
| # error locations in Python 3.11+, or single-caret markers |
| # for syntax errors before that. |
| (r'^( {4,})([~^]+)(\n)', |
| bygroups(Text, Punctuation.Marker, Text), |
| '#pop'), |
| default('#pop'), |
| ], |
| } |
| |
| |
| Python3TracebackLexer = PythonTracebackLexer |
| |
| |
| class Python2TracebackLexer(RegexLexer): |
| """ |
| For Python tracebacks. |
| |
| .. versionadded:: 0.7 |
| |
| .. versionchanged:: 2.5 |
| This class has been renamed from ``PythonTracebackLexer``. |
| ``PythonTracebackLexer`` now refers to the Python 3 variant. |
| """ |
| |
| name = 'Python 2.x Traceback' |
| aliases = ['py2tb'] |
| filenames = ['*.py2tb'] |
| mimetypes = ['text/x-python2-traceback'] |
| |
| tokens = { |
| 'root': [ |
| # Cover both (most recent call last) and (innermost last) |
| # The optional ^C allows us to catch keyboard interrupt signals. |
| (r'^(\^C)?(Traceback.*\n)', |
| bygroups(Text, Generic.Traceback), 'intb'), |
| # SyntaxError starts with this. |
| (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), |
| (r'^.*\n', Other), |
| ], |
| 'intb': [ |
| (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', |
| bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)), |
| (r'^( File )("[^"]+")(, line )(\d+)(\n)', |
| bygroups(Text, Name.Builtin, Text, Number, Text)), |
| (r'^( )(.+)(\n)', |
| bygroups(Text, using(Python2Lexer), Text), 'marker'), |
| (r'^([ \t]*)(\.\.\.)(\n)', |
| bygroups(Text, Comment, Text)), # for doctests... |
| (r'^([^:]+)(: )(.+)(\n)', |
| bygroups(Generic.Error, Text, Name, Text), '#pop'), |
| (r'^([a-zA-Z_]\w*)(:?\n)', |
| bygroups(Generic.Error, Text), '#pop') |
| ], |
| 'marker': [ |
| # For syntax errors. |
| (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'), |
| default('#pop'), |
| ], |
| } |
| |
| |
| class CythonLexer(RegexLexer): |
| """ |
| For Pyrex and Cython source code. |
| |
| .. versionadded:: 1.1 |
| """ |
| |
| name = 'Cython' |
| url = 'http://cython.org' |
| aliases = ['cython', 'pyx', 'pyrex'] |
| filenames = ['*.pyx', '*.pxd', '*.pxi'] |
| mimetypes = ['text/x-cython', 'application/x-cython'] |
| |
| tokens = { |
| 'root': [ |
| (r'\n', Text), |
| (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Text, String.Doc)), |
| (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Text, String.Doc)), |
| (r'[^\S\n]+', Text), |
| (r'#.*$', Comment), |
| (r'[]{}:(),;[]', Punctuation), |
| (r'\\\n', Text), |
| (r'\\', Text), |
| (r'(in|is|and|or|not)\b', Operator.Word), |
| (r'(<)([a-zA-Z0-9.?]+)(>)', |
| bygroups(Punctuation, Keyword.Type, Punctuation)), |
| (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator), |
| (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)', |
| bygroups(Keyword, Number.Integer, Operator, Name, Operator, |
| Name, Punctuation)), |
| include('keywords'), |
| (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'), |
| (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'), |
| # (should actually start a block with only cdefs) |
| (r'(cdef)(:)', bygroups(Keyword, Punctuation)), |
| (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'), |
| (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'), |
| (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'), |
| include('builtins'), |
| include('backtick'), |
| ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'), |
| ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'), |
| ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'), |
| ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'), |
| ('[uU]?"""', String, combined('stringescape', 'tdqs')), |
| ("[uU]?'''", String, combined('stringescape', 'tsqs')), |
| ('[uU]?"', String, combined('stringescape', 'dqs')), |
| ("[uU]?'", String, combined('stringescape', 'sqs')), |
| include('name'), |
| include('numbers'), |
| ], |
| 'keywords': [ |
| (words(( |
| 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif', |
| 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil', |
| 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print', |
| 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'), |
| Keyword), |
| (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc), |
| ], |
| 'builtins': [ |
| (words(( |
| '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint', |
| 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', |
| 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr', |
| 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit', |
| 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals', |
| 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance', |
| 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max', |
| 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'Py_ssize_t', |
| 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed', |
| 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', |
| 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned', |
| 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'), |
| Name.Builtin), |
| (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL' |
| r')\b', Name.Builtin.Pseudo), |
| (words(( |
| 'ArithmeticError', 'AssertionError', 'AttributeError', |
| 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError', |
| 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', |
| 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', |
| 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', |
| 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError', |
| 'OSError', 'OverflowError', 'OverflowWarning', |
| 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError', |
| 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError', |
| 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', |
| 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', |
| 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', |
| 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', |
| 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'), |
| Name.Exception), |
| ], |
| 'numbers': [ |
| (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), |
| (r'0\d+', Number.Oct), |
| (r'0[xX][a-fA-F0-9]+', Number.Hex), |
| (r'\d+L', Number.Integer.Long), |
| (r'\d+', Number.Integer) |
| ], |
| 'backtick': [ |
| ('`.*?`', String.Backtick), |
| ], |
| 'name': [ |
| (r'@\w+', Name.Decorator), |
| (r'[a-zA-Z_]\w*', Name), |
| ], |
| 'funcname': [ |
| (r'[a-zA-Z_]\w*', Name.Function, '#pop') |
| ], |
| 'cdef': [ |
| (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), |
| (r'(struct|enum|union|class)\b', Keyword), |
| (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)', |
| bygroups(Name.Function, Text), '#pop'), |
| (r'([a-zA-Z_]\w*)(\s*)(,)', |
| bygroups(Name.Function, Text, Punctuation)), |
| (r'from\b', Keyword, '#pop'), |
| (r'as\b', Keyword), |
| (r':', Punctuation, '#pop'), |
| (r'(?=["\'])', Text, '#pop'), |
| (r'[a-zA-Z_]\w*', Keyword.Type), |
| (r'.', Text), |
| ], |
| 'classname': [ |
| (r'[a-zA-Z_]\w*', Name.Class, '#pop') |
| ], |
| 'import': [ |
| (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), |
| (r'[a-zA-Z_][\w.]*', Name.Namespace), |
| (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), |
| default('#pop') # all else: go back |
| ], |
| 'fromimport': [ |
| (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'), |
| (r'[a-zA-Z_.][\w.]*', Name.Namespace), |
| # ``cdef foo from "header"``, or ``for foo from 0 < i < 10`` |
| default('#pop'), |
| ], |
| 'stringescape': [ |
| (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' |
| r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) |
| ], |
| 'strings': [ |
| (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' |
| '[hlL]?[E-GXc-giorsux%]', String.Interpol), |
| (r'[^\\\'"%\n]+', String), |
| # quotes, percents and backslashes must be parsed one at a time |
| (r'[\'"\\]', String), |
| # unhandled string formatting sign |
| (r'%', String) |
| # newlines are an error (use "nl" state) |
| ], |
| 'nl': [ |
| (r'\n', String) |
| ], |
| 'dqs': [ |
| (r'"', String, '#pop'), |
| (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings |
| include('strings') |
| ], |
| 'sqs': [ |
| (r"'", String, '#pop'), |
| (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings |
| include('strings') |
| ], |
| 'tdqs': [ |
| (r'"""', String, '#pop'), |
| include('strings'), |
| include('nl') |
| ], |
| 'tsqs': [ |
| (r"'''", String, '#pop'), |
| include('strings'), |
| include('nl') |
| ], |
| } |
| |
| |
| class DgLexer(RegexLexer): |
| """ |
| Lexer for dg, |
| a functional and object-oriented programming language |
| running on the CPython 3 VM. |
| |
| .. versionadded:: 1.6 |
| """ |
| name = 'dg' |
| aliases = ['dg'] |
| filenames = ['*.dg'] |
| mimetypes = ['text/x-dg'] |
| |
| tokens = { |
| 'root': [ |
| (r'\s+', Text), |
| (r'#.*?$', Comment.Single), |
| |
| (r'(?i)0b[01]+', Number.Bin), |
| (r'(?i)0o[0-7]+', Number.Oct), |
| (r'(?i)0x[0-9a-f]+', Number.Hex), |
| (r'(?i)[+-]?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?j?', Number.Float), |
| (r'(?i)[+-]?[0-9]+e[+-]?\d+j?', Number.Float), |
| (r'(?i)[+-]?[0-9]+j?', Number.Integer), |
| |
| (r"(?i)(br|r?b?)'''", String, combined('stringescape', 'tsqs', 'string')), |
| (r'(?i)(br|r?b?)"""', String, combined('stringescape', 'tdqs', 'string')), |
| (r"(?i)(br|r?b?)'", String, combined('stringescape', 'sqs', 'string')), |
| (r'(?i)(br|r?b?)"', String, combined('stringescape', 'dqs', 'string')), |
| |
| (r"`\w+'*`", Operator), |
| (r'\b(and|in|is|or|where)\b', Operator.Word), |
| (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator), |
| |
| (words(( |
| 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'', |
| 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object', |
| 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str', |
| 'super', 'tuple', 'tuple\'', 'type'), |
| prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), |
| Name.Builtin), |
| (words(( |
| '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile', |
| 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate', |
| 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst', |
| 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init', |
| 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len', |
| 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow', |
| 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd', |
| 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'), |
| prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), |
| Name.Builtin), |
| (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])", |
| Name.Builtin.Pseudo), |
| |
| (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])", |
| Name.Exception), |
| (r"(?<!\.)(Exception|GeneratorExit|KeyboardInterrupt|StopIteration|" |
| r"SystemExit)(?!['\w])", Name.Exception), |
| |
| (r"(?<![\w.])(except|finally|for|if|import|not|otherwise|raise|" |
| r"subclass|while|with|yield)(?!['\w])", Keyword.Reserved), |
| |
| (r"[A-Z_]+'*(?!['\w])", Name), |
| (r"[A-Z]\w+'*(?!['\w])", Keyword.Type), |
| (r"\w+'*", Name), |
| |
| (r'[()]', Punctuation), |
| (r'.', Error), |
| ], |
| 'stringescape': [ |
| (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' |
| r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) |
| ], |
| 'string': [ |
| (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' |
| '[hlL]?[E-GXc-giorsux%]', String.Interpol), |
| (r'[^\\\'"%\n]+', String), |
| # quotes, percents and backslashes must be parsed one at a time |
| (r'[\'"\\]', String), |
| # unhandled string formatting sign |
| (r'%', String), |
| (r'\n', String) |
| ], |
| 'dqs': [ |
| (r'"', String, '#pop') |
| ], |
| 'sqs': [ |
| (r"'", String, '#pop') |
| ], |
| 'tdqs': [ |
| (r'"""', String, '#pop') |
| ], |
| 'tsqs': [ |
| (r"'''", String, '#pop') |
| ], |
| } |
| |
| |
| class NumPyLexer(PythonLexer): |
| """ |
| A Python lexer recognizing Numerical Python builtins. |
| |
| .. versionadded:: 0.10 |
| """ |
| |
| name = 'NumPy' |
| url = 'https://numpy.org/' |
| aliases = ['numpy'] |
| |
| # override the mimetypes to not inherit them from python |
| mimetypes = [] |
| filenames = [] |
| |
| EXTRA_KEYWORDS = { |
| 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose', |
| 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append', |
| 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh', |
| 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin', |
| 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal', |
| 'array_equiv', 'array_repr', 'array_split', 'array_str', 'arrayrange', |
| 'asanyarray', 'asarray', 'asarray_chkfinite', 'ascontiguousarray', |
| 'asfarray', 'asfortranarray', 'asmatrix', 'asscalar', 'astype', |
| 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', 'bartlett', |
| 'base_repr', 'beta', 'binary_repr', 'bincount', 'binomial', |
| 'bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor', 'blackman', |
| 'bmat', 'broadcast', 'byte_bounds', 'bytes', 'byteswap', 'c_', |
| 'can_cast', 'ceil', 'choose', 'clip', 'column_stack', 'common_type', |
| 'compare_chararrays', 'compress', 'concatenate', 'conj', 'conjugate', |
| 'convolve', 'copy', 'corrcoef', 'correlate', 'cos', 'cosh', 'cov', |
| 'cross', 'cumprod', 'cumproduct', 'cumsum', 'delete', 'deprecate', |
| 'diag', 'diagflat', 'diagonal', 'diff', 'digitize', 'disp', 'divide', |
| 'dot', 'dsplit', 'dstack', 'dtype', 'dump', 'dumps', 'ediff1d', 'empty', |
| 'empty_like', 'equal', 'exp', 'expand_dims', 'expm1', 'extract', 'eye', |
| 'fabs', 'fastCopyAndTranspose', 'fft', 'fftfreq', 'fftshift', 'fill', |
| 'finfo', 'fix', 'flat', 'flatnonzero', 'flatten', 'fliplr', 'flipud', |
| 'floor', 'floor_divide', 'fmod', 'frexp', 'fromarrays', 'frombuffer', |
| 'fromfile', 'fromfunction', 'fromiter', 'frompyfunc', 'fromstring', |
| 'generic', 'get_array_wrap', 'get_include', 'get_numarray_include', |
| 'get_numpy_include', 'get_printoptions', 'getbuffer', 'getbufsize', |
| 'geterr', 'geterrcall', 'geterrobj', 'getfield', 'gradient', 'greater', |
| 'greater_equal', 'gumbel', 'hamming', 'hanning', 'histogram', |
| 'histogram2d', 'histogramdd', 'hsplit', 'hstack', 'hypot', 'i0', |
| 'identity', 'ifft', 'imag', 'index_exp', 'indices', 'inf', 'info', |
| 'inner', 'insert', 'int_asbuffer', 'interp', 'intersect1d', |
| 'intersect1d_nu', 'inv', 'invert', 'iscomplex', 'iscomplexobj', |
| 'isfinite', 'isfortran', 'isinf', 'isnan', 'isneginf', 'isposinf', |
| 'isreal', 'isrealobj', 'isscalar', 'issctype', 'issubclass_', |
| 'issubdtype', 'issubsctype', 'item', 'itemset', 'iterable', 'ix_', |
| 'kaiser', 'kron', 'ldexp', 'left_shift', 'less', 'less_equal', 'lexsort', |
| 'linspace', 'load', 'loads', 'loadtxt', 'log', 'log10', 'log1p', 'log2', |
| 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'logspace', |
| 'lstsq', 'mat', 'matrix', 'max', 'maximum', 'maximum_sctype', |
| 'may_share_memory', 'mean', 'median', 'meshgrid', 'mgrid', 'min', |
| 'minimum', 'mintypecode', 'mod', 'modf', 'msort', 'multiply', 'nan', |
| 'nan_to_num', 'nanargmax', 'nanargmin', 'nanmax', 'nanmin', 'nansum', |
| 'ndenumerate', 'ndim', 'ndindex', 'negative', 'newaxis', 'newbuffer', |
| 'newbyteorder', 'nonzero', 'not_equal', 'obj2sctype', 'ogrid', 'ones', |
| 'ones_like', 'outer', 'permutation', 'piecewise', 'pinv', 'pkgload', |
| 'place', 'poisson', 'poly', 'poly1d', 'polyadd', 'polyder', 'polydiv', |
| 'polyfit', 'polyint', 'polymul', 'polysub', 'polyval', 'power', 'prod', |
| 'product', 'ptp', 'put', 'putmask', 'r_', 'randint', 'random_integers', |
| 'random_sample', 'ranf', 'rank', 'ravel', 'real', 'real_if_close', |
| 'recarray', 'reciprocal', 'reduce', 'remainder', 'repeat', 'require', |
| 'reshape', 'resize', 'restoredot', 'right_shift', 'rint', 'roll', |
| 'rollaxis', 'roots', 'rot90', 'round', 'round_', 'row_stack', 's_', |
| 'sample', 'savetxt', 'sctype2char', 'searchsorted', 'seed', 'select', |
| 'set_numeric_ops', 'set_printoptions', 'set_string_function', |
| 'setbufsize', 'setdiff1d', 'seterr', 'seterrcall', 'seterrobj', |
| 'setfield', 'setflags', 'setmember1d', 'setxor1d', 'shape', |
| 'show_config', 'shuffle', 'sign', 'signbit', 'sin', 'sinc', 'sinh', |
| 'size', 'slice', 'solve', 'sometrue', 'sort', 'sort_complex', 'source', |
| 'split', 'sqrt', 'square', 'squeeze', 'standard_normal', 'std', |
| 'subtract', 'sum', 'svd', 'swapaxes', 'take', 'tan', 'tanh', 'tensordot', |
| 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose', |
| 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict', |
| 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index', |
| 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises', |
| 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like' |
| } |
| |
| def get_tokens_unprocessed(self, text): |
| for index, token, value in \ |
| PythonLexer.get_tokens_unprocessed(self, text): |
| if token is Name and value in self.EXTRA_KEYWORDS: |
| yield index, Keyword.Pseudo, value |
| else: |
| yield index, token, value |
| |
| def analyse_text(text): |
| ltext = text[:1000] |
| return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or |
| 'import ' in ltext) \ |
| and ('import numpy' in ltext or 'from numpy import' in ltext) |
