# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 # For details: https://github.com/coveragepy/coveragepy/blob/main/NOTICE.txt """Better tokenizing for coverage.py.""" # pragma: enforce 100% coverage from __future__ import annotations import ast import io import keyword import re import sys import token import tokenize from collections.abc import Iterable from coverage import env from coverage.types import TLineNo, TSourceTokenLines # The same tuple that tokenize.tokenize returns: TToken = tuple[int, str, tuple[int, int], tuple[int, int], str] TokenInfos = Iterable[TToken] def _phys_tokens(toks: TokenInfos) -> TokenInfos: """Return all physical tokens, even line continuations. tokenize.generate_tokens() doesn't return a token for the backslash that continues lines. This wrapper provides those tokens so that we can re-create a faithful representation of the original source. Returns the same values as generate_tokens() """ last_line: str | None = None last_lineno = -1 last_ttext: str = "" for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks: if last_lineno != elineno: if last_line and last_line.endswith("\\\n"): # We are at the beginning of a new line, and the last line # ended with a backslash. We probably have to inject a # backslash token into the stream. Unfortunately, there's more # to figure out. This code:: # # usage = """\ # HEY THERE # """ # # triggers this condition, but the token text is:: # # '"""\\\nHEY THERE\n"""' # # so we need to figure out if the backslash is already in the # string token or not. inject_backslash = True if last_ttext.endswith("\\"): inject_backslash = False elif ttype == token.STRING: if ( # pylint: disable=simplifiable-if-statement last_line.endswith("\\\n") and last_line.rstrip(" \\\n").endswith(last_ttext) ): # Deal with special cases like such code:: # # a = ["aaa",\ # there may be zero or more blanks between "," and "\". # "bbb \ # ccc"] # inject_backslash = True else: # It's a multi-line string and the first line ends with # a backslash, so we don't need to inject another. inject_backslash = False elif env.PYBEHAVIOR.fstring_syntax and ttype == token.FSTRING_MIDDLE: inject_backslash = False if inject_backslash: # Figure out what column the backslash is in. ccol = len(last_line.split("\n")[-2]) - 1 # Yield the token, with a fake token type. yield (99999, "\\\n", (slineno, ccol), (slineno, ccol + 2), last_line) last_line = ltext if ttype not in (tokenize.NEWLINE, tokenize.NL): last_ttext = ttext yield (ttype, ttext, (slineno, scol), (elineno, ecol), ltext) last_lineno = elineno def find_soft_key_lines(source: str) -> set[TLineNo]: """Helper for finding lines with soft keywords, like match/case lines.""" # Do a quick check first, to eliminate files with no possibility of soft keywords. # match/case is only a soft keyword if both words are in the source. if "match" not in source or "case" not in source: if sys.version_info < (3, 12) or "type" not in source: if sys.version_info < (3, 15) or "lazy" not in source: return set() soft_key_lines: set[TLineNo] = set() for node in ast.walk(ast.parse(source)): # PYVERSION: we use sys.version_info here so that mypy will be ok with # us accessing attributes that appeared in those versions. if isinstance(node, ast.Match): soft_key_lines.add(node.lineno) for case in node.cases: soft_key_lines.add(case.pattern.lineno) elif sys.version_info >= (3, 12) and isinstance(node, ast.TypeAlias): soft_key_lines.add(node.lineno) elif sys.version_info >= (3, 15) and isinstance(node, (ast.Import, ast.ImportFrom)): if node.is_lazy: soft_key_lines.add(node.lineno) return soft_key_lines def source_token_lines(source: str) -> TSourceTokenLines: """Generate a series of lines, one for each line in `source`. Each line is a list of pairs, each pair is a token:: [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ] Each pair has a token class, and the token text. If you concatenate all the token texts, and then join them with newlines, you should have your original `source` back, with two differences: trailing white space is not preserved, and a final line with no newline is indistinguishable from a final line with a newline. """ ws_tokens = {token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL} line: list[tuple[str, str]] = [] col = 0 source = source.expandtabs(8).replace("\r\n", "\n") tokgen = generate_tokens(source) soft_key_lines = find_soft_key_lines(source) for ttype, ttext, (sline, scol), (_, ecol), _ in _phys_tokens(tokgen): mark_start = True if "\n" in ttext: parts = re.split("(\n)", ttext) else: parts = [ttext] for part in parts: if part == "\n": yield line line = [] col = 0 mark_end = False elif part == "": mark_end = False elif ttype in ws_tokens: mark_end = False else: if env.PYBEHAVIOR.fstring_syntax and ttype == token.FSTRING_MIDDLE: part = part.replace("{", "{{").replace("}", "}}") ecol = scol + len(part) if mark_start and scol > col: line.append(("ws", " " * (scol - col))) mark_start = False tok_class = tokenize.tok_name.get(ttype, "xx").lower()[:3] if ttype == token.NAME: if keyword.iskeyword(ttext): # Hard keywords are always keywords. tok_class = "key" elif keyword.issoftkeyword(ttext): # Soft keywords appear at the start of their line. if len(line) == 0: is_start_of_line = True elif (len(line) == 1) and line[0][0] == "ws": is_start_of_line = True else: is_start_of_line = False if is_start_of_line and sline in soft_key_lines: tok_class = "key" line.append((tok_class, part)) mark_end = True scol = 0 if mark_end: col = ecol if line: yield line def generate_tokens(text: str) -> TokenInfos: """A helper around `tokenize.generate_tokens`. Originally this was used to cache the results, but it didn't seem to make reporting go faster, and caused issues with using too much memory. """ readline = io.StringIO(text).readline return tokenize.generate_tokens(readline) def source_encoding(source: bytes) -> str: """Determine the encoding for `source`, according to PEP 263. `source` is a byte string: the text of the program. Returns a string, the name of the encoding. """ readline = iter(source.splitlines(True)).__next__ return tokenize.detect_encoding(readline)[0]