gh-131507: Add support for syntax highlighting in PyREPL

Doc/whatsnew/3.14.rst

@@ -556,6 +556,23 @@ For further information on how to build Python, see
 (Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this
 in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.)
 
+Syntax highlighting in PyREPL
+-----------------------------
+
+The default :term:`interactive` shell now highlights Python syntax as you
+type.  The feature is enabled by default unless the
+:envvar:`PYTHON_BASIC_REPL` environment is set or any color-disabling
+environment variables are used. See :ref:`using-on-controlling-color` for
+details.
+
+The default color theme for syntax highlighting strives for good contrast
+and uses exclusively the 4-bit VGA standard ANSI color codes for maximum
+compatibility. The theme can be customized using an experimental API
+``_colorize.set_theme()``.  This can be called interactively, as well as
+in the :envvar:`PYTHONSTARTUP` script.
+
+(Contributed by Łukasz Langa in :gh:`131507`.)
+
 
 Other language changes
 ======================

Lib/_colorize.py

@@ -7,7 +7,22 @@
 
 # types
 if False:
-    from typing import IO
+    from typing import IO, Literal
+
+    type ColorTag = Literal[
+        "PROMPT",
+        "KEYWORD",
+        "BUILTIN",
+        "COMMENT",
+        "STRING",
+        "NUMBER",
+        "OP",
+        "DEFINITION",
+        "SOFT_KEYWORD",
+        "RESET",
+    ]
+
+    theme: dict[ColorTag, str]
 
 
 class ANSIColors:
@@ -22,6 +37,7 @@ class ANSIColors:
     WHITE = "\x1b[37m"  # more like LIGHT GRAY
     YELLOW = "\x1b[33m"
 
+    BOLD = "\x1b[1m"
     BOLD_BLACK = "\x1b[1;30m"  # DARK GRAY
     BOLD_BLUE = "\x1b[1;34m"
     BOLD_CYAN = "\x1b[1;36m"
@@ -110,3 +126,28 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool:
         return os.isatty(file.fileno())
     except io.UnsupportedOperation:
         return hasattr(file, "isatty") and file.isatty()
+
+
+def set_theme(t: dict[ColorTag, str] | None = None) -> None:
+    global theme
+
+    if t:
+        theme = t
+        return
+
+    colors = get_colors()
+    theme = {
+        "PROMPT": colors.BOLD_MAGENTA,
+        "KEYWORD": colors.BOLD_BLUE,
+        "BUILTIN": colors.CYAN,
+        "COMMENT": colors.RED,
+        "STRING": colors.GREEN,
+        "NUMBER": colors.YELLOW,
+        "OP": colors.RESET,
+        "DEFINITION": colors.BOLD,
+        "SOFT_KEYWORD": colors.BOLD_BLUE,
+        "RESET": colors.RESET,
+    }
+
+
+set_theme()

Lib/_pyrepl/_module_completer.py

@@ -2,6 +2,7 @@
 
 import pkgutil
 import sys
+import token
 import tokenize
 from io import StringIO
 from contextlib import contextmanager
@@ -180,8 +181,8 @@ class ImportParser:
     when parsing multiple statements.
     """
     _ignored_tokens = {
-        tokenize.INDENT, tokenize.DEDENT, tokenize.COMMENT,
-        tokenize.NL, tokenize.NEWLINE, tokenize.ENDMARKER
+        token.INDENT, token.DEDENT, token.COMMENT,
+        token.NL, token.NEWLINE, token.ENDMARKER
     }
     _keywords = {'import', 'from', 'as'}
 
@@ -350,11 +351,11 @@ def peek(self) -> TokenInfo | None:
     def peek_name(self) -> bool:
         if not (tok := self.peek()):
             return False
-        return tok.type == tokenize.NAME
+        return tok.type == token.NAME
 
     def pop_name(self) -> str:
         tok = self.pop()
-        if tok.type != tokenize.NAME:
+        if tok.type != token.NAME:
             raise ParseError('pop_name')
         return tok.string
 

Lib/_pyrepl/commands.py

@@ -21,6 +21,7 @@
 
 from __future__ import annotations
 import os
+import time
 
 # Categories of actions:
 #  killing
@@ -31,6 +32,7 @@
 #  finishing
 # [completion]
 
+from .trace import trace
 
 # types
 if False:
@@ -471,19 +473,24 @@ def do(self) -> None:
 
 
 class paste_mode(Command):
-
     def do(self) -> None:
         self.reader.paste_mode = not self.reader.paste_mode
         self.reader.dirty = True
 
 
-class enable_bracketed_paste(Command):
-    def do(self) -> None:
-        self.reader.paste_mode = True
-        self.reader.in_bracketed_paste = True
-
-class disable_bracketed_paste(Command):
-    def do(self) -> None:
-        self.reader.paste_mode = False
-        self.reader.in_bracketed_paste = False
-        self.reader.dirty = True
+class perform_bracketed_paste(Command):
+    def do(self) -> None:
+        done = "\x1b[201~"
+        data = ""
+        start = time.time()
+        while done not in data:
+            self.reader.console.wait(100)
+            ev = self.reader.console.getpending()
+            data += ev.data
+        trace(
+            "bracketed pasting of {l} chars done in {s:.2f}s",
+            l=len(data),
+            s=time.time() - start,
+        )
+        self.reader.insert(data.replace(done, ""))
+        self.reader.last_refresh_cache.invalidated = True

Lib/_pyrepl/mypy.ini

@@ -23,7 +23,3 @@ check_untyped_defs = False
 # Various internal modules that typeshed deliberately doesn't have stubs for:
 [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
 ignore_missing_imports = True
-
-# Other untyped parts of the stdlib
-[mypy-idlelib.*]
-ignore_missing_imports = True

Lib/_pyrepl/reader.py

@@ -22,14 +22,13 @@
 from __future__ import annotations
 
 import sys
+import _colorize
 
 from contextlib import contextmanager
 from dataclasses import dataclass, field, fields
-from _colorize import can_colorize, ANSIColors
-
 
 from . import commands, console, input
-from .utils import wlen, unbracket, disp_str
+from .utils import wlen, unbracket, disp_str, gen_colors
 from .trace import trace
 
 
@@ -38,8 +37,7 @@
 from .types import Callback, SimpleContextManager, KeySpec, CommandName
 
 
-# syntax classes:
-
+# syntax classes
 SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3)
 
 
@@ -105,8 +103,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]:
         (r"\M-9", "digit-arg"),
         (r"\M-\n", "accept"),
         ("\\\\", "self-insert"),
-        (r"\x1b[200~", "enable_bracketed_paste"),
-        (r"\x1b[201~", "disable_bracketed_paste"),
+        (r"\x1b[200~", "perform-bracketed-paste"),
         (r"\x03", "ctrl-c"),
     ]
     + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"]
@@ -144,16 +141,17 @@ class Reader:
     Instance variables of note include:
 
       * buffer:
-        A *list* (*not* a string at the moment :-) containing all the
-        characters that have been entered.
+        A per-character list containing all the characters that have been
+        entered. Does not include color information.
       * console:
         Hopefully encapsulates the OS dependent stuff.
       * pos:
         A 0-based index into 'buffer' for where the insertion point
         is.
       * screeninfo:
-        Ahem.  This list contains some info needed to move the
-        insertion point around reasonably efficiently.
+        A list of screen position tuples. Each list element is a tuple
+        representing information on visible line length for a given line.
+        Allows for efficient skipping of color escape sequences.
       * cxy, lxy:
         the position of the insertion point in screen ...
       * syntax_table:
@@ -203,7 +201,6 @@ class Reader:
     dirty: bool = False
     finished: bool = False
     paste_mode: bool = False
-    in_bracketed_paste: bool = False
     commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
     last_command: type[Command] | None = None
     syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
@@ -221,7 +218,6 @@ class Reader:
     ## cached metadata to speed up screen refreshes
     @dataclass
     class RefreshCache:
-        in_bracketed_paste: bool = False
         screen: list[str] = field(default_factory=list)
         screeninfo: list[tuple[int, list[int]]] = field(init=False)
         line_end_offsets: list[int] = field(default_factory=list)
@@ -235,7 +231,6 @@ def update_cache(self,
                          screen: list[str],
                          screeninfo: list[tuple[int, list[int]]],
             ) -> None:
-            self.in_bracketed_paste = reader.in_bracketed_paste
             self.screen = screen.copy()
             self.screeninfo = screeninfo.copy()
             self.pos = reader.pos
@@ -248,8 +243,7 @@ def valid(self, reader: Reader) -> bool:
                 return False
             dimensions = reader.console.width, reader.console.height
             dimensions_changed = dimensions != self.dimensions
-            paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste
-            return not (dimensions_changed or paste_changed)
+            return not dimensions_changed
 
         def get_cached_location(self, reader: Reader) -> tuple[int, int]:
             if self.invalidated:
@@ -279,7 +273,7 @@ def __post_init__(self) -> None:
         self.screeninfo = [(0, [])]
         self.cxy = self.pos2xy()
         self.lxy = (self.pos, 0)
-        self.can_colorize = can_colorize()
+        self.can_colorize = _colorize.can_colorize()
 
         self.last_refresh_cache.screeninfo = self.screeninfo
         self.last_refresh_cache.pos = self.pos
@@ -316,6 +310,12 @@ def calc_screen(self) -> list[str]:
         pos -= offset
 
         prompt_from_cache = (offset and self.buffer[offset - 1] != "\n")
+
+        if self.can_colorize:
+            colors = list(gen_colors(self.get_unicode()))
+        else:
+            colors = None
+        trace("colors = {colors}", colors=colors)
         lines = "".join(self.buffer[offset:]).split("\n")
         cursor_found = False
         lines_beyond_cursor = 0
@@ -343,9 +343,8 @@ def calc_screen(self) -> list[str]:
                 screeninfo.append((0, []))
             pos -= line_len + 1
             prompt, prompt_len = self.process_prompt(prompt)
-            chars, char_widths = disp_str(line)
+            chars, char_widths = disp_str(line, colors, offset)
             wrapcount = (sum(char_widths) + prompt_len) // self.console.width
-            trace("wrapcount = {wrapcount}", wrapcount=wrapcount)
             if wrapcount == 0 or not char_widths:
                 offset += line_len + 1  # Takes all of the line plus the newline
                 last_refresh_line_end_offsets.append(offset)
@@ -479,7 +478,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
         'lineno'."""
         if self.arg is not None and cursor_on_line:
             prompt = f"(arg: {self.arg}) "
-        elif self.paste_mode and not self.in_bracketed_paste:
+        elif self.paste_mode:
             prompt = "(paste) "
         elif "\n" in self.buffer:
             if lineno == 0:
@@ -492,7 +491,11 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
             prompt = self.ps1
 
         if self.can_colorize:
-            prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}"
+            prompt = (
+                f"{_colorize.theme["PROMPT"]}"
+                f"{prompt}"
+                f"{_colorize.theme["RESET"]}"
+            )
         return prompt
 
     def push_input_trans(self, itrans: input.KeymapTranslator) -> None:
@@ -567,6 +570,7 @@ def insert(self, text: str | list[str]) -> None:
     def update_cursor(self) -> None:
         """Move the cursor to reflect changes in self.pos"""
         self.cxy = self.pos2xy()
+        trace("update_cursor({pos}) = {cxy}", pos=self.pos, cxy=self.cxy)
         self.console.move_cursor(*self.cxy)
 
     def after_command(self, cmd: Command) -> None:
@@ -633,9 +637,6 @@ def update_screen(self) -> None:
 
     def refresh(self) -> None:
         """Recalculate and refresh the screen."""
-        if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n":
-            return
-
         # this call sets up self.cxy, so call it first.
         self.screen = self.calc_screen()
         self.console.refresh(self.screen, self.cxy)

Lib/_pyrepl/readline.py

@@ -276,10 +276,6 @@ def do(self) -> None:
         r = self.reader  # type: ignore[assignment]
         r.dirty = True  # this is needed to hide the completion menu, if visible
 
-        if self.reader.in_bracketed_paste:
-            r.insert("\n")
-            return
-
         # if there are already several lines and the cursor
         # is not on the last one, always insert a new \n.
         text = r.get_unicode()

Lib/_pyrepl/simple_interact.py

@@ -157,7 +157,6 @@ def maybe_run_command(statement: str) -> bool:
             r.pos = len(r.get_unicode())
             r.dirty = True
             r.refresh()
-            r.in_bracketed_paste = False
             console.write("\nKeyboardInterrupt\n")
             console.resetbuffer()
         except MemoryError: