gh-132449: Improve syntax error messages for keywords with typos

Signed-off-by: Pablo Galindo <[email protected]>

Author: pablogsal

Grammar/python.gram

@@ -87,16 +87,18 @@ _PyPegen_parse(Parser *p)
 # ==============
 
 file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) }
-interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(a, p->arena) }
+interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(_PyPegen_register_stmts(p, a), p->arena) }
 eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { _PyAST_Expression(a, p->arena) }
 func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { _PyAST_FunctionType(a, b, p->arena) }
 
 # GENERAL STATEMENTS
 # ==================
 
-statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
+statements[asdl_stmt_seq*]: a=statement+ { _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a)) }
 
-statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | a[asdl_stmt_seq*]=simple_stmts { a }
+statement[asdl_stmt_seq*]: 
+    | a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } 
+    | a[asdl_stmt_seq*]=simple_stmts { a }
 
 statement_newline[asdl_stmt_seq*]:
     | a=compound_stmt NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) }

Include/cpython/pyerrors.h

@@ -30,6 +30,7 @@ typedef struct {
     PyObject *end_offset;
     PyObject *text;
     PyObject *print_file_and_line;
+    PyObject *metadata;
 } PySyntaxErrorObject;
 
 typedef struct {

Lib/codeop.py

@@ -47,7 +47,7 @@
 PyCF_ONLY_AST = 0x400
 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
 
-def _maybe_compile(compiler, source, filename, symbol):
+def _maybe_compile(compiler, source, filename, symbol, flags):
     # Check for source consisting of only blank lines and comments.
     for line in source.split("\n"):
         line = line.strip()
@@ -61,10 +61,10 @@ def _maybe_compile(compiler, source, filename, symbol):
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
         try:
-            compiler(source, filename, symbol)
+            compiler(source, filename, symbol, flags=flags)
         except SyntaxError:  # Let other compile() errors propagate.
             try:
-                compiler(source + "\n", filename, symbol)
+                compiler(source + "\n", filename, symbol, flags=flags)
                 return None
             except _IncompleteInputError as e:
                 return None
@@ -74,14 +74,13 @@ def _maybe_compile(compiler, source, filename, symbol):
 
     return compiler(source, filename, symbol, incomplete_input=False)
 
-def _compile(source, filename, symbol, incomplete_input=True):
-    flags = 0
+def _compile(source, filename, symbol, incomplete_input=True, *, flags=0):
     if incomplete_input:
         flags |= PyCF_ALLOW_INCOMPLETE_INPUT
         flags |= PyCF_DONT_IMPLY_DEDENT
     return compile(source, filename, symbol, flags)
 
-def compile_command(source, filename="<input>", symbol="single"):
+def compile_command(source, filename="<input>", symbol="single", flags=0):
     r"""Compile a command and determine whether it is incomplete.
 
     Arguments:
@@ -100,7 +99,7 @@ def compile_command(source, filename="<input>", symbol="single"):
       syntax error (OverflowError and ValueError can be produced by
       malformed literals).
     """
-    return _maybe_compile(_compile, source, filename, symbol)
+    return _maybe_compile(_compile, source, filename, symbol, flags)
 
 class Compile:
     """Instances of this class behave much like the built-in compile
@@ -152,4 +151,4 @@ def __call__(self, source, filename="<input>", symbol="single"):
           syntax error (OverflowError and ValueError can be produced by
           malformed literals).
         """
-        return _maybe_compile(self.compiler, source, filename, symbol)
+        return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags)

Lib/test/test_exceptions.py

@@ -2462,7 +2462,7 @@ def test_incorrect_constructor(self):
         args = ("bad.py", 1, 2)
         self.assertRaises(TypeError, SyntaxError, "bad bad", args)
 
-        args = ("bad.py", 1, 2, 4, 5, 6, 7)
+        args = ("bad.py", 1, 2, 4, 5, 6, 7, 8)
         self.assertRaises(TypeError, SyntaxError, "bad bad", args)
 
         args = ("bad.py", 1, 2, "abcdefg", 1)

Lib/test/test_syntax.py

@@ -1189,7 +1189,7 @@
    >>> with block ad something:
    ...   pass
    Traceback (most recent call last):
-   SyntaxError: invalid syntax
+   SyntaxError: invalid syntax. Did you mean 'and'?
 
    >>> try
    ...   pass
@@ -1713,6 +1713,74 @@
    Traceback (most recent call last):
    SyntaxError: expected one or more exception types
 
+Check custom exceptions for keywords with typos
+
+>>> fur a in b:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'for'?
+
+>>> whille True:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'while'?
+
+>>> iff x > 5:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'if'?
+
+>>> classe MyClass:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'class'?
+
+
+>>> impor math
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'import'?
+
+>>> defn calculate_sum(a, b):
+...   return a + b
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'def'?
+
+>>> def foo():
+...   returm result
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'return'?
+
+>>> lamda x: x ** 2
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'lambda'?
+
+>>> def foo():
+...   yeld i
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'yield'?
+
+>>> def foo():
+...   globel counter
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'global'?
+
+>>> frum math import sqrt
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'from'?
+
+>>> asynch def fetch_data():
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'async'?
+
+>>> async def foo():
+...   awaid fetch_data()
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'await'?
+
+>>> raisee ValueError("Error")
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'raise'?
 
 >>> f(a=23, a=234)
 Traceback (most recent call last):

Lib/traceback.py

@@ -6,6 +6,10 @@
 import sys
 import textwrap
 import warnings
+import codeop
+import keyword
+import tokenize
+import io
 from contextlib import suppress
 import _colorize
 from _colorize import ANSIColors
@@ -1090,6 +1094,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None,
             self.end_offset = exc_value.end_offset
             self.msg = exc_value.msg
             self._is_syntax_error = True
+            self._exc_metadata = getattr(exc_value, "_metadata", None)
         elif exc_type and issubclass(exc_type, ImportError) and \
                 getattr(exc_value, "name_from", None) is not None:
             wrong_name = getattr(exc_value, "name_from", None)
@@ -1272,6 +1277,86 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs):
         if self.exceptions and show_group:
             for ex in self.exceptions:
                 yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize)
+    
+    def _find_keyword_typos(self):
+        try:
+            import _suggestions
+        except ImportError:
+            return
+        
+        assert self._is_syntax_error
+
+        # Only try to find keyword typos if there is no custom message
+        if self.msg != "invalid syntax":
+            return
+
+        if not self._exc_metadata:
+            return
+        
+        line, offset, source = self._exc_metadata
+        end_line = int(self.lineno) if self.lineno is not None else 0
+        lines = None
+        from_filename = False
+
+        if source is None:
+            if self.filename:
+                try:
+                    with open(self.filename) as f:
+                        lines = f.readlines()
+                except Exception:
+                    line, end_line, offset = 0,1,0
+                else:
+                    from_filename = True
+            lines = lines if lines is not None else self.text.splitlines()
+        else:
+            lines = source.splitlines()
+        
+        error_code = lines[line -1 if line > 0 else 0:end_line]
+        error_code[0] = error_code[0][offset:]
+        error_code = textwrap.dedent(''.join(error_code))
+
+        # Do not continue if the source is too large
+        if len(error_code) > 1024:
+            return
+
+        tokens = tokenize.generate_tokens(io.StringIO(error_code).readline)
+        tokens_left_to_process = 10
+        for token in tokens:
+            tokens_left_to_process -= 1
+            if tokens_left_to_process < 0:
+                break
+            start, end = token.start, token.end
+            if token.type != tokenize.NAME:
+                continue
+            if from_filename and token.start[0]+line != end_line+1:
+                continue
+            wrong_name = token.string
+            if wrong_name in keyword.kwlist:
+                continue
+            suggestion = _suggestions._generate_suggestions(keyword.kwlist, wrong_name)
+            if not suggestion or suggestion == wrong_name:
+                continue
+            # Try to replace the token with the keyword
+            the_lines = error_code.splitlines()
+            the_line = the_lines[start[0] - 1]
+            chars = list(the_line)
+            chars[token.start[1]:token.end[1]] = suggestion
+            the_lines[start[0] - 1] = ''.join(chars)
+            code = ''.join(the_lines)
+            # Check if it works
+            try:
+                codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST)
+            except SyntaxError as e:
+                continue
+            # Keep token.line but handle offsets correctly
+            self.text = token.line
+            self.offset = token.start[1] + 1
+            self.end_offset = token.end[1] + 1
+            self.lineno = start[0]
+            self.end_lineno = end[0]
+            self.msg = f"invalid syntax. Did you mean '{suggestion}'?"
+            return
+        
 
     def _format_syntax_error(self, stype, **kwargs):
         """Format SyntaxError exceptions (internal helper)."""
@@ -1299,6 +1384,9 @@ def _format_syntax_error(self, stype, **kwargs):
             # text  = "   foo\n"
             # rtext = "   foo"
             # ltext =    "foo"
+            with suppress(Exception):
+                self._find_keyword_typos()
+            text = self.text
             rtext = text.rstrip('\n')
             ltext = rtext.lstrip(' \n\f')
             spaces = len(rtext) - len(ltext)

Objects/exceptions.c

@@ -2668,10 +2668,10 @@ SyntaxError_init(PyObject *op, PyObject *args, PyObject *kwds)
 
         self->end_lineno = NULL;
         self->end_offset = NULL;
-        if (!PyArg_ParseTuple(info, "OOOO|OO",
+        if (!PyArg_ParseTuple(info, "OOOO|OOO",
                               &self->filename, &self->lineno,
                               &self->offset, &self->text,
-                              &self->end_lineno, &self->end_offset)) {
+                              &self->end_lineno, &self->end_offset, &self->metadata)) {
             Py_DECREF(info);
             return -1;
         }
@@ -2682,6 +2682,7 @@ SyntaxError_init(PyObject *op, PyObject *args, PyObject *kwds)
         Py_INCREF(self->text);
         Py_XINCREF(self->end_lineno);
         Py_XINCREF(self->end_offset);
+        Py_XINCREF(self->metadata);
         Py_DECREF(info);
 
         if (self->end_lineno != NULL && self->end_offset == NULL) {
@@ -2704,6 +2705,7 @@ SyntaxError_clear(PyObject *op)
     Py_CLEAR(self->end_offset);
     Py_CLEAR(self->text);
     Py_CLEAR(self->print_file_and_line);
+    Py_CLEAR(self->metadata);
     return BaseException_clear(op);
 }
 
@@ -2727,6 +2729,7 @@ SyntaxError_traverse(PyObject *op, visitproc visit, void *arg)
     Py_VISIT(self->end_offset);
     Py_VISIT(self->text);
     Py_VISIT(self->print_file_and_line);
+    Py_VISIT(self->metadata);
     return BaseException_traverse(op, visit, arg);
 }
 
@@ -2822,6 +2825,8 @@ static PyMemberDef SyntaxError_members[] = {
     {"print_file_and_line", _Py_T_OBJECT,
         offsetof(PySyntaxErrorObject, print_file_and_line), 0,
         PyDoc_STR("exception print_file_and_line")},
+    {"_metadata", _Py_T_OBJECT, offsetof(PySyntaxErrorObject, metadata), 0,
+                   PyDoc_STR("exception private metadata")},
     {NULL}  /* Sentinel */
 };
 

Parser/action_helpers.c

@@ -1711,3 +1711,26 @@ _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * na
     }
     return _PyAST_ImportFrom(module, names, level, lineno, col_offset, end_lineno, end_col_offset, arena);
 }
+
+stmt_ty
+_PyPegen_register_stmt(Parser *p, stmt_ty stmt) {
+    p->last_stmt_location.lineno = stmt->lineno;
+    p->last_stmt_location.col_offset = stmt->col_offset;
+    p->last_stmt_location.end_lineno = stmt->end_lineno;
+    p->last_stmt_location.end_col_offset = stmt->end_col_offset;
+    return stmt;
+}
+
+asdl_stmt_seq*
+_PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) {
+    Py_ssize_t len = asdl_seq_LEN(stmts);
+    if (len == 0) {
+        return stmts;
+    }
+    stmt_ty last_stmt = asdl_seq_GET(stmts, len - 1);
+    p->last_stmt_location.lineno = last_stmt->lineno;
+    p->last_stmt_location.col_offset = last_stmt->col_offset;
+    p->last_stmt_location.end_lineno = last_stmt->end_lineno;
+    p->last_stmt_location.end_col_offset = last_stmt->end_col_offset;
+    return stmts;
+}