gh-107450: Check for overflow in the tokenizer and fix overflow test

Author: lysnikolaou

Include/errcode.h

@@ -19,24 +19,25 @@
 extern "C" {
 #endif
 
-#define E_OK            10      /* No error */
-#define E_EOF           11      /* End Of File */
-#define E_INTR          12      /* Interrupted */
-#define E_TOKEN         13      /* Bad token */
-#define E_SYNTAX        14      /* Syntax error */
-#define E_NOMEM         15      /* Ran out of memory */
-#define E_DONE          16      /* Parsing complete */
-#define E_ERROR         17      /* Execution error */
-#define E_TABSPACE      18      /* Inconsistent mixing of tabs and spaces */
-#define E_OVERFLOW      19      /* Node had too many children */
-#define E_TOODEEP       20      /* Too many indentation levels */
-#define E_DEDENT        21      /* No matching outer block for dedent */
-#define E_DECODE        22      /* Error in decoding into Unicode */
-#define E_EOFS          23      /* EOF in triple-quoted string */
-#define E_EOLS          24      /* EOL in single-quoted string */
-#define E_LINECONT      25      /* Unexpected characters after a line continuation */
-#define E_BADSINGLE     27      /* Ill-formed single statement input */
-#define E_INTERACT_STOP 28      /* Interactive mode stopped tokenization */
+#define E_OK             10      /* No error */
+#define E_EOF            11      /* End Of File */
+#define E_INTR           12      /* Interrupted */
+#define E_TOKEN          13      /* Bad token */
+#define E_SYNTAX         14      /* Syntax error */
+#define E_NOMEM          15      /* Ran out of memory */
+#define E_DONE           16      /* Parsing complete */
+#define E_ERROR          17      /* Execution error */
+#define E_TABSPACE       18      /* Inconsistent mixing of tabs and spaces */
+#define E_OVERFLOW       19      /* Node had too many children */
+#define E_TOODEEP        20      /* Too many indentation levels */
+#define E_DEDENT         21      /* No matching outer block for dedent */
+#define E_DECODE         22      /* Error in decoding into Unicode */
+#define E_EOFS           23      /* EOF in triple-quoted string */
+#define E_EOLS           24      /* EOL in single-quoted string */
+#define E_LINECONT       25      /* Unexpected characters after a line continuation */
+#define E_BADSINGLE      27      /* Ill-formed single statement input */
+#define E_INTERACT_STOP  28      /* Interactive mode stopped tokenization */
+#define E_COLUMNOVERFLOW 29      /* Column offset overflow */
 
 #ifdef __cplusplus
 }

Lib/test/test_exceptions.py

@@ -1,6 +1,7 @@
 # Python test set -- part 5, built-in exceptions
 
 import copy
+import ctypes
 import os
 import sys
 import unittest
@@ -318,11 +319,22 @@ def baz():
         check('(yield i) = 2', 1, 2)
         check('def f(*):\n  pass', 1, 7)
 
+    @unittest.skipIf(ctypes.sizeof(ctypes.c_int) >= ctypes.sizeof(ctypes.c_ssize_t),
+                     "Downcasting to int is safe for col_offset")
     @support.requires_resource('cpu')
-    @support.bigmemtest(support._2G, memuse=1.5)
-    def testMemoryErrorBigSource(self, _size):
-        with self.assertRaises(OverflowError):
-            exec(f"if True:\n {' ' * 2**31}print('hello world')")
+    @support.bigmemtest(2**(ctypes.sizeof(ctypes.c_int)*8-1)-1-len("pass"), memuse=1)
+    def testMemoryErrorBigSource(self, size):
+        if size < 2**(ctypes.sizeof(ctypes.c_int)*8-1)-1-len("pass"):
+            self.skipTest('Not enough memory for overflow to occur')
+
+        # Construct buffer to hold just enough characters so that the tokenizer offset overflows.
+        # This makes sure that we don't overflow in the string creation itself
+        distance_to_prev_divisible_by_8 = size & 7
+        padding = ' ' * distance_to_prev_divisible_by_8
+        padding += '        ' * ((size - distance_to_prev_divisible_by_8) // 8)
+
+        with self.assertRaisesRegex(OverflowError, "Parser column offset overflow"):
+            exec(f"if True:\n{padding}pass")
 
     @cpython_only
     def testSettingException(self):

Parser/lexer/lexer.c

@@ -59,6 +59,10 @@ tok_nextc(struct tok_state *tok)
     int rc;
     for (;;) {
         if (tok->cur != tok->inp) {
+            if (INT_MAX - tok->col_offset - 1 < 0) {
+                tok->done = E_COLUMNOVERFLOW;
+                return EOF;
+            }
             tok->col_offset++;
             return Py_CHARMASK(*tok->cur++); /* Fast path */
         }

Parser/pegen_errors.c

@@ -68,6 +68,7 @@ _Pypegen_tokenizer_error(Parser *p)
     const char *msg = NULL;
     PyObject* errtype = PyExc_SyntaxError;
     Py_ssize_t col_offset = -1;
+    p->error_indicator = 1;
     switch (p->tok->done) {
         case E_TOKEN:
             msg = "invalid token";
@@ -103,6 +104,10 @@ _Pypegen_tokenizer_error(Parser *p)
             msg = "unexpected character after line continuation character";
             break;
         }
+        case E_COLUMNOVERFLOW:
+            PyErr_SetString(PyExc_OverflowError,
+                    "Parser column offset overflow - source line is too big");
+            return -1;
         default:
             msg = "unknown parsing error";
     }