pythongh-98744: Prevent column-level decoding crashes when using the Python traceback

Author: isidentical

Lib/test/test_traceback.py

@@ -804,6 +804,36 @@ def f():
         ]
         self.assertEqual(actual, expected)
 
+    def test_wide_characters_unicode_with_problematic_byte_offset(self):
+        def f():
+            ｗｉｄｔｈ
+
+        actual = self.get_exception(f)
+        expected = [
+            f"Traceback (most recent call last):",
+            f"  File \"{__file__}\", line {self.callable_line}, in get_exception",
+            f"    callable()",
+            f"  File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f",
+            f"    ｗｉｄｔｈ",
+        ]
+        self.assertEqual(actual, expected)
+
+
+    def test_byte_offset_with_wide_characters_middle(self):
+        def f():
+            ｗｉｄｔｈ = 1
+            raise ValueError(ｗｉｄｔｈ)
+
+        actual = self.get_exception(f)
+        expected = [
+            f"Traceback (most recent call last):",
+            f"  File \"{__file__}\", line {self.callable_line}, in get_exception",
+            f"    callable()",
+            f"  File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f",
+            f"    raise ValueError(ｗｉｄｔｈ)",
+        ]
+        self.assertEqual(actual, expected)
+
 
 @requires_debug_ranges()
 class PurePythonTracebackErrorCaretTests(

Lib/traceback.py

@@ -485,16 +485,16 @@ def format_frame_summary(self, frame_summary):
                 if frame_summary.lineno == frame_summary.end_lineno:
                     with suppress(Exception):
                         anchors = _extract_caret_anchors_from_line_segment(
-                            frame_summary._original_line[colno - 1:end_colno - 1]
+                            frame_summary._original_line[colno:end_colno]
                         )
                 else:
-                    end_colno = stripped_characters + len(stripped_line)
+                    end_colno = stripped_characters + len(stripped_line) - 1
 
                 # show indicators if primary char doesn't span the frame line
                 if end_colno - colno < len(stripped_line) or (
                         anchors and anchors.right_start_offset - anchors.left_end_offset > 0):
                     row.append('    ')
-                    row.append(' ' * (colno - stripped_characters))
+                    row.append(' ' * (colno - stripped_characters + 1))
 
                     if anchors:
                         row.append(anchors.primary_char * (anchors.left_end_offset))
@@ -561,10 +561,10 @@ def format(self):
 
 def _byte_offset_to_character_offset(str, offset):
     as_utf8 = str.encode('utf-8')
-    if offset > len(as_utf8):
-        offset = len(as_utf8)
+    if offset > len(as_utf8) + 1:
+        offset = len(as_utf8) + 1
 
-    return len(as_utf8[:offset + 1].decode("utf-8"))
+    return len(as_utf8[:offset].decode("utf-8", errors="replace"))
 
 
 _Anchors = collections.namedtuple(

Misc/NEWS.d/next/Library/2022-10-28-23-44-17.gh-issue-98744.sGHDWm.rst

@@ -0,0 +1,2 @@
+Prevent crashing in :mod:`traceback` when retrieving the byte-offset for
+some source files that contain certain unicode characters.