From 04812d4750da7c7b52072f6cdf64e14c4e0717f5 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 30 Mar 2025 17:00:28 +0500 Subject: [PATCH 01/12] Fix input of long unicode characters - with two codepoints or more --- Lib/_pyrepl/base_eventqueue.py | 23 +++++--- Lib/_pyrepl/simple_interact.py | 5 ++ Lib/_pyrepl/windows_console.py | 3 +- Lib/test/test_pyrepl/test_eventqueue.py | 71 +++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index e018c4fc18308e..2aa3492ce37d8c 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -73,14 +73,19 @@ def push(self, char: int | bytes | str) -> None: """ Processes a character by updating the buffer and handling special key mappings. """ - ord_char = char if isinstance(char, int) else ord(char) - if ord_char > 255: - assert isinstance(char, str) - char = bytes(char.encode(self.encoding, "replace")) + + if isinstance(char, bytes): self.buf.extend(char) else: - char = bytes(bytearray((ord_char,))) - self.buf.append(ord_char) + + ord_char = char if isinstance(char, int) else ord(char) + if ord_char > 255: + assert isinstance(char, str) + char = bytes(char.encode(self.encoding, "replace")) + self.buf.extend(char) + else: + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) if char in self.keymap: if self.keymap is self.compiled_keymap: @@ -108,7 +113,9 @@ def push(self, char: int | bytes | str) -> None: try: decoded = bytes(self.buf).decode(self.encoding) except UnicodeError: - return + self.flush_buf() + raise else: self.insert(Event('key', decoded, self.flush_buf())) - self.keymap = self.compiled_keymap + finally: + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index a08546a9319824..a033cb2d651c4a 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -158,3 +158,8 @@ def maybe_run_command(statement: str) -> bool: except MemoryError: console.write("\nMemoryError\n") console.resetbuffer() + # except SystemExit: + # break + # except: + # console.showtraceback() + # console.resetbuffer() diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 47fd3fd8f8909b..065d1168130187 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -468,7 +468,8 @@ def get_event(self, block: bool = True) -> Event | None: return None elif self.__vt_support: # If virtual terminal is enabled, scanning VT sequences - self.event_queue.push(rec.Event.KeyEvent.uChar.UnicodeChar) + key_bytes = raw_key.encode(self.event_queue.encoding) + self.event_queue.push(key_bytes) continue if key_event.dwControlKeyState & ALT_ACTIVE: diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index afb557103424a6..b08d9b3ccc7822 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -129,6 +129,77 @@ def test_push_unicode_character(self): self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "ч") + def test_push_unicode_character_as_bytes(self): + eq = self.make_eventqueue() + eq.keymap = {} + + eq.push("ч".encode(eq.encoding, "replace")) + e = eq.get() + self.assertEqual(e.evt, "key") + self.assertEqual(e.data, "ч") + + def test_push_long_unicode_character_as_bytes(self): + eq = self.make_eventqueue() + eq.keymap = {} + + def _event(evt, data, raw=None): + r = raw if raw is not None else data.encode(eq.encoding) + e = Event(evt, data, r) + return e + + def _push(keys): + for k in keys: + eq.push(k) + + _push("\x1b[200") + eq.push("ñ".encode(eq.encoding, "replace")) + _push("\x1b[201") + + self.assertEqual(eq.get(), _event("key", "\x1b")) + self.assertEqual(eq.get(), _event("key", "[")) + self.assertEqual(eq.get(), _event("key", "2")) + self.assertEqual(eq.get(), _event("key", "0")) + self.assertEqual(eq.get(), _event("key", "0")) + + self.assertEqual(eq.get(), _event("key", "ñ", bytearray(b'\xc3\xb1'))) + + self.assertEqual(eq.get(), _event("key", "\x1b")) + self.assertEqual(eq.get(), _event("key", "[")) + self.assertEqual(eq.get(), _event("key", "2")) + self.assertEqual(eq.get(), _event("key", "0")) + self.assertEqual(eq.get(), _event("key", "1")) + + def test_push_long_unicode_character(self): + eq = self.make_eventqueue() + eq.keymap = {} + + def _event(evt, data, raw=None): + r = raw if raw is not None else data.encode(eq.encoding) + e = Event(evt, data, r) + return e + + def _push(keys): + for k in keys: + eq.push(k) + + _push("\x1b[200") + msg = "'utf-8' codec can't decode byte 0xf1 in position 0: unexpected end of data" + with self.assertRaisesRegex(UnicodeDecodeError, msg): + eq.push("ñ") + _push("\x1b[201") + + self.assertEqual(eq.get(), _event("key", "\x1b")) + self.assertEqual(eq.get(), _event("key", "[")) + self.assertEqual(eq.get(), _event("key", "2")) + self.assertEqual(eq.get(), _event("key", "0")) + self.assertEqual(eq.get(), _event("key", "0")) + + self.assertEqual(eq.get(), _event("key", "\x1b")) + self.assertEqual(eq.get(), _event("key", "[")) + self.assertEqual(eq.get(), _event("key", "2")) + self.assertEqual(eq.get(), _event("key", "0")) + self.assertEqual(eq.get(), _event("key", "1")) + @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): From 9203df4a8e0cc0045df73c0971003090e7a2f0ba Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 30 Mar 2025 19:49:23 +0500 Subject: [PATCH 02/12] Add new entry --- .../2025-03-30-19-49-00.gh-issue-131878.J8_cHB.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-03-30-19-49-00.gh-issue-131878.J8_cHB.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-30-19-49-00.gh-issue-131878.J8_cHB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-30-19-49-00.gh-issue-131878.J8_cHB.rst new file mode 100644 index 00000000000000..b1223dac52decc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-30-19-49-00.gh-issue-131878.J8_cHB.rst @@ -0,0 +1,2 @@ +Fix support of unicode characters with two or more codepoints on Windows in +the new REPL. From 9d7e3e51ad021038d47637ab7f93392c5924decc Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 30 Mar 2025 19:49:55 +0500 Subject: [PATCH 03/12] Do not change simple_interact --- Lib/_pyrepl/simple_interact.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index a033cb2d651c4a..a08546a9319824 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -158,8 +158,3 @@ def maybe_run_command(statement: str) -> bool: except MemoryError: console.write("\nMemoryError\n") console.resetbuffer() - # except SystemExit: - # break - # except: - # console.showtraceback() - # console.resetbuffer() From 7fd3418834e3afedb2afde7bf1e3b3205139e767 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sat, 26 Apr 2025 21:31:21 +0500 Subject: [PATCH 04/12] Do not use bytearray to convert char to bytes Co-authored-by: Tomas R. --- Lib/_pyrepl/base_eventqueue.py | 2 +- Lib/test/test_pyrepl/test_eventqueue.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index 2aa3492ce37d8c..c53c50755a0139 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -84,7 +84,7 @@ def push(self, char: int | bytes | str) -> None: char = bytes(char.encode(self.encoding, "replace")) self.buf.extend(char) else: - char = bytes(bytearray((ord_char,))) + char = bytes([ord_char]) self.buf.append(ord_char) if char in self.keymap: diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index b08d9b3ccc7822..92080697ef578a 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -161,7 +161,7 @@ def _push(keys): self.assertEqual(eq.get(), _event("key", "0")) self.assertEqual(eq.get(), _event("key", "0")) - self.assertEqual(eq.get(), _event("key", "ñ", bytearray(b'\xc3\xb1'))) + self.assertEqual(eq.get(), _event("key", "ñ", b'\xc3\xb1')) self.assertEqual(eq.get(), _event("key", "\x1b")) self.assertEqual(eq.get(), _event("key", "[")) From b1fa88dd1df5cb95679ede29a24e15737095b0ae Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 27 Apr 2025 21:52:01 +0500 Subject: [PATCH 05/12] Push char one by one in windows_console Co-authored-by: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- Lib/_pyrepl/windows_console.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 065d1168130187..14159083835c5a 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -468,8 +468,8 @@ def get_event(self, block: bool = True) -> Event | None: return None elif self.__vt_support: # If virtual terminal is enabled, scanning VT sequences - key_bytes = raw_key.encode(self.event_queue.encoding) - self.event_queue.push(key_bytes) + for char in raw_key.encode(self.event_queue.encoding, "replace"): + self.event_queue.push(char) continue if key_event.dwControlKeyState & ALT_ACTIVE: From 925f3a27e16f5a3d20787b6e401fcfcfeaff1ac6 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 27 Apr 2025 22:11:54 +0500 Subject: [PATCH 06/12] =?UTF-8?q?Apply=20Chris=20Eibl=20suggestion=D1=8B?= =?UTF-8?q?=20and=20use=20only=20int=20and=20bytes=20in=20eventqueue.push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/_pyrepl/base_eventqueue.py | 22 +++----- Lib/test/test_pyrepl/test_eventqueue.py | 67 ++++++++++++++----------- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index c53c50755a0139..0701c77044a852 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -69,23 +69,14 @@ def insert(self, event: Event) -> None: trace('added event {event}', event=event) self.events.append(event) - def push(self, char: int | bytes | str) -> None: + def push(self, char: int | bytes) -> None: """ Processes a character by updating the buffer and handling special key mappings. """ - - if isinstance(char, bytes): - self.buf.extend(char) - else: - - ord_char = char if isinstance(char, int) else ord(char) - if ord_char > 255: - assert isinstance(char, str) - char = bytes(char.encode(self.encoding, "replace")) - self.buf.extend(char) - else: - char = bytes([ord_char]) - self.buf.append(ord_char) + assert isinstance(char, (int, bytes)) + ord_char = char if isinstance(char, int) else ord(char) + char = bytes((ord_char,)) + self.buf.append(ord_char) if char in self.keymap: if self.keymap is self.compiled_keymap: @@ -113,8 +104,7 @@ def push(self, char: int | bytes | str) -> None: try: decoded = bytes(self.buf).decode(self.encoding) except UnicodeError: - self.flush_buf() - raise + return else: self.insert(Event('key', decoded, self.flush_buf())) finally: diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index 92080697ef578a..7030e0df2b4e2b 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -53,7 +53,7 @@ def test_push_with_key_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"a": "b"} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "b") @@ -63,7 +63,7 @@ def test_push_without_key_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"c": "d"} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "a") @@ -73,13 +73,13 @@ def test_push_with_keymap_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"a": {b"b": "c"}} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertTrue(eq.empty()) - eq.push("b") + eq.push(b"b") self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "c") - eq.push("d") + eq.push(b"d") self.assertEqual(eq.events[1].evt, "key") self.assertEqual(eq.events[1].data, "d") @@ -88,32 +88,32 @@ def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"a": {b"b": "c"}} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertTrue(eq.empty()) eq.flush_buf() - eq.push("\033") + eq.push(b"\033") self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "\033") - eq.push("b") + eq.push(b"b") self.assertEqual(eq.events[1].evt, "key") self.assertEqual(eq.events[1].data, "b") def test_push_special_key(self): eq = self.make_eventqueue() eq.keymap = {} - eq.push("\x1b") - eq.push("[") - eq.push("A") + eq.push(b"\x1b") + eq.push(b"[") + eq.push(b"A") self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "\x1b") def test_push_unrecognized_escape_sequence(self): eq = self.make_eventqueue() eq.keymap = {} - eq.push("\x1b") - eq.push("[") - eq.push("Z") + eq.push(b"\x1b") + eq.push(b"[") + eq.push(b"Z") self.assertEqual(len(eq.events), 3) self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "\x1b") @@ -122,23 +122,31 @@ def test_push_unrecognized_escape_sequence(self): self.assertEqual(eq.events[2].evt, "key") self.assertEqual(eq.events[2].data, "Z") - def test_push_unicode_character(self): + def test_push_unicode_character_as_str(self): eq = self.make_eventqueue() eq.keymap = {} - eq.push("ч") - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "ч") + with self.assertRaises(AssertionError): + eq.push("ч") + with self.assertRaises(AssertionError): + eq.push("ñ") def test_push_unicode_character_as_bytes(self): eq = self.make_eventqueue() eq.keymap = {} - eq.push("ч".encode(eq.encoding, "replace")) + encoded = "ч".encode(eq.encoding, "replace") + self.assertEqual(len(encoded), 2) + + eq.push(encoded[0]) + e = eq.get() + self.assertIsNone(e) + + eq.push(encoded[1]) e = eq.get() self.assertEqual(e.evt, "key") self.assertEqual(e.data, "ч") - def test_push_long_unicode_character_as_bytes(self): + def test_push_unicode_character_as_bytes_in_paste_mode(self): eq = self.make_eventqueue() eq.keymap = {} @@ -151,9 +159,9 @@ def _push(keys): for k in keys: eq.push(k) - _push("\x1b[200") - eq.push("ñ".encode(eq.encoding, "replace")) - _push("\x1b[201") + _push(b"\x1b[200") + _push("ñ".encode(eq.encoding, "replace")) + _push(b"\x1b[201") self.assertEqual(eq.get(), _event("key", "\x1b")) self.assertEqual(eq.get(), _event("key", "[")) @@ -169,7 +177,7 @@ def _push(keys): self.assertEqual(eq.get(), _event("key", "0")) self.assertEqual(eq.get(), _event("key", "1")) - def test_push_long_unicode_character(self): + def test_push_unicode_character_as_str_in_paste_mode(self): eq = self.make_eventqueue() eq.keymap = {} @@ -182,11 +190,12 @@ def _push(keys): for k in keys: eq.push(k) - _push("\x1b[200") - msg = "'utf-8' codec can't decode byte 0xf1 in position 0: unexpected end of data" - with self.assertRaisesRegex(UnicodeDecodeError, msg): - eq.push("ñ") - _push("\x1b[201") + self.assertIsInstance("ñ", str) + + _push(b"\x1b[200") + with self.assertRaises(AssertionError): + _push("ñ") + _push(b"\x1b[201") self.assertEqual(eq.get(), _event("key", "\x1b")) self.assertEqual(eq.get(), _event("key", "[")) From aa56581a6af4e31ba952731e4b5aedd90ebf529d Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 27 Apr 2025 22:24:46 +0500 Subject: [PATCH 07/12] Do not use finally to reset keymap --- Lib/_pyrepl/base_eventqueue.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index 0701c77044a852..c13d072e2b7587 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -107,5 +107,4 @@ def push(self, char: int | bytes) -> None: return else: self.insert(Event('key', decoded, self.flush_buf())) - finally: - self.keymap = self.compiled_keymap + self.keymap = self.compiled_keymap From 1eaeca36879ed28d97d932bf6d787ce557a1e8f0 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 27 Apr 2025 22:26:29 +0500 Subject: [PATCH 08/12] Rename tests along Chris Eibl suggestions --- Lib/test/test_pyrepl/test_eventqueue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index 7030e0df2b4e2b..df2cf11a0b90b9 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -130,7 +130,7 @@ def test_push_unicode_character_as_str(self): with self.assertRaises(AssertionError): eq.push("ñ") - def test_push_unicode_character_as_bytes(self): + def test_push_unicode_character_two_bytes(self): eq = self.make_eventqueue() eq.keymap = {} @@ -146,7 +146,7 @@ def test_push_unicode_character_as_bytes(self): self.assertEqual(e.evt, "key") self.assertEqual(e.data, "ч") - def test_push_unicode_character_as_bytes_in_paste_mode(self): + def test_push_unicode_character_two_bytes_in_paste_mode(self): eq = self.make_eventqueue() eq.keymap = {} From 68d978e0f8ad966e76ead4858fd0568c4e4042fc Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 28 Apr 2025 23:03:43 +0500 Subject: [PATCH 09/12] Fix tests --- Lib/test/test_pyrepl/test_eventqueue.py | 52 +++---------------------- 1 file changed, 6 insertions(+), 46 deletions(-) diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index df2cf11a0b90b9..d498771c22ec61 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -146,38 +146,7 @@ def test_push_unicode_character_two_bytes(self): self.assertEqual(e.evt, "key") self.assertEqual(e.data, "ч") - def test_push_unicode_character_two_bytes_in_paste_mode(self): - eq = self.make_eventqueue() - eq.keymap = {} - - def _event(evt, data, raw=None): - r = raw if raw is not None else data.encode(eq.encoding) - e = Event(evt, data, r) - return e - - def _push(keys): - for k in keys: - eq.push(k) - - _push(b"\x1b[200") - _push("ñ".encode(eq.encoding, "replace")) - _push(b"\x1b[201") - - self.assertEqual(eq.get(), _event("key", "\x1b")) - self.assertEqual(eq.get(), _event("key", "[")) - self.assertEqual(eq.get(), _event("key", "2")) - self.assertEqual(eq.get(), _event("key", "0")) - self.assertEqual(eq.get(), _event("key", "0")) - - self.assertEqual(eq.get(), _event("key", "ñ", b'\xc3\xb1')) - - self.assertEqual(eq.get(), _event("key", "\x1b")) - self.assertEqual(eq.get(), _event("key", "[")) - self.assertEqual(eq.get(), _event("key", "2")) - self.assertEqual(eq.get(), _event("key", "0")) - self.assertEqual(eq.get(), _event("key", "1")) - - def test_push_unicode_character_as_str_in_paste_mode(self): + def test_push_single_chars_and_unicode_character_as_str(self): eq = self.make_eventqueue() eq.keymap = {} @@ -192,22 +161,13 @@ def _push(keys): self.assertIsInstance("ñ", str) - _push(b"\x1b[200") + _push(b"b") with self.assertRaises(AssertionError): _push("ñ") - _push(b"\x1b[201") - - self.assertEqual(eq.get(), _event("key", "\x1b")) - self.assertEqual(eq.get(), _event("key", "[")) - self.assertEqual(eq.get(), _event("key", "2")) - self.assertEqual(eq.get(), _event("key", "0")) - self.assertEqual(eq.get(), _event("key", "0")) - - self.assertEqual(eq.get(), _event("key", "\x1b")) - self.assertEqual(eq.get(), _event("key", "[")) - self.assertEqual(eq.get(), _event("key", "2")) - self.assertEqual(eq.get(), _event("key", "0")) - self.assertEqual(eq.get(), _event("key", "1")) + _push(b"a") + + self.assertEqual(eq.get(), _event("key", "b")) + self.assertEqual(eq.get(), _event("key", "a")) @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") From 7f8fb7f909c25b2b8e188b3c02afff0275977f76 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 28 Apr 2025 23:05:52 +0500 Subject: [PATCH 10/12] Add comment --- Lib/test/test_pyrepl/test_eventqueue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index d498771c22ec61..2bcb100a938bea 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -161,6 +161,8 @@ def _push(keys): self.assertIsInstance("ñ", str) + # we expect that no part of the multibyte string will remain in the + # internal buffer after an encode/decode error _push(b"b") with self.assertRaises(AssertionError): _push("ñ") From 95192084f40449cfdf44d89502936bba53d25578 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 29 Apr 2025 10:19:27 +0500 Subject: [PATCH 11/12] Update comments in the test Co-authored-by: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- Lib/test/test_pyrepl/test_eventqueue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index 2bcb100a938bea..edfe6ac4748f33 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -161,8 +161,8 @@ def _push(keys): self.assertIsInstance("ñ", str) - # we expect that no part of the multibyte string will remain in the - # internal buffer after an encode/decode error + # If an exception happens during push, the existing events must be + # preserved and we can continue to push. _push(b"b") with self.assertRaises(AssertionError): _push("ñ") From a855ba80617d823eccb80e1afd80583e54cbdd9d Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 5 May 2025 20:58:12 +0500 Subject: [PATCH 12/12] Use to_bytes Co-authored-by: Tomas R. --- Lib/_pyrepl/base_eventqueue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index c13d072e2b7587..842599bd1877fb 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -75,7 +75,7 @@ def push(self, char: int | bytes) -> None: """ assert isinstance(char, (int, bytes)) ord_char = char if isinstance(char, int) else ord(char) - char = bytes((ord_char,)) + char = ord_char.to_bytes() self.buf.append(ord_char) if char in self.keymap: