From 755b3852ae9fc22368283a670ce6a4d3107caf43 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Fri, 31 Jan 2025 19:01:46 -0800 Subject: [PATCH] gh-129005: Fix buffer expansion in _pyio.FileIO.readall Move to a linear slice append with an iterator which has a length hint. This is more expensive then PyByteArray_Resize, but I think as efficient as can get without a new bytearray Python API to resize. The previous code didn't append as I had intended: ```python a = bytearray() >>> a[0:5] = b'\0' >>> a bytearray(b'\x00') >>> a[5:16] = b'\01' >>> a bytearray(b'\x00\x01') >>> len(a) 2 ``` --- Lib/_pyio.py | 4 +++- .../Library/2025-01-31-19-07-31.gh-issue-129005.II6go0.rst | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-31-19-07-31.gh-issue-129005.II6go0.rst diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 76a27910da4d5f..0be61ff8a959f8 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -10,6 +10,7 @@ import sys # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock +from itertools import repeat if sys.platform in {'win32', 'cygwin'}: from msvcrt import setmode as _setmode else: @@ -1686,7 +1687,8 @@ def readall(self): if addend < DEFAULT_BUFFER_SIZE: addend = DEFAULT_BUFFER_SIZE bufsize += addend - result[bytes_read:bufsize] = b'\0' + result.extend(repeat(0, addend)) + assert len(result) == bufsize, "Should have expanded in size" assert bufsize - bytes_read > 0, "Should always try and read at least one byte" try: n = os.readinto(self._fd, memoryview(result)[bytes_read:]) diff --git a/Misc/NEWS.d/next/Library/2025-01-31-19-07-31.gh-issue-129005.II6go0.rst b/Misc/NEWS.d/next/Library/2025-01-31-19-07-31.gh-issue-129005.II6go0.rst new file mode 100644 index 00000000000000..260a8ba63a45d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-31-19-07-31.gh-issue-129005.II6go0.rst @@ -0,0 +1 @@ +:mod:`!pyio`: Fix expansion of buffer in _pyio.FileIO.readall