pythongh-117151: IO performance improvement, increase io.DEFAULT_BUFFER_SIZE to 128k, adjust open() to use max(st_blksize, io.DEFAULT_BUFFER_SIZE)

Author: rmmancom

Lib/_pyio.py

@@ -23,8 +23,8 @@
     valid_seek_flags.add(os.SEEK_HOLE)
     valid_seek_flags.add(os.SEEK_DATA)
 
-# open() uses st_blksize whenever we can
-DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
+# open() uses max(st_blksize, io.DEFAULT_BUFFER_SIZE) when st_blksize is available
+DEFAULT_BUFFER_SIZE = 128 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
 # defined in io.py. We don't use real inheritance though, because we don't want
@@ -123,10 +123,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     the size of a fixed-size chunk buffer.  When no buffering argument is
     given, the default buffering policy works as follows:
 
-    * Binary files are buffered in fixed-size chunks; the size of the buffer
-      is chosen using a heuristic trying to determine the underlying device's
-      "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
-      On many systems, the buffer will typically be 4096 or 8192 bytes long.
+   * Binary files are buffered in fixed-size chunks; the size of the buffer
+     is set to `max(io.DEFAULT_BUFFER_SIZE, st_blksize)` using a heuristic
+     trying to determine the underlying device's "block size" when available
+     and falling back on `io.DEFAULT_BUFFER_SIZE`.
+     On most systems, the buffer will typically be 131072 bytes long.
 
     * "Interactive" text files (files for which isatty() returns True)
       use line buffering.  Other text files use the policy described above
@@ -242,7 +243,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
             buffering = -1
             line_buffering = True
         if buffering < 0:
-            buffering = raw._blksize
+            buffering = max(raw._blksize, DEFAULT_BUFFER_SIZE)
         if buffering < 0:
             raise ValueError("invalid buffering size")
         if buffering == 0:

Lib/test/test_file.py

@@ -216,6 +216,18 @@ def testSetBufferSize(self):
         with self.assertWarnsRegex(RuntimeWarning, 'line buffering'):
             self._checkBufferSize(1)
 
+    def testDefaultBufferSize(self):
+        f = self.open(TESTFN, 'wb')
+        blksize = f.raw._blksize
+        f.write(bytes([0] * 5_000_000))
+        f.close()
+
+        f = self.open(TESTFN, 'rb')
+        data = f.read1()
+        expected_size = max(blksize, io.DEFAULT_BUFFER_SIZE)
+        self.assertEqual(len(data), expected_size)
+        f.close()
+
     def testTruncateOnWindows(self):
         # SF bug <https://bugs.python.org/issue801631>
         # "file.truncate fault on windows"

Misc/NEWS.d/next/Library/2024-04-30-14-03-09.gh-issue-117151.yt2H8c.rst

@@ -0,0 +1,5 @@
+Increase ``io.DEFAULT_BUFFER_SIZE`` from 8k to 128k and adjust :func:`open` on
+platforms where ``fstat`` provides a ``st_blksize`` field (such as Linux) to use
+``max(io.DEFAULT_BUFFER_SIZE, device block size)`` rather than always using the
+device block size. This should improve I/O performance.
+Patch by Romain Morotti.

Modules/_io/_iomodule.c

@@ -132,9 +132,10 @@ the size of a fixed-size chunk buffer.  When no buffering argument is
 given, the default buffering policy works as follows:
 
 * Binary files are buffered in fixed-size chunks; the size of the buffer
-  is chosen using a heuristic trying to determine the underlying device's
-  "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
-  On many systems, the buffer will typically be 4096 or 8192 bytes long.
+  is set to `max(io.DEFAULT_BUFFER_SIZE, st_blksize)` using a heuristic
+  trying to determine the underlying device's "block size" when available
+  and falling back on `io.DEFAULT_BUFFER_SIZE`.
+  On most systems, the buffer will typically be 131072 bytes long.
 
 * "Interactive" text files (files for which isatty() returns True)
   use line buffering.  Other text files use the policy described above
@@ -200,7 +201,7 @@ static PyObject *
 _io_open_impl(PyObject *module, PyObject *file, const char *mode,
               int buffering, const char *encoding, const char *errors,
               const char *newline, int closefd, PyObject *opener)
-/*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/
+/*[clinic end generated code: output=aefafc4ce2b46dc0 input=bac1cd70f431fe9a]*/
 {
     size_t i;
 
@@ -368,6 +369,8 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
         if (blksize_obj == NULL)
             goto error;
         buffering = PyLong_AsLong(blksize_obj);
+        if (buffering < DEFAULT_BUFFER_SIZE)
+            buffering = DEFAULT_BUFFER_SIZE;
         Py_DECREF(blksize_obj);
         if (buffering == -1 && PyErr_Occurred())
             goto error;

Modules/_io/_iomodule.h

@@ -78,7 +78,7 @@ extern Py_ssize_t _PyIO_find_line_ending(
 */
 extern int _PyIO_trap_eintr(void);
 
-#define DEFAULT_BUFFER_SIZE (8 * 1024)  /* bytes */
+#define DEFAULT_BUFFER_SIZE (128 * 1024)  /* bytes */
 
 /*
  * Offset type for positioning.