bpo-13601: always use line-buffering for sys.stderr (GH-17646)

Author: jendrikseipp

Doc/library/sys.rst

@@ -1446,9 +1446,15 @@ always available.
      for the Windows console, this only applies when
      :envvar:`PYTHONLEGACYWINDOWSSTDIO` is also set.
 
-   * When interactive, ``stdout`` and ``stderr`` streams are line-buffered.
-     Otherwise, they are block-buffered like regular text files.  You can
-     override this value with the :option:`-u` command-line option.
+   * When interactive, the ``stdout`` stream is line-buffered. Otherwise,
+     it is block-buffered like regular text files.  The ``stderr`` stream
+     is line-buffered in both cases.  You can make both streams unbuffered
+     by passing the :option:`-u` command-line option or setting the
+     :envvar:`PYTHONUNBUFFERED` environment variable.
+
+   .. versionchanged:: 3.9
+      Non-interactive ``stderr`` is now line-buffered instead of fully
+      buffered.
 
    .. note::
 

Lib/test/test_cmd_line.py

@@ -6,6 +6,7 @@
 import subprocess
 import sys
 import tempfile
+import textwrap
 import unittest
 from test import support
 from test.support.script_helper import (
@@ -219,6 +220,21 @@ def check_output(text):
         )
         check_output(text)
 
+    def test_non_interactive_output_buffering(self):
+        code = textwrap.dedent("""
+            import sys
+            out = sys.stdout
+            print(out.isatty(), out.write_through, out.line_buffering)
+            err = sys.stderr
+            print(err.isatty(), err.write_through, err.line_buffering)
+        """)
+        args = [sys.executable, '-c', code]
+        proc = subprocess.run(args, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE, text=True, check=True)
+        self.assertEqual(proc.stdout,
+                         'False False False\n'
+                         'False False True\n')
+
     def test_unbuffered_output(self):
         # Test expected operation of the '-u' switch
         for stream in ('stdout', 'stderr'):

Misc/ACKS

@@ -1510,6 +1510,7 @@ Steven Scott
 Nick Seidenman
 Michael Seifert
 Žiga Seilnacht
+Jendrik Seipp
 Michael Selik
 Yury Selivanov
 Fred Sells

Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst

@@ -0,0 +1,6 @@
+By default, ``sys.stderr`` is line-buffered now, even if ``stderr`` is
+redirected to a file. You can still make ``sys.stderr`` unbuffered by
+passing the :option:`-u` command-line option or setting the
+:envvar:`PYTHONUNBUFFERED` environment variable.
+
+(Contributed by Jendrik Seipp in bpo-13601.)

Python/pylifecycle.c

@@ -1817,7 +1817,7 @@ create_stdio(const PyConfig *config, PyObject* io,
         write_through = Py_True;
     else
         write_through = Py_False;
-    if (isatty && buffered_stdio)
+    if (buffered_stdio && (isatty || fd == fileno(stderr)))
         line_buffering = Py_True;
     else
         line_buffering = Py_False;