Raise a warning when encoding is omitted

The warning is raised only in dev mode.

pathlib uses the new `io.text_encoding` helper function.
Other libraries will follow.

Author: methane

Lib/_pyio.py

@@ -40,6 +40,28 @@
 _CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE
 
 
+def text_encoding(encoding, stacklevel=1):
+    """
+    Helper function to choose the text encoding.
+
+    When encoding is not None, just return it.
+    Otherwise, return the default text encoding ("locale" for now)
+    and raise a DeprecationWarning in dev mode.
+
+    This function can be used in APIs having encoding=None option.
+    But please consider encoding="utf-8" for new APIs.
+    """
+    if encoding is None:
+        if sys.flags.dev_mode:
+            import warnings
+            warnings.warn(
+                    "'encoding' option is not specified. The default encoding "
+                    "will be changed to 'utf-8' in the future",
+                    DeprecationWarning, stacklevel + 2)
+        encoding = "locale"
+    return encoding
+
+
 def open(file, mode="r", buffering=-1, encoding=None, errors=None,
          newline=None, closefd=True, opener=None):
 
@@ -248,6 +270,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
         result = buffer
         if binary:
             return result
+        encoding = text_encoding(encoding)
         text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
         result = text
         text.mode = mode
@@ -2004,19 +2027,22 @@ class TextIOWrapper(TextIOBase):
     def __init__(self, buffer, encoding=None, errors=None, newline=None,
                  line_buffering=False, write_through=False):
         self._check_newline(newline)
-        if encoding is None:
+        encoding = text_encoding(encoding)
+
+        if encoding == "locale":
             try:
-                encoding = os.device_encoding(buffer.fileno())
+                encoding = os.device_encoding(buffer.fileno()) or "locale"
             except (AttributeError, UnsupportedOperation):
                 pass
-            if encoding is None:
-                try:
-                    import locale
-                except ImportError:
-                    # Importing locale may fail if Python is being built
-                    encoding = "ascii"
-                else:
-                    encoding = locale.getpreferredencoding(False)
+
+        if encoding == "locale":
+            try:
+                import locale
+            except ImportError:
+                # Importing locale may fail if Python is being built
+                encoding = "utf-8"
+            else:
+                encoding = locale.getpreferredencoding(False)
 
         if not isinstance(encoding, str):
             raise ValueError("invalid encoding: %r" % encoding)

Lib/io.py

@@ -54,7 +54,7 @@
 from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
                  open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
-                 IncrementalNewlineDecoder, TextIOWrapper)
+                 IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
 
 OpenWrapper = _io.open # for compatibility with _pyio
 

Lib/pathlib.py

@@ -1241,6 +1241,8 @@ def open(self, mode='r', buffering=-1, encoding=None,
         Open the file pointed by this path and return a file object, as
         the built-in open() function does.
         """
+        if "b" not in mode:
+            encoding = io.text_encoding(encoding)
         return io.open(self, mode, buffering, encoding, errors, newline,
                        opener=self._opener)
 
@@ -1255,6 +1257,7 @@ def read_text(self, encoding=None, errors=None):
         """
         Open the file in text mode, read it, and close the file.
         """
+        encoding = io.text_encoding(encoding)
         with self.open(mode='r', encoding=encoding, errors=errors) as f:
             return f.read()
 
@@ -1274,6 +1277,7 @@ def write_text(self, data, encoding=None, errors=None, newline=None):
         if not isinstance(data, str):
             raise TypeError('data must be str, not %s' %
                             data.__class__.__name__)
+        encoding = io.text_encoding(encoding)
         with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f:
             return f.write(data)
 

Modules/_io/_iomodule.c

@@ -33,6 +33,7 @@ PyObject *_PyIO_str_fileno = NULL;
 PyObject *_PyIO_str_flush = NULL;
 PyObject *_PyIO_str_getstate = NULL;
 PyObject *_PyIO_str_isatty = NULL;
+PyObject *_PyIO_str_locale = NULL;
 PyObject *_PyIO_str_newlines = NULL;
 PyObject *_PyIO_str_nl = NULL;
 PyObject *_PyIO_str_peek = NULL;
@@ -504,6 +505,43 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
     return NULL;
 }
 
+
+/*[clinic input]
+_io.text_encoding
+    encoding: object = NULL
+    stacklevel: int = 1
+    /
+
+Helper function to choose the text encoding.
+
+When encoding is not None, just return it.
+Otherwise, return the default text encoding ("locale" for now)
+and raise a DeprecationWarning in dev mode.
+
+This function can be used in APIs having encoding=None option.
+But please consider encoding="utf-8" for new APIs.
+[clinic start generated code]*/
+
+static PyObject *
+_io_text_encoding_impl(PyObject *module, PyObject *encoding, int stacklevel)
+/*[clinic end generated code: output=91b2cfea6934cc0c input=46b896c6a7111a95]*/
+{
+    if (encoding == NULL || encoding == Py_None) {
+        PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE();
+        if (interp->config.dev_mode) {
+            PyErr_WarnEx(PyExc_DeprecationWarning,
+                "'encoding' option is not specified. The default encoding "
+                "will be changed to 'utf-8' in the future",
+                stacklevel + 1);
+        }
+        Py_INCREF(_PyIO_str_locale);
+        return _PyIO_str_locale;
+    }
+    Py_INCREF(encoding);
+    return encoding;
+}
+
+
 /*[clinic input]
 _io.open_code
 
@@ -629,6 +667,7 @@ iomodule_free(PyObject *mod) {
 
 static PyMethodDef module_methods[] = {
     _IO_OPEN_METHODDEF
+    _IO_TEXT_ENCODING_METHODDEF
     _IO_OPEN_CODE_METHODDEF
     {NULL, NULL}
 };
@@ -747,6 +786,7 @@ PyInit__io(void)
     ADD_INTERNED(flush)
     ADD_INTERNED(getstate)
     ADD_INTERNED(isatty)
+    ADD_INTERNED(locale)
     ADD_INTERNED(newlines)
     ADD_INTERNED(peek)
     ADD_INTERNED(read)

Modules/_io/clinic/_iomodule.c.h

@@ -272,6 +272,55 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw
     return return_value;
 }
 
+PyDoc_STRVAR(_io_text_encoding__doc__,
+"text_encoding($module, encoding=<unrepresentable>, stacklevel=1, /)\n"
+"--\n"
+"\n"
+"Select text encoding for TextIOWrapper.\n"
+"\n"
+"Returns the default text encoding for TextIOWrapper when encoding is None.\n"
+"The default text encoding is \"locale\" for now, but it will be changed\n"
+"to \"utf-8\" in the future.");
+
+#define _IO_TEXT_ENCODING_METHODDEF    \
+    {"text_encoding", (PyCFunction)(void(*)(void))_io_text_encoding, METH_FASTCALL, _io_text_encoding__doc__},
+
+static PyObject *
+_io_text_encoding_impl(PyObject *module, PyObject *encoding, int stacklevel);
+
+static PyObject *
+_io_text_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+{
+    PyObject *return_value = NULL;
+    PyObject *encoding = NULL;
+    int stacklevel = 1;
+
+    if (!_PyArg_CheckPositional("text_encoding", nargs, 0, 2)) {
+        goto exit;
+    }
+    if (nargs < 1) {
+        goto skip_optional;
+    }
+    encoding = args[0];
+    if (nargs < 2) {
+        goto skip_optional;
+    }
+    if (PyFloat_Check(args[1])) {
+        PyErr_SetString(PyExc_TypeError,
+                        "integer argument expected, got float" );
+        goto exit;
+    }
+    stacklevel = _PyLong_AsInt(args[1]);
+    if (stacklevel == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+skip_optional:
+    return_value = _io_text_encoding_impl(module, encoding, stacklevel);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(_io_open_code__doc__,
 "open_code($module, /, path)\n"
 "--\n"

Modules/_io/textio.c

@@ -1123,6 +1123,18 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
     self->encodefunc = NULL;
     self->b2cratio = 0.0;
 
+    if (encoding == NULL) {
+        PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE();
+        if (interp->config.dev_mode) {
+            PyErr_WarnEx(PyExc_DeprecationWarning,
+                "'encoding' option is not specified. The default encoding "
+                "will be changed to 'utf-8' in the future", 1);
+        }
+    }
+    else if (strcmp(encoding, "locale") == 0) {
+        encoding = NULL;
+    }
+
     if (encoding == NULL) {
         /* Try os.device_encoding(fileno) */
         PyObject *fileno;