centralize encoding_for_open

Author: Avasam

pkg_resources/__init__.py

@@ -27,6 +27,8 @@
 import time
 import re
 import types
+import locale
+import sys
 from typing import Protocol
 import zipfile
 import zipimport
@@ -48,7 +50,6 @@
 import importlib
 import importlib.machinery
 from pkgutil import get_importer
-
 import _imp
 
 # capture these to bypass sandboxing
@@ -73,14 +74,22 @@
 
 from pkg_resources.extern import platformdirs
 from pkg_resources.extern import packaging
-import locale
 
 __import__('pkg_resources.extern.packaging.version')
 __import__('pkg_resources.extern.packaging.specifiers')
 __import__('pkg_resources.extern.packaging.requirements')
 __import__('pkg_resources.extern.packaging.markers')
 __import__('pkg_resources.extern.packaging.utils')
 
+encoding_for_open = (
+    "locale" if sys.version_info >= (3, 10) else locale.getpreferredencoding(False)
+)
+"""
+This variable exists to centralize calls to `getpreferredencoding`
+to reduce the amount of `EncodingWarning` in tests logs.
+"""
+
+
 # declare some globals that will be defined later to
 # satisfy the linters.
 require = None
@@ -1539,9 +1548,7 @@ def run_script(self, script_name, namespace):
         script_filename = self._fn(self.egg_info, script)
         namespace['__file__'] = script_filename
         if os.path.exists(script_filename):
-            with open(
-                script_filename, encoding=locale.getpreferredencoding(False)
-            ) as fid:
+            with open(script_filename, encoding=encoding_for_open) as fid:
                 source = fid.read()
             code = compile(source, script_filename, 'exec')
             exec(code, namespace, namespace)
@@ -2192,7 +2199,7 @@ def non_empty_lines(path):
     """
     Yield non-empty lines from file at path
     """
-    with open(path, encoding=locale.getpreferredencoding(False)) as f:
+    with open(path, encoding=encoding_for_open) as f:
         for line in f:
             line = line.strip()
             if line:

pkg_resources/tests/test_pkg_resources.py

@@ -16,12 +16,12 @@
     DistInfoDistribution,
     Distribution,
     EggInfoDistribution,
+    encoding_for_open,
 )
 
 import pytest
 
 import pkg_resources
-import locale
 
 
 class EggRemover(str):
@@ -111,13 +111,13 @@ def test_resource_filename_rewrites_on_change(self):
         filename = zp.get_resource_filename(manager, 'data.dat')
         actual = datetime.datetime.fromtimestamp(os.stat(filename).st_mtime)
         assert actual == self.ref_time
-        f = open(filename, 'w', encoding=locale.getpreferredencoding(False))
+        f = open(filename, 'w', encoding=encoding_for_open)
         f.write('hello, world?')
         f.close()
         ts = self.ref_time.timestamp()
         os.utime(filename, (ts, ts))
         filename = zp.get_resource_filename(manager, 'data.dat')
-        with open(filename, encoding=locale.getpreferredencoding(False)) as f:
+        with open(filename, encoding=encoding_for_open) as f:
             assert f.read() == 'hello, world!'
         manager.cleanup_resources()
 

setuptools/command/bdist_egg.py

@@ -14,9 +14,9 @@
 from setuptools.extension import Library
 from setuptools import Command
 from .._path import ensure_directory
+from ..compat.encoding import encoding_for_open
 
 from sysconfig import get_path, get_python_version
-import locale
 
 
 def _get_purelib():
@@ -55,7 +55,7 @@ def __bootstrap__():
         __bootstrap__()
         """
     ).lstrip()
-    with open(pyfile, 'w', encoding=locale.getpreferredencoding(False)) as f:
+    with open(pyfile, 'w', encoding=encoding_for_open) as f:
         f.write(_stub_template % resource)
 
 
@@ -201,9 +201,7 @@ def run(self):  # noqa: C901  # is too complex (14)  # FIXME
             log.info("writing %s", native_libs)
             if not self.dry_run:
                 ensure_directory(native_libs)
-                libs_file = open(
-                    native_libs, 'wt', encoding=locale.getpreferredencoding(False)
-                )
+                libs_file = open(native_libs, 'wt', encoding=encoding_for_open)
                 libs_file.write('\n'.join(all_outputs))
                 libs_file.write('\n')
                 libs_file.close()
@@ -353,7 +351,7 @@ def write_safety_flag(egg_dir, safe):
             if safe is None or bool(safe) != flag:
                 os.unlink(fn)
         elif safe is not None and bool(safe) == flag:
-            f = open(fn, 'wt', encoding=locale.getpreferredencoding(False))
+            f = open(fn, 'wt', encoding=encoding_for_open)
             f.write('\n')
             f.close()
 

setuptools/command/build_ext.py

@@ -1,7 +1,6 @@
 import os
 import sys
 import itertools
-import locale
 from importlib.machinery import EXTENSION_SUFFIXES
 from importlib.util import cache_from_source as _compiled_file_name
 from typing import Dict, Iterator, List, Tuple
@@ -15,6 +14,8 @@
 from setuptools.errors import BaseError
 from setuptools.extension import Extension, Library
 
+from ..compat.encoding import encoding_for_open
+
 try:
     # Attempt to use Cython for building extensions, if available
     from Cython.Distutils.build_ext import build_ext as _build_ext
@@ -341,7 +342,7 @@ def _write_stub_file(self, stub_file: str, ext: Extension, compile=False):
         if compile and os.path.exists(stub_file):
             raise BaseError(stub_file + " already exists! Please delete.")
         if not self.dry_run:
-            f = open(stub_file, 'w', encoding=locale.getpreferredencoding(False))
+            f = open(stub_file, 'w', encoding=encoding_for_open)
             f.write(
                 '\n'.join([
                     "def __bootstrap__():",

setuptools/command/develop.py

@@ -9,7 +9,8 @@
 from setuptools import _path
 from setuptools import namespaces
 import setuptools
-import locale
+
+from ..compat.encoding import encoding_for_open
 
 
 class develop(namespaces.DevelopInstaller, easy_install):
@@ -120,9 +121,7 @@ def install_for_development(self):
         # create an .egg-link in the installation dir, pointing to our egg
         log.info("Creating %s (link to %s)", self.egg_link, self.egg_base)
         if not self.dry_run:
-            with open(
-                self.egg_link, "w", encoding=locale.getpreferredencoding(False)
-            ) as f:
+            with open(self.egg_link, "w", encoding=encoding_for_open) as f:
                 f.write(self.egg_path + "\n" + self.setup_path)
         # postprocess the installed distro, fixing up .pth, installing scripts,
         # and handling requirements
@@ -131,9 +130,7 @@ def install_for_development(self):
     def uninstall_link(self):
         if os.path.exists(self.egg_link):
             log.info("Removing %s (link to %s)", self.egg_link, self.egg_base)
-            egg_link_file = open(
-                self.egg_link, encoding=locale.getpreferredencoding(False)
-            )
+            egg_link_file = open(self.egg_link, encoding=encoding_for_open)
             contents = [line.rstrip() for line in egg_link_file]
             egg_link_file.close()
             if contents not in ([self.egg_path], [self.egg_path, self.setup_path]):
@@ -161,7 +158,7 @@ def install_egg_scripts(self, dist):
         for script_name in self.distribution.scripts or []:
             script_path = os.path.abspath(convert_path(script_name))
             script_name = os.path.basename(script_path)
-            with open(script_path, encoding=locale.getpreferredencoding(False)) as strm:
+            with open(script_path, encoding=encoding_for_open) as strm:
                 script_text = strm.read()
             self.install_script(dist, script_name, script_text, script_path)
 

setuptools/command/easy_install.py

@@ -75,10 +75,9 @@
 )
 import pkg_resources
 from ..compat import py311
+from ..compat.encoding import encoding_for_open
 from .._path import ensure_directory
 from ..extern.jaraco.text import yield_lines
-import locale
-
 
 # Turn on PEP440Warnings
 warnings.filterwarnings("default", category=pkg_resources.PEP440Warning)
@@ -492,7 +491,7 @@ def check_site_dir(self):  # noqa: C901  # is too complex (12)  # FIXME
             try:
                 if test_exists:
                     os.unlink(testfile)
-                open(testfile, 'w', encoding=locale.getpreferredencoding(False)).close()
+                open(testfile, 'w', encoding=encoding_for_open).close()
                 os.unlink(testfile)
             except OSError:
                 self.cant_write_to_target()
@@ -589,7 +588,7 @@ def check_pth_processing(self):
                 os.unlink(ok_file)
             dirname = os.path.dirname(ok_file)
             os.makedirs(dirname, exist_ok=True)
-            f = open(pth_file, 'w', encoding=locale.getpreferredencoding(False))
+            f = open(pth_file, 'w', encoding=encoding_for_open)
         except OSError:
             self.cant_write_to_target()
         else:
@@ -1017,7 +1016,7 @@ def install_exe(self, dist_filename, tmpdir):
 
         # Write EGG-INFO/PKG-INFO
         if not os.path.exists(pkg_inf):
-            f = open(pkg_inf, 'w', encoding=locale.getpreferredencoding(False))
+            f = open(pkg_inf, 'w', encoding=encoding_for_open)
             f.write('Metadata-Version: 1.0\n')
             for k, v in cfg.items('metadata'):
                 if k != 'target_version':
@@ -1088,7 +1087,7 @@ def process(src, dst):
             if locals()[name]:
                 txt = os.path.join(egg_tmp, 'EGG-INFO', name + '.txt')
                 if not os.path.exists(txt):
-                    f = open(txt, 'w', encoding=locale.getpreferredencoding(False))
+                    f = open(txt, 'w', encoding=encoding_for_open)
                     f.write('\n'.join(locals()[name]) + '\n')
                     f.close()
 
@@ -1278,7 +1277,7 @@ def update_pth(self, dist):  # noqa: C901  # is too complex (11)  # FIXME
         filename = os.path.join(self.install_dir, 'setuptools.pth')
         if os.path.islink(filename):
             os.unlink(filename)
-        with open(filename, 'wt', encoding=locale.getpreferredencoding(False)) as f:
+        with open(filename, 'wt', encoding=encoding_for_open) as f:
             f.write(self.pth_file.make_relative(dist.location) + '\n')
 
     def unpack_progress(self, src, dst):
@@ -1504,9 +1503,7 @@ def expand_paths(inputs):  # noqa: C901  # is too complex (11)  # FIXME
                 continue
 
             # Read the .pth file
-            f = open(
-                os.path.join(dirname, name), encoding=locale.getpreferredencoding(False)
-            )
+            f = open(os.path.join(dirname, name), encoding=encoding_for_open)
             lines = list(yield_lines(f))
             f.close()
 
@@ -1622,7 +1619,7 @@ def _load_raw(self):
         paths = []
         dirty = saw_import = False
         seen = dict.fromkeys(self.sitedirs)
-        f = open(self.filename, 'rt', encoding=locale.getpreferredencoding(False))
+        f = open(self.filename, 'rt', encoding=encoding_for_open)
         for line in f:
             path = line.rstrip()
             # still keep imports and empty/commented lines for formatting
@@ -1693,9 +1690,7 @@ def save(self):
             data = '\n'.join(lines) + '\n'
             if os.path.islink(self.filename):
                 os.unlink(self.filename)
-            with open(
-                self.filename, 'wt', encoding=locale.getpreferredencoding(False)
-            ) as f:
+            with open(self.filename, 'wt', encoding=encoding_for_open) as f:
                 f.write(data)
         elif os.path.exists(self.filename):
             log.debug("Deleting empty %s", self.filename)

setuptools/command/setopt.py

@@ -6,7 +6,8 @@
 import configparser
 
 from setuptools import Command
-import locale
+
+from ..compat.encoding import encoding_for_open
 
 __all__ = ['config_file', 'edit_config', 'option_base', 'setopt']
 
@@ -63,7 +64,7 @@ def edit_config(filename, settings, dry_run=False):
 
     log.info("Writing %s", filename)
     if not dry_run:
-        with open(filename, 'w', encoding=locale.getpreferredencoding(False)) as f:
+        with open(filename, 'w', encoding=encoding_for_open) as f:
             opts.write(f)
 
 

setuptools/compat/encoding.py

@@ -0,0 +1,23 @@
+import locale
+import sys
+
+
+encoding_for_open = (
+    "locale" if sys.version_info >= (3, 10) else locale.getpreferredencoding(False)
+)
+"""
+This variable exists to centralize calls to `getpreferredencoding`
+to reduce the amount of `EncodingWarning` in tests logs.
+"""
+
+encoding_for_pth = locale.getencoding() if sys.version_info >= (3, 11) else None
+"""
+When working with ``.pth`` files, let's ignore UTF-8 mode (``PYTHONUTF8`` or ``python -X utf8``)
+Ref.: https://peps.python.org/pep-0686/#locale-getencoding
+
+``.pth`` files break with UTF-8 encoding.
+https://github.com/python/cpython/issues/77102#issuecomment-1568457039
+
+Until Python 3.11 `TextIOWrapper` still used UTF-8 even with ``encoding='locale'``
+Ref.: https://peps.python.org/pep-0686/#fixing-encoding-locale-option
+"""

setuptools/dist.py

@@ -22,6 +22,7 @@
 from distutils.fancy_getopt import translate_longopt
 from distutils.util import strtobool
 
+
 from .extern.more_itertools import partition, unique_everseen
 from .extern.ordered_set import OrderedSet
 from .extern.packaging.markers import InvalidMarker, Marker
@@ -34,10 +35,10 @@
 from . import command as _  # noqa  -- imported for side-effects
 from ._importlib import metadata
 from .config import setupcfg, pyprojecttoml
+from .compat.encoding import encoding_for_open
 from .discovery import ConfigDiscovery
 from .monkey import get_unpatched
 from .warnings import InformationOnly, SetuptoolsDeprecationWarning
-import locale
 
 
 sequence = tuple, list
@@ -682,9 +683,7 @@ def get_egg_cache_dir(self):
             os.mkdir(egg_cache_dir)
             windows_support.hide_file(egg_cache_dir)
             readme_txt_filename = os.path.join(egg_cache_dir, 'README.txt')
-            with open(
-                readme_txt_filename, 'w', encoding=locale.getpreferredencoding(False)
-            ) as f:
+            with open(readme_txt_filename, 'w', encoding=encoding_for_open) as f:
                 f.write(
                     'This directory contains eggs that were downloaded '
                     'by setuptools to build, test, and run plug-ins.\n\n'

setuptools/namespaces.py

@@ -1,8 +1,8 @@
 import os
 from distutils import log
 import itertools
-import locale
 
+from .compat.encoding import encoding_for_open
 
 flatten = itertools.chain.from_iterable
 
@@ -24,7 +24,7 @@ def install_namespaces(self):
             list(lines)
             return
 
-        with open(filename, 'wt', encoding=locale.getpreferredencoding(False)) as f:
+        with open(filename, 'wt', encoding=encoding_for_open) as f:
             f.writelines(lines)
 
     def uninstall_namespaces(self):