Explicitly use "locale" encoding for .pth files (#4265)

Author: abravalheri

docs/conf.py

@@ -55,6 +55,10 @@
                 pattern=r'(Python #|bpo-)(?P<python>\d+)',
                 url='https://bugs.python.org/issue{python}',
             ),
+            dict(
+                pattern=r'\bpython/cpython#(?P<cpython>\d+)',
+                url='{GH}/python/cpython/issues/{cpython}',
+            ),
             dict(
                 pattern=r'Interop #(?P<interop>\d+)',
                 url='{GH}/pypa/interoperability-peps/issues/{interop}',

newsfragments/4265.feature.rst

@@ -0,0 +1,3 @@
+Explicitly use ``encoding="locale"`` for ``.pth`` files whenever possible,
+to  reduce ``EncodingWarnings``.
+This avoid errors with UTF-8 (see discussion in python/cpython#77102).

setuptools/command/easy_install.py

@@ -74,7 +74,7 @@
     DEVELOP_DIST,
 )
 import pkg_resources
-from ..compat import py311
+from ..compat import py39, py311
 from .._path import ensure_directory
 from ..extern.jaraco.text import yield_lines
 
@@ -491,7 +491,7 @@ def check_site_dir(self):  # noqa: C901  # is too complex (12)  # FIXME
             try:
                 if test_exists:
                     os.unlink(testfile)
-                open(testfile, 'w').close()
+                open(testfile, 'wb').close()
                 os.unlink(testfile)
             except OSError:
                 self.cant_write_to_target()
@@ -576,7 +576,7 @@ def check_pth_processing(self):
             _one_liner(
                 """
             import os
-            f = open({ok_file!r}, 'w')
+            f = open({ok_file!r}, 'w', encoding="utf-8")
             f.write('OK')
             f.close()
             """
@@ -588,7 +588,8 @@ def check_pth_processing(self):
                 os.unlink(ok_file)
             dirname = os.path.dirname(ok_file)
             os.makedirs(dirname, exist_ok=True)
-            f = open(pth_file, 'w')
+            f = open(pth_file, 'w', encoding=py39.LOCALE_ENCODING)
+            # ^-- Requires encoding="locale" instead of "utf-8" (python/cpython#77102).
         except OSError:
             self.cant_write_to_target()
         else:
@@ -872,7 +873,7 @@ def write_script(self, script_name, contents, mode="t", blockers=()):
         ensure_directory(target)
         if os.path.exists(target):
             os.unlink(target)
-        with open(target, "w" + mode) as f:
+        with open(target, "w" + mode) as f:  # TODO: is it safe to use utf-8?
             f.write(contents)
         chmod(target, 0o777 - mask)
 
@@ -1016,7 +1017,7 @@ def install_exe(self, dist_filename, tmpdir):
 
         # Write EGG-INFO/PKG-INFO
         if not os.path.exists(pkg_inf):
-            f = open(pkg_inf, 'w')
+            f = open(pkg_inf, 'w')  # TODO: probably it is safe to use utf-8
             f.write('Metadata-Version: 1.0\n')
             for k, v in cfg.items('metadata'):
                 if k != 'target_version':
@@ -1087,7 +1088,7 @@ def process(src, dst):
             if locals()[name]:
                 txt = os.path.join(egg_tmp, 'EGG-INFO', name + '.txt')
                 if not os.path.exists(txt):
-                    f = open(txt, 'w')
+                    f = open(txt, 'w')  # TODO: probably it is safe to use utf-8
                     f.write('\n'.join(locals()[name]) + '\n')
                     f.close()
 
@@ -1277,7 +1278,9 @@ def update_pth(self, dist):  # noqa: C901  # is too complex (11)  # FIXME
         filename = os.path.join(self.install_dir, 'setuptools.pth')
         if os.path.islink(filename):
             os.unlink(filename)
-        with open(filename, 'wt') as f:
+
+        with open(filename, 'wt', encoding=py39.LOCALE_ENCODING) as f:
+            # Requires encoding="locale" instead of "utf-8" (python/cpython#77102).
             f.write(self.pth_file.make_relative(dist.location) + '\n')
 
     def unpack_progress(self, src, dst):
@@ -1503,9 +1506,9 @@ def expand_paths(inputs):  # noqa: C901  # is too complex (11)  # FIXME
                 continue
 
             # Read the .pth file
-            f = open(os.path.join(dirname, name))
-            lines = list(yield_lines(f))
-            f.close()
+            with open(os.path.join(dirname, name), encoding=py39.LOCALE_ENCODING) as f:
+                # Requires encoding="locale" instead of "utf-8" (python/cpython#77102).
+                lines = list(yield_lines(f))
 
             # Yield existing non-dupe, non-import directory lines from it
             for line in lines:
@@ -1619,7 +1622,8 @@ def _load_raw(self):
         paths = []
         dirty = saw_import = False
         seen = dict.fromkeys(self.sitedirs)
-        f = open(self.filename, 'rt')
+        f = open(self.filename, 'rt', encoding=py39.LOCALE_ENCODING)
+        # ^-- Requires encoding="locale" instead of "utf-8" (python/cpython#77102).
         for line in f:
             path = line.rstrip()
             # still keep imports and empty/commented lines for formatting
@@ -1690,7 +1694,8 @@ def save(self):
             data = '\n'.join(lines) + '\n'
             if os.path.islink(self.filename):
                 os.unlink(self.filename)
-            with open(self.filename, 'wt') as f:
+            with open(self.filename, 'wt', encoding=py39.LOCALE_ENCODING) as f:
+                # Requires encoding="locale" instead of "utf-8" (python/cpython#77102).
                 f.write(data)
         elif os.path.exists(self.filename):
             log.debug("Deleting empty %s", self.filename)

setuptools/command/editable_wheel.py

@@ -14,7 +14,6 @@
 import io
 import os
 import shutil
-import sys
 import traceback
 from contextlib import suppress
 from enum import Enum
@@ -44,6 +43,7 @@
     namespaces,
 )
 from .._path import StrPath
+from ..compat import py39
 from ..discovery import find_package_path
 from ..dist import Distribution
 from ..warnings import (
@@ -558,9 +558,8 @@ def _encode_pth(content: str) -> bytes:
     (There seems to be some variety in the way different version of Python handle
     ``encoding=None``, not all of them use ``locale.getpreferredencoding(False)``).
     """
-    encoding = "locale" if sys.version_info >= (3, 10) else None
     with io.BytesIO() as buffer:
-        wrapper = io.TextIOWrapper(buffer, encoding)
+        wrapper = io.TextIOWrapper(buffer, encoding=py39.LOCALE_ENCODING)
         wrapper.write(content)
         wrapper.flush()
         buffer.seek(0)

setuptools/compat/py39.py

@@ -0,0 +1,9 @@
+import sys
+
+# Explicitly use the ``"locale"`` encoding in versions that support it,
+# otherwise just rely on the implicit handling of ``encoding=None``.
+# Since all platforms that support ``EncodingWarning`` also support
+# ``encoding="locale"``, this can be used to suppress the warning.
+# However, please try to use UTF-8 when possible
+# (.pth files are the notorious exception: python/cpython#77102, pypa/setuptools#3937).
+LOCALE_ENCODING = "locale" if sys.version_info >= (3, 10) else None

setuptools/namespaces.py

@@ -2,6 +2,8 @@
 from distutils import log
 import itertools
 
+from .compat import py39
+
 
 flatten = itertools.chain.from_iterable
 
@@ -23,7 +25,8 @@ def install_namespaces(self):
             list(lines)
             return
 
-        with open(filename, 'wt') as f:
+        with open(filename, 'wt', encoding=py39.LOCALE_ENCODING) as f:
+            # Requires encoding="locale" instead of "utf-8" (python/cpython#77102).
             f.writelines(lines)
 
     def uninstall_namespaces(self):