Merge pull request #6683 from cjerdonek/make-subprocess-error-path-display

Support non-ascii cwds in make_subprocess_output_error()

Author: cjerdonek

src/pip/_internal/utils/misc.py

@@ -22,7 +22,7 @@
 # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is
 #       why we ignore the type on this import.
 from pip._vendor.retrying import retry  # type: ignore
-from pip._vendor.six import PY2
+from pip._vendor.six import PY2, text_type
 from pip._vendor.six.moves import input, shlex_quote
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 from pip._vendor.six.moves.urllib import request as urllib_request
@@ -186,6 +186,40 @@ def rmtree_errorhandler(func, path, exc_info):
         raise
 
 
+def path_to_display(path):
+    # type: (Optional[Union[str, Text]]) -> Optional[Text]
+    """
+    Convert a bytes (or text) path to text (unicode in Python 2) for display
+    and logging purposes.
+
+    This function should never error out. Also, this function is mainly needed
+    for Python 2 since in Python 3 str paths are already text.
+    """
+    if path is None:
+        return None
+    if isinstance(path, text_type):
+        return path
+    # Otherwise, path is a bytes object (str in Python 2).
+    try:
+        display_path = path.decode(sys.getfilesystemencoding(), 'strict')
+    except UnicodeDecodeError:
+        # Include the full bytes to make troubleshooting easier, even though
+        # it may not be very human readable.
+        if PY2:
+            # Convert the bytes to a readable str representation using
+            # repr(), and then convert the str to unicode.
+            #   Also, we add the prefix "b" to the repr() return value both
+            # to make the Python 2 output look like the Python 3 output, and
+            # to signal to the user that this is a bytes representation.
+            display_path = str_to_display('b{!r}'.format(path))
+        else:
+            # Silence the "F821 undefined name 'ascii'" flake8 error since
+            # in Python 3 ascii() is a built-in.
+            display_path = ascii(path)  # noqa: F821
+
+    return display_path
+
+
 def display_path(path):
     # type: (Union[str, Text]) -> str
     """Gives the display value for a given path, making it relative to cwd
@@ -751,11 +785,12 @@ def make_subprocess_output_error(
     :param lines: A list of lines, each ending with a newline.
     """
     command = format_command_args(cmd_args)
-    # Convert `command` to text (unicode in Python 2) so we can use it as
-    # an argument in the unicode format string below. This avoids
+    # Convert `command` and `cwd` to text (unicode in Python 2) so we can use
+    # them as arguments in the unicode format string below. This avoids
     # "UnicodeDecodeError: 'ascii' codec can't decode byte ..." in Python 2
-    # when the formatted command contains a non-ascii character.
+    # if either contains a non-ascii character.
     command_display = str_to_display(command, desc='command bytes')
+    cwd_display = path_to_display(cwd)
 
     # We know the joined output value ends in a newline.
     output = ''.join(lines)
@@ -765,12 +800,12 @@ def make_subprocess_output_error(
         # argument (e.g. `output`) has a non-ascii character.
         u'Command errored out with exit status {exit_status}:\n'
         ' command: {command_display}\n'
-        '     cwd: {cwd}\n'
+        '     cwd: {cwd_display}\n'
         'Complete output ({line_count} lines):\n{output}{divider}'
     ).format(
         exit_status=exit_status,
         command_display=command_display,
-        cwd=cwd,
+        cwd_display=cwd_display,
         line_count=len(lines),
         output=output,
         divider=LOG_DIVIDER,

tests/unit/test_utils.py

@@ -32,8 +32,8 @@
 from pip._internal.utils.misc import (
     call_subprocess, egg_link_path, ensure_dir, format_command_args,
     get_installed_distributions, get_prog, make_subprocess_output_error,
-    normalize_path, normalize_version_info, path_to_url, redact_netloc,
-    redact_password_from_url, remove_auth_from_url, rmtree,
+    normalize_path, normalize_version_info, path_to_display, path_to_url,
+    redact_netloc, redact_password_from_url, remove_auth_from_url, rmtree,
     split_auth_from_netloc, split_auth_netloc_from_url, untar_file, unzip_file,
 )
 from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
@@ -384,6 +384,24 @@ def test_rmtree_retries_for_3sec(tmpdir, monkeypatch):
         rmtree('foo')
 
 
+@pytest.mark.parametrize('path, fs_encoding, expected', [
+    (None, None, None),
+    # Test passing a text (unicode) string.
+    (u'/path/déf', None, u'/path/déf'),
+    # Test a bytes object with a non-ascii character.
+    (u'/path/déf'.encode('utf-8'), 'utf-8', u'/path/déf'),
+    # Test a bytes object with a character that can't be decoded.
+    (u'/path/déf'.encode('utf-8'), 'ascii', u"b'/path/d\\xc3\\xa9f'"),
+    (u'/path/déf'.encode('utf-16'), 'utf-8',
+     u"b'\\xff\\xfe/\\x00p\\x00a\\x00t\\x00h\\x00/"
+     "\\x00d\\x00\\xe9\\x00f\\x00'"),
+])
+def test_path_to_display(monkeypatch, path, fs_encoding, expected):
+    monkeypatch.setattr(sys, 'getfilesystemencoding', lambda: fs_encoding)
+    actual = path_to_display(path)
+    assert actual == expected, 'actual: {!r}'.format(actual)
+
+
 class Test_normalize_path(object):
     # Technically, symlinks are possible on Windows, but you need a special
     # permission bit to create them, and Python 2 doesn't support it anyway, so
@@ -796,6 +814,58 @@ def test_make_subprocess_output_error__non_ascii_command_arg(monkeypatch):
     assert actual == expected, u'actual: {}'.format(actual)
 
 
+@pytest.mark.skipif("sys.version_info < (3,)")
+def test_make_subprocess_output_error__non_ascii_cwd_python_3(monkeypatch):
+    """
+    Test a str (text) cwd with a non-ascii character in Python 3.
+    """
+    cmd_args = ['test']
+    cwd = '/path/to/cwd/déf'
+    actual = make_subprocess_output_error(
+        cmd_args=cmd_args,
+        cwd=cwd,
+        lines=[],
+        exit_status=1,
+    )
+    expected = dedent("""\
+    Command errored out with exit status 1:
+     command: test
+         cwd: /path/to/cwd/déf
+    Complete output (0 lines):
+    ----------------------------------------""")
+    assert actual == expected, 'actual: {}'.format(actual)
+
+
+@pytest.mark.parametrize('encoding', [
+    'utf-8',
+    # Test a Windows encoding.
+    'cp1252',
+])
+@pytest.mark.skipif("sys.version_info >= (3,)")
+def test_make_subprocess_output_error__non_ascii_cwd_python_2(
+    monkeypatch, encoding,
+):
+    """
+    Test a str (bytes object) cwd with a non-ascii character in Python 2.
+    """
+    cmd_args = ['test']
+    cwd = u'/path/to/cwd/déf'.encode(encoding)
+    monkeypatch.setattr(sys, 'getfilesystemencoding', lambda: encoding)
+    actual = make_subprocess_output_error(
+        cmd_args=cmd_args,
+        cwd=cwd,
+        lines=[],
+        exit_status=1,
+    )
+    expected = dedent(u"""\
+    Command errored out with exit status 1:
+     command: test
+         cwd: /path/to/cwd/déf
+    Complete output (0 lines):
+    ----------------------------------------""")
+    assert actual == expected, u'actual: {}'.format(actual)
+
+
 # This test is mainly important for checking unicode in Python 2.
 def test_make_subprocess_output_error__non_ascii_line():
     """