Issue #16414: Add support.FS_NONASCII and support.TESTFN_NONASCII

These constants are used to test functions with non-ASCII data, especially
filenames.

Author: vstinner

Lib/test/support.py

@@ -603,6 +603,32 @@ def _is_ipv6_enabled():
 # module name.
 TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
 
+# FS_NONASCII: non-ASCII character encodable by os.fsencode(),
+# or None if there is no such character.
+FS_NONASCII = None
+for character in (
+    # U+00E6 (Latin small letter AE): Encodable to cp1252, cp1254, cp1257, iso-8859-1
+    '\u00E6',
+    # U+0141 (Latin capital letter L with stroke): Encodable to cp1250, cp1257
+    '\u0141',
+    # U+041A (Cyrillic capital letter KA): Encodable to cp932, cp950, cp1251
+    '\u041A',
+    # U+05D0 (Hebrew Letter Alef): Encodable to cp424, cp1255
+    '\u05D0',
+    # U+06A9 (Arabic letter KEHEH): Encodable to cp1256
+    '\u06A9',
+    # U+03A9 (Greek capital letter OMEGA): Encodable to cp932, cp950, cp1253
+    '\u03A9',
+    # U+0E01 (Thai character KO KAI): Encodable to cp874
+    '\u0E01',
+):
+    try:
+        os.fsdecode(os.fsencode(character))
+    except UnicodeError:
+        pass
+    else:
+        FS_NONASCII = character
+        break
 
 # TESTFN_UNICODE is a non-ascii filename
 TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
@@ -658,6 +684,11 @@ def _is_ipv6_enabled():
         TESTFN_UNDECODABLE = name
         break
 
+if FS_NONASCII:
+    TESTFN_NONASCII = TESTFN + '- ' + FS_NONASCII
+else:
+    TESTFN_NONASCII = None
+
 # Save the initial cwd
 SAVEDCWD = os.getcwd()
 

Lib/test/test_cmd_line.py

@@ -93,15 +93,15 @@ def test_run_code(self):
         # All good if execution is successful
         assert_python_ok('-c', 'pass')
 
-    @unittest.skipIf(sys.getfilesystemencoding() == 'ascii',
-                     'need a filesystem encoding different than ASCII')
+    @unittest.skipUnless(test.support.FS_NONASCII, 'need support.FS_NONASCII')
     def test_non_ascii(self):
         # Test handling of non-ascii data
         if test.support.verbose:
             import locale
             print('locale encoding = %s, filesystem encoding = %s'
                   % (locale.getpreferredencoding(), sys.getfilesystemencoding()))
-        command = "assert(ord('\xe9') == 0xe9)"
+        command = ("assert(ord(%r) == %s)"
+                   % (test.support.FS_NONASCII, ord(test.support.FS_NONASCII)))
         assert_python_ok('-c', command)
 
     # On Windows, pass bytes to subprocess doesn't test how Python decodes the

Lib/test/test_cmd_line_script.py

@@ -363,19 +363,12 @@ def test_pep_409_verbiage(self):
             self.assertTrue(text[1].startswith('  File '))
             self.assertTrue(text[3].startswith('NameError'))
 
+    @unittest.skipUnless(support.TESTFN_NONASCII, 'need support.TESTFN_NONASCII')
     def test_non_ascii(self):
         # Issue #16218
         # non-ascii filename encodable to cp1252, cp932, latin1 and utf8
-        filename = support.TESTFN + '\xa3'
-        try:
-            os.fsencode(filename)
-        except UnicodeEncodeError:
-            self.skipTest(
-                "Filesystem encoding %r cannot encode "
-                "the filename: %a"
-                % (sys.getfilesystemencoding(), filename))
         source = 'print(ascii(__file__))\n'
-        script_name = _make_test_script(os.curdir, filename, source)
+        script_name = _make_test_script(os.curdir, support.TESTFN_NONASCII, source)
         self.addCleanup(support.unlink, script_name)
         rc, stdout, stderr = assert_python_ok(script_name)
         self.assertEqual(

Lib/test/test_genericpath.py

@@ -313,6 +313,8 @@ def test_abspath_issue3426(self):
     def test_nonascii_abspath(self):
         if support.TESTFN_UNDECODABLE:
             name = support.TESTFN_UNDECODABLE
+        elif support.TESTFN_NONASCII:
+            name = support.TESTFN_NONASCII
         else:
             name = b'a\xffb\xe7w\xf0'
 

Lib/test/test_os.py

@@ -1243,6 +1243,8 @@ class Pep383Tests(unittest.TestCase):
         def setUp(self):
             if support.TESTFN_UNENCODABLE:
                 self.dir = support.TESTFN_UNENCODABLE
+            elif support.TESTFN_NONASCII:
+                self.dir = support.TESTFN_NONASCII
             else:
                 self.dir = support.TESTFN
             self.bdir = os.fsencode(self.dir)
@@ -1257,6 +1259,8 @@ def add_filename(fn):
             add_filename(support.TESTFN_UNICODE)
             if support.TESTFN_UNENCODABLE:
                 add_filename(support.TESTFN_UNENCODABLE)
+            if support.TESTFN_NONASCII:
+                add_filename(support.TESTFN_NONASCII)
             if not bytesfn:
                 self.skipTest("couldn't create any non-ascii filename")