BUG: Fixed tm.set_locale context manager, it could error and leak when category LC_ALL was used (pandas-dev#47570)

Author: smarie

pandas/_config/localization.py

@@ -39,7 +39,8 @@ def set_locale(
     particular locale, without globally setting the locale. This probably isn't
     thread-safe.
     """
-    current_locale = locale.getlocale()
+    # getlocale is not always compliant with setlocale, use setlocale. GH#46595
+    current_locale = locale.setlocale(lc_var)
 
     try:
         locale.setlocale(lc_var, new_locale)

pandas/tests/config/test_localization.py

@@ -10,31 +10,67 @@
     set_locale,
 )
 
-from pandas.compat import is_platform_windows
-
 import pandas as pd
 
 _all_locales = get_locales() or []
-_current_locale = locale.getlocale()
+_current_locale = locale.setlocale(locale.LC_ALL)  # getlocale() is wrong, see GH#46595
 
-# Don't run any of these tests if we are on Windows or have no locales.
-pytestmark = pytest.mark.skipif(
-    is_platform_windows() or not _all_locales, reason="Need non-Windows and locales"
-)
+# Don't run any of these tests if we have no locales.
+pytestmark = pytest.mark.skipif(not _all_locales, reason="Need locales")
 
 _skip_if_only_one_locale = pytest.mark.skipif(
     len(_all_locales) <= 1, reason="Need multiple locales for meaningful test"
 )
 
 
-def test_can_set_locale_valid_set():
+def _get_current_locale(lc_var: int = locale.LC_ALL) -> str:
+    # getlocale is not always compliant with setlocale, use setlocale. GH#46595
+    return locale.setlocale(lc_var)
+
+
+@pytest.mark.parametrize("lc_var", (locale.LC_ALL, locale.LC_CTYPE, locale.LC_TIME))
+def test_can_set_current_locale(lc_var):
+    # Can set the current locale
+    before_locale = _get_current_locale(lc_var)
+    assert can_set_locale(before_locale, lc_var=lc_var)
+    after_locale = _get_current_locale(lc_var)
+    assert before_locale == after_locale
+
+
+@pytest.mark.parametrize("lc_var", (locale.LC_ALL, locale.LC_CTYPE, locale.LC_TIME))
+def test_can_set_locale_valid_set(lc_var):
     # Can set the default locale.
-    assert can_set_locale("")
+    before_locale = _get_current_locale(lc_var)
+    assert can_set_locale("", lc_var=lc_var)
+    after_locale = _get_current_locale(lc_var)
+    assert before_locale == after_locale
 
 
-def test_can_set_locale_invalid_set():
+@pytest.mark.parametrize("lc_var", (locale.LC_ALL, locale.LC_CTYPE, locale.LC_TIME))
+def test_can_set_locale_invalid_set(lc_var):
     # Cannot set an invalid locale.
-    assert not can_set_locale("non-existent_locale")
+    before_locale = _get_current_locale(lc_var)
+    assert not can_set_locale("non-existent_locale", lc_var=lc_var)
+    after_locale = _get_current_locale(lc_var)
+    assert before_locale == after_locale
+
+
+@pytest.mark.parametrize(
+    "lang,enc",
+    [
+        ("it_CH", "UTF-8"),
+        ("en_US", "ascii"),
+        ("zh_CN", "GB2312"),
+        ("it_IT", "ISO-8859-1"),
+    ],
+)
+@pytest.mark.parametrize("lc_var", (locale.LC_ALL, locale.LC_CTYPE, locale.LC_TIME))
+def test_can_set_locale_no_leak(lang, enc, lc_var):
+    # Test that can_set_locale does not leak even when returning False. See GH#46595
+    before_locale = _get_current_locale(lc_var)
+    can_set_locale((lang, enc), locale.LC_ALL)
+    after_locale = _get_current_locale(lc_var)
+    assert before_locale == after_locale
 
 
 def test_can_set_locale_invalid_get(monkeypatch):
@@ -72,10 +108,7 @@ def test_get_locales_prefix():
     ],
 )
 def test_set_locale(lang, enc):
-    if all(x is None for x in _current_locale):
-        # Not sure why, but on some Travis runs with pytest,
-        #  getlocale() returned (None, None).
-        pytest.skip("Current locale is not set.")
+    before_locale = _get_current_locale()
 
     enc = codecs.lookup(enc).name
     new_locale = lang, enc
@@ -95,8 +128,8 @@ def test_set_locale(lang, enc):
             assert normalized_locale == new_locale
 
     # Once we exit the "with" statement, locale should be back to what it was.
-    current_locale = locale.getlocale()
-    assert current_locale == _current_locale
+    after_locale = _get_current_locale()
+    assert before_locale == after_locale
 
 
 def test_encoding_detected():

pandas/tests/tools/test_to_datetime.py

@@ -284,13 +284,16 @@ def test_to_datetime_format_microsecond(self, cache):
                 "%m/%d/%Y %H:%M:%S",
                 Timestamp("2010-01-10 13:56:01"),
             ],
+            # The 3 tests below are locale-dependent.
+            # They pass, except when the machine locale is zh_CN or it_IT .
             pytest.param(
                 "01/10/2010 08:14 PM",
                 "%m/%d/%Y %I:%M %p",
                 Timestamp("2010-01-10 20:14"),
                 marks=pytest.mark.xfail(
                     locale.getlocale()[0] in ("zh_CN", "it_IT"),
                     reason="fail on a CI build with LC_ALL=zh_CN.utf8/it_IT.utf8",
+                    strict=False,
                 ),
             ),
             pytest.param(
@@ -300,6 +303,7 @@ def test_to_datetime_format_microsecond(self, cache):
                 marks=pytest.mark.xfail(
                     locale.getlocale()[0] in ("zh_CN", "it_IT"),
                     reason="fail on a CI build with LC_ALL=zh_CN.utf8/it_IT.utf8",
+                    strict=False,
                 ),
             ),
             pytest.param(
@@ -309,6 +313,7 @@ def test_to_datetime_format_microsecond(self, cache):
                 marks=pytest.mark.xfail(
                     locale.getlocale()[0] in ("zh_CN", "it_IT"),
                     reason="fail on a CI build with LC_ALL=zh_CN.utf8/it_IT.utf8",
+                    strict=False,
                 ),
             ),
         ],

pandas/tests/tools/test_to_time.py

@@ -9,9 +9,12 @@
 from pandas.core.tools.datetimes import to_time as to_time_alias
 from pandas.core.tools.times import to_time
 
+# The tests marked with this are locale-dependent.
+# They pass, except when the machine locale is zh_CN or it_IT.
 fails_on_non_english = pytest.mark.xfail(
     locale.getlocale()[0] in ("zh_CN", "it_IT"),
     reason="fail on a CI build with LC_ALL=zh_CN.utf8/it_IT.utf8",
+    strict=False,
 )