share paths and fix bugs

Author: MarcoGorelli

doc/source/whatsnew/v2.0.0.rst

@@ -790,6 +790,8 @@ Datetimelike
 - Bug in :func:`to_datetime` was raising ``ValueError`` when parsing :class:`Timestamp`, ``datetime.datetime``, ``datetime.date``, or ``np.datetime64`` objects when non-ISO8601 ``format`` was passed (:issue:`49298`, :issue:`50036`)
 - Bug in :func:`to_datetime` was raising ``ValueError`` when parsing empty string and non-ISO8601 format was passed. Now, empty strings will be parsed as :class:`NaT`, for compatibility with how is done for ISO8601 formats (:issue:`50251`)
 - Bug in :class:`Timestamp` was showing ``UserWarning``, which was not actionable by users, when parsing non-ISO8601 delimited date strings (:issue:`50232`)
+- Bug in :class:`Timestamp` was showing ``UserWarning`` which was not actionable by users (:issue:`50232`)
+- Bug in :func:`to_datetime` was raising ``ValueError`` when parsing empty string and non-ISO8601 format was passed. Now, empty strings will be parsed as :class:`NaT`, for compatibility with how is done for ISO8601 formats (:issue:`50251`)
 -
 
 Timedelta

pandas/_libs/tslib.pyx

@@ -410,8 +410,6 @@ cpdef array_to_datetime(
     bint yearfirst=False,
     bint utc=False,
     bint require_iso8601=False,
-    format: str | None=None,
-    bint exact=True,
 ):
     """
     Converts a 1D array of date-like values to a numpy array of either:
@@ -517,7 +515,7 @@ cpdef array_to_datetime(
                         elif is_raise:
                             raise ValueError(
                                 f"time data \"{val}\" at position {i} doesn't "
-                                f"match format \"{format}\""
+                                f"match ISO8601 standard"
                             )
                         return values, tz_out
                     # these must be ns unit by-definition
@@ -550,7 +548,7 @@ cpdef array_to_datetime(
 
                     string_to_dts_failed = string_to_dts(
                         val, &dts, &out_bestunit, &out_local,
-                        &out_tzoffset, False, format, exact
+                        &out_tzoffset, False, None, False
                     )
                     if string_to_dts_failed:
                         # An error at this point is a _parsing_ error

pandas/_libs/tslibs/parsing.pyx

@@ -846,7 +846,7 @@ def format_is_iso(f: str) -> bint:
     but must be consistent.  Leading 0s in dates and times are optional.
     """
     iso_template = "%Y{date_sep}%m{date_sep}%d{time_sep}%H:%M:%S{micro_or_tz}".format
-    excluded_formats = ["%Y%m%d", "%Y%m", "%Y"]
+    excluded_formats = ["%Y%m", "%Y"]
 
     for date_sep in [" ", "/", "\\", "-", ".", ""]:
         for time_sep in [" ", "T"]:

pandas/_libs/tslibs/strptime.pyx

@@ -34,12 +34,14 @@ from pandas._libs.tslibs.nattype cimport (
     c_nat_strings as nat_strings,
 )
 from pandas._libs.tslibs.np_datetime cimport (
+    NPY_DATETIMEUNIT,
     NPY_FR_ns,
     check_dts_bounds,
     npy_datetimestruct,
     npy_datetimestruct_to_datetime,
     pydate_to_dt64,
     pydatetime_to_dt64,
+    string_to_dts,
 )
 from pandas._libs.tslibs.timestamps cimport _Timestamp
 from pandas._libs.util cimport (
@@ -93,6 +95,7 @@ def array_strptime(
     exact : matches must be exact if True, search if False
     errors : string specifying error handling, {'raise', 'ignore', 'coerce'}
     """
+    from pandas._libs.tslibs.parsing import format_is_iso
 
     cdef:
         Py_ssize_t i, n = len(values)
@@ -110,6 +113,9 @@ def array_strptime(
         bint found_naive = False
         bint found_tz = False
         tzinfo tz_out = None
+        bint iso_format = fmt is not None and format_is_iso(fmt)
+        NPY_DATETIMEUNIT out_bestunit
+        int out_local = 0, out_tzoffset = 0
 
     assert is_raise or is_ignore or is_coerce
 
@@ -198,6 +204,39 @@ def array_strptime(
         else:
             val = str(val)
 
+        if iso_format:
+            string_to_dts_failed = string_to_dts(
+                val, &dts, &out_bestunit, &out_local,
+                &out_tzoffset, False, fmt, exact
+            )
+            if not string_to_dts_failed:
+                # No error reported by string_to_dts, pick back up
+                # where we left off
+                value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
+                if out_local == 1:
+                    # Store the out_tzoffset in seconds
+                    # since we store the total_seconds of
+                    # dateutil.tz.tzoffset objects
+                    # out_tzoffset_vals.add(out_tzoffset * 60.)
+                    tz = timezone(timedelta(minutes=out_tzoffset))
+                    result_timezone[i] = tz
+                    # value = tz_localize_to_utc_single(value, tz)
+                    out_local = 0
+                    out_tzoffset = 0
+                iresult[i] = value
+                try:
+                    check_dts_bounds(&dts)
+                except ValueError:
+                    if is_coerce:
+                        iresult[i] = NPY_NAT
+                        continue
+                    raise
+                continue
+
+        # Some ISO formats can't be parsed by string_to_dts
+        # For example, 6-digit YYYYMD. So, if there's an error,
+        # try the string-matching code below.
+
         # exact matching
         if exact:
             found = format_regex.match(val)

pandas/core/arrays/datetimes.py

@@ -2120,8 +2120,6 @@ def objects_to_datetime64ns(
     errors: DateTimeErrorChoices = "raise",
     require_iso8601: bool = False,
     allow_object: bool = False,
-    format: str | None = None,
-    exact: bool = True,
 ):
     """
     Convert data to array of timestamps.
@@ -2166,8 +2164,6 @@ def objects_to_datetime64ns(
             dayfirst=dayfirst,
             yearfirst=yearfirst,
             require_iso8601=require_iso8601,
-            format=format,
-            exact=exact,
         )
         result = result.reshape(data.shape, order=order)
     except OverflowError as err:

pandas/core/tools/datetimes.py

@@ -36,7 +36,6 @@
 from pandas._libs.tslibs.conversion import precision_from_unit
 from pandas._libs.tslibs.parsing import (
     DateParseError,
-    format_is_iso,
     guess_datetime_format,
 )
 from pandas._libs.tslibs.strptime import array_strptime
@@ -419,7 +418,6 @@ def _convert_listlike_datetimes(
 
     # warn if passing timedelta64, raise for PeriodDtype
     # NB: this must come after unit transformation
-    orig_arg = arg
     try:
         arg, _ = maybe_convert_dtype(arg, copy=False, tz=libtimezones.maybe_get_tz(tz))
     except TypeError:
@@ -432,35 +430,20 @@ def _convert_listlike_datetimes(
         raise
 
     arg = ensure_object(arg)
-    require_iso8601 = False
 
     if format is None:
         format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
 
-    # There is a special fast-path for iso8601 formatted datetime strings
-    require_iso8601 = format is not None and format_is_iso(format)
-
-    if format is not None and not require_iso8601:
-        return _to_datetime_with_format(
-            arg,
-            orig_arg,
-            name,
-            utc,
-            format,
-            exact,
-            errors,
-        )
+    if format is not None:
+        return _array_strptime_with_fallback(arg, name, utc, format, exact, errors)
 
     result, tz_parsed = objects_to_datetime64ns(
         arg,
         dayfirst=dayfirst,
         yearfirst=yearfirst,
         utc=utc,
         errors=errors,
-        require_iso8601=require_iso8601,
         allow_object=True,
-        format=format,
-        exact=exact,
     )
 
     if tz_parsed is not None:
@@ -512,40 +495,6 @@ def _array_strptime_with_fallback(
     return _box_as_indexlike(result, utc=utc, name=name)
 
 
-def _to_datetime_with_format(
-    arg,
-    orig_arg,
-    name,
-    utc: bool,
-    fmt: str,
-    exact: bool,
-    errors: str,
-) -> Index:
-    """
-    Try parsing with the given format.
-    """
-    result = None
-
-    # shortcut formatting here
-    if fmt == "%Y%m%d":
-        # pass orig_arg as float-dtype may have been converted to
-        # datetime64[ns]
-        orig_arg = ensure_object(orig_arg)
-        try:
-            # may return None without raising
-            result = _attempt_YYYYMMDD(orig_arg, errors=errors)
-        except (ValueError, TypeError, OutOfBoundsDatetime) as err:
-            raise ValueError(
-                "cannot convert the input to '%Y%m%d' date format"
-            ) from err
-        if result is not None:
-            return _box_as_indexlike(result, utc=utc, name=name)
-
-    # fallback
-    res = _array_strptime_with_fallback(arg, name, utc, fmt, exact, errors)
-    return res
-
-
 def _to_datetime_with_unit(arg, unit, name, utc: bool, errors: str) -> Index:
     """
     to_datetime specalized to the case where a 'unit' is passed.
@@ -1000,7 +949,7 @@ def to_datetime(
     in addition to forcing non-dates (or non-parseable dates) to :const:`NaT`.
 
     >>> pd.to_datetime('13000101', format='%Y%m%d', errors='ignore')
-    datetime.datetime(1300, 1, 1, 0, 0)
+    '13000101'
     >>> pd.to_datetime('13000101', format='%Y%m%d', errors='coerce')
     NaT
 
@@ -1033,14 +982,12 @@ def to_datetime(
     Index([2020-10-25 02:00:00+02:00, 2020-10-25 04:00:00+01:00],
           dtype='object')
 
-    - A mix of timezone-aware and timezone-naive inputs is converted to
-      a timezone-aware :class:`DatetimeIndex` if the offsets of the timezone-aware
-      are constant:
+    - A mix of timezone-aware and timezone-naive inputs is also converted to
+      a simple :class:`Index` containing :class:`datetime.datetime` objects:
 
     >>> from datetime import datetime
     >>> pd.to_datetime(["2020-01-01 01:00:00-01:00", datetime(2020, 1, 1, 3, 0)])
-    DatetimeIndex(['2020-01-01 01:00:00-01:00', '2020-01-01 02:00:00-01:00'],
-                  dtype='datetime64[ns, UTC-01:00]', freq=None)
+    Index([2020-01-01 01:00:00-01:00, 2020-01-01 03:00:00], dtype='object')
 
     |
 

pandas/tests/tools/test_to_datetime.py

@@ -132,8 +132,8 @@ def test_to_datetime_format_YYYYMMDD_with_nat(self, cache):
         # string with NaT
         ser2 = ser.apply(str)
         ser2[2] = "nat"
-        result = to_datetime(ser2, format="%Y%m%d", cache=cache)
-        tm.assert_series_equal(result, expected)
+        with pytest.raises(ValueError, match="unconverted data remains: .0"):
+            to_datetime(ser2, format="%Y%m%d", cache=cache)
 
     def test_to_datetime_format_YYYYMM_with_nat(self, cache):
         # https://github.com/pandas-dev/pandas/issues/50237
@@ -146,13 +146,25 @@ def test_to_datetime_format_YYYYMM_with_nat(self, cache):
         result = to_datetime(ser, format="%Y%m", cache=cache)
         tm.assert_series_equal(result, expected)
 
+    def test_to_datetime_format_YYYYMM_with_nat_2(self, cache):
+        # is this different from above? can remove?
+        # https://github.com/pandas-dev/pandas/issues/50237
+        ser = Series([198012, 198012] + [198101] * 5)
+        expected = Series(
+            [Timestamp("19801201"), Timestamp("19801201")] + [Timestamp("19810101")] * 5
+        )
+        expected[2] = np.nan
+        ser[2] = np.nan
+        result = to_datetime(ser, format="%Y%m", cache=cache)
+        tm.assert_series_equal(result, expected)
+
     def test_to_datetime_format_YYYYMMDD_ignore(self, cache):
         # coercion
         # GH 7930
         ser = Series([20121231, 20141231, 99991231])
         result = to_datetime(ser, format="%Y%m%d", errors="ignore", cache=cache)
         expected = Series(
-            [datetime(2012, 12, 31), datetime(2014, 12, 31), datetime(9999, 12, 31)],
+            [20121231, 20141231, 99991231],
             dtype=object,
         )
         tm.assert_series_equal(result, expected)
@@ -1737,8 +1749,8 @@ def test_dataframe_coerce(self, cache):
         df2 = DataFrame({"year": [2015, 2016], "month": [2, 20], "day": [4, 5]})
 
         msg = (
-            "cannot assemble the datetimes: time data .+ does not "
-            r"match format '%Y%m%d' \(match\)"
+            r"cannot assemble the datetimes: time data '.*' does not match "
+            r"format '%Y%m%d' \(match\)"
         )
         with pytest.raises(ValueError, match=msg):
             to_datetime(df2, cache=cache)
@@ -1873,10 +1885,7 @@ def test_to_datetime_iso8601_fails(self, input, format, exact):
         # `format` is longer than the string, so this fails regardless of `exact`
         with pytest.raises(
             ValueError,
-            match=(
-                rf"time data \"{input}\" at position 0 doesn't match format "
-                rf"\"{format}\""
-            ),
+            match=(rf"time data '{input}' does not match format '{format}'"),
         ):
             to_datetime(input, format=format, exact=exact)
 
@@ -1895,10 +1904,7 @@ def test_to_datetime_iso8601_exact_fails(self, input, format):
         # `format` is shorter than the date string, so only fails with `exact=True`
         with pytest.raises(
             ValueError,
-            match=(
-                rf"time data \"{input}\" at position 0 doesn't match format "
-                rf"\"{format}\""
-            ),
+            match=("unconverted data remains: |does not match format"),
         ):
             to_datetime(input, format=format)
 
@@ -1934,10 +1940,7 @@ def test_to_datetime_iso8601_separator(self, input, format):
         # https://github.com/pandas-dev/pandas/issues/12649
         with pytest.raises(
             ValueError,
-            match=(
-                rf"time data \"{input}\" at position 0 doesn\'t match format "
-                rf"\"{format}\""
-            ),
+            match=(rf"time data \'{input}\' does not match format '{format}'"),
         ):
             to_datetime(input, format=format)
 
@@ -1963,6 +1966,28 @@ def test_to_datetime_iso8601_valid(self, input, format):
         result = to_datetime(input, format=format)
         assert result == expected
 
+    @pytest.mark.parametrize(
+        "input, format",
+        [
+            ("2020-1", "%Y-%m"),
+            ("2020-1-1", "%Y-%m-%d"),
+            ("2020-1-1 0", "%Y-%m-%d %H"),
+            ("2020-1-1T0", "%Y-%m-%dT%H"),
+            ("2020-1-1 0:0", "%Y-%m-%d %H:%M"),
+            ("2020-1-1T0:0", "%Y-%m-%dT%H:%M"),
+            ("2020-1-1 0:0:0", "%Y-%m-%d %H:%M:%S"),
+            ("2020-1-1T0:0:0", "%Y-%m-%dT%H:%M:%S"),
+            ("2020-1-1T0:0:0.000", "%Y-%m-%dT%H:%M:%S.%f"),
+            ("2020-1-1T0:0:0.000000", "%Y-%m-%dT%H:%M:%S.%f"),
+            ("2020-1-1T0:0:0.000000000", "%Y-%m-%dT%H:%M:%S.%f"),
+        ],
+    )
+    def test_to_datetime_iso8601_non_padded(self, input, format):
+        # https://github.com/pandas-dev/pandas/issues/21422
+        expected = Timestamp(2020, 1, 1)
+        result = to_datetime(input, format=format)
+        assert result == expected
+
     @pytest.mark.parametrize(
         "input, format",
         [
@@ -2465,7 +2490,7 @@ def test_day_not_in_month_raise(self, cache):
 
     @pytest.mark.parametrize("arg", ["2015-02-29", "2015-02-32", "2015-04-31"])
     def test_day_not_in_month_raise_value(self, cache, arg):
-        msg = f'time data "{arg}" at position 0 doesn\'t match format "%Y-%m-%d"'
+        msg = "day is out of range for month|unconverted data remains"
         with pytest.raises(ValueError, match=msg):
             to_datetime(arg, errors="raise", format="%Y-%m-%d", cache=cache)
 
@@ -2859,10 +2884,7 @@ def test_incorrect_value_exception(self):
     )
     def test_to_datetime_out_of_bounds_with_format_arg(self, format, warning):
         # see gh-23830
-        msg = (
-            "Out of bounds nanosecond timestamp: 2417-10-27 00:00:00 "
-            "present at position 0"
-        )
+        msg = "Out of bounds nanosecond timestamp: 2417-10-27 00:00:00"
         with pytest.raises(OutOfBoundsDatetime, match=msg):
             with tm.assert_produces_warning(warning, match="Could not infer format"):
                 to_datetime("2417-10-27 00:00:00", format=format)