Skip to content

BUG: DTI/TDI .insert accepting incorrectly-dtyped NaT #30754

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 6, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions pandas/core/indexes/datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from pandas.core.dtypes.common import _NS_DTYPE, is_float, is_integer, is_scalar
from pandas.core.dtypes.dtypes import DatetimeTZDtype
from pandas.core.dtypes.missing import isna
from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna

from pandas.core.accessor import delegate_names
from pandas.core.arrays.datetimes import (
Expand Down Expand Up @@ -922,9 +922,14 @@ def insert(self, loc, item):
-------
new_index : Index
"""
if is_scalar(item) and isna(item):
if is_valid_nat_for_dtype(item, self.dtype):
# GH 18295
item = self._na_value
elif is_scalar(item) and isna(item):
# i.e. timedeltat64("NaT")
raise TypeError(
f"cannot insert {type(self).__name__} with incompatible label"
)

freq = None

Expand Down
17 changes: 9 additions & 8 deletions pandas/core/indexes/timedeltas.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
is_timedelta64_ns_dtype,
pandas_dtype,
)
from pandas.core.dtypes.missing import isna
from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna

from pandas.core.accessor import delegate_names
from pandas.core.arrays import datetimelike as dtl
Expand Down Expand Up @@ -397,15 +397,16 @@ def insert(self, loc, item):
new_index : Index
"""
# try to convert if possible
if _is_convertible_to_td(item):
try:
item = Timedelta(item)
except ValueError:
# e.g. str that can't be parsed to timedelta
pass
elif is_scalar(item) and isna(item):
if isinstance(item, self._data._recognized_scalars):
item = Timedelta(item)
elif is_valid_nat_for_dtype(item, self.dtype):
# GH 18295
item = self._na_value
elif is_scalar(item) and isna(item):
# i.e. datetime64("NaT")
raise TypeError(
f"cannot insert {type(self).__name__} with incompatible label"
)

freq = None
if isinstance(item, Timedelta) or (is_scalar(item) and isna(item)):
Expand Down
10 changes: 9 additions & 1 deletion pandas/tests/indexes/datetimes/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,9 @@ def test_take_fill_value_with_timezone(self):


class TestDatetimeIndex:
@pytest.mark.parametrize("null", [None, np.nan, pd.NaT])
@pytest.mark.parametrize(
"null", [None, np.nan, np.datetime64("NaT"), pd.NaT, pd.NA]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should create a fixture like this (similar to nulls_fixture), but this is more datetime specialized, but for another pass.

)
@pytest.mark.parametrize("tz", [None, "UTC", "US/Eastern"])
def test_insert_nat(self, tz, null):
# GH#16537, GH#18295 (test missing)
Expand All @@ -326,6 +328,12 @@ def test_insert_nat(self, tz, null):
res = idx.insert(0, null)
tm.assert_index_equal(res, expected)

@pytest.mark.parametrize("tz", [None, "UTC", "US/Eastern"])
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

try to fixturize at some point (the tzs)

def test_insert_invalid_na(self, tz):
idx = pd.DatetimeIndex(["2017-01-01"], tz=tz)
with pytest.raises(TypeError, match="incompatible label"):
idx.insert(0, np.timedelta64("NaT"))

def test_insert(self):
idx = DatetimeIndex(["2000-01-04", "2000-01-01", "2000-01-02"], name="idx")

Expand Down
24 changes: 21 additions & 3 deletions pandas/tests/indexes/timedeltas/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,11 +219,29 @@ def test_insert(self):
assert result.name == expected.name
assert result.freq == expected.freq

@pytest.mark.parametrize(
"null", [None, np.nan, np.timedelta64("NaT"), pd.NaT, pd.NA]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

similar to above

)
def test_insert_nat(self, null):
# GH 18295 (test missing)
idx = timedelta_range("1day", "3day")
result = idx.insert(1, null)
expected = TimedeltaIndex(["1day", pd.NaT, "2day", "3day"])
for na in (np.nan, pd.NaT, None):
result = timedelta_range("1day", "3day").insert(1, na)
tm.assert_index_equal(result, expected)
tm.assert_index_equal(result, expected)

def test_insert_invalid_na(self):
idx = TimedeltaIndex(["4day", "1day", "2day"], name="idx")
with pytest.raises(TypeError, match="incompatible label"):
idx.insert(0, np.datetime64("NaT"))

def test_insert_dont_cast_strings(self):
# To match DatetimeIndex and PeriodIndex behavior, dont try to
# parse strings to Timedelta
idx = timedelta_range("1day", "3day")

result = idx.insert(0, "1 Day")
assert result.dtype == object
assert result[0] == "1 Day"

def test_delete(self):
idx = timedelta_range(start="1 Days", periods=5, freq="D", name="idx")
Expand Down