Skip to content

Fix StringArray.astype for category dtype #40450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Apr 2, 2021
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pandas/core/arrays/string_.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
is_array_like,
is_bool_dtype,
is_dtype_equal,
is_extension_array_dtype,
is_integer_dtype,
is_object_dtype,
is_string_dtype,
Expand Down Expand Up @@ -307,7 +308,6 @@ def __setitem__(self, key, value):

def astype(self, dtype, copy=True):
dtype = pandas_dtype(dtype)

if is_dtype_equal(dtype, self.dtype):
if copy:
return self.copy()
Expand All @@ -327,6 +327,9 @@ def astype(self, dtype, copy=True):
arr[mask] = "0"
values = arr.astype(dtype.numpy_dtype)
return FloatingArray(values, mask, copy=False)
elif is_extension_array_dtype(dtype):
cls = dtype.construct_array_type()
return cls._from_sequence(self, dtype=dtype, copy=copy)
elif np.issubdtype(dtype, np.floating):
arr = self._ndarray.copy()
mask = self.isna()
Expand Down
45 changes: 45 additions & 0 deletions pandas/tests/series/methods/test_astype.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,39 @@ def test_astype_bytes(self):
assert result.dtypes == np.dtype("S3")


class TestAstypeString:
@pytest.mark.parametrize(
"data, dtype",
[
(["A", NA], "category"),
(["2020-10-10", "2020-10-10"], "datetime64[ns]"),
(["2020-10-10", "2020-10-10", NaT], "datetime64[ns]"),
(
["2012-01-01 00:00:00-05:00", NaT],
"datetime64[ns, US/Eastern]",
),
([1, None], "UInt16"),
(["1/1/2021", "2/1/2021"], "period[M]"),
(["1/1/2021", "2/1/2021", NaT], "period[M]"),
(["1 Day", "59 Days", NaT], "timedelta64[ns]"),
# currently no way to parse BooleanArray, IntervalArray from a
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BooleanArray can be parsed from string (see _from_sequence_of_strings, the general method)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had an implementation that used _from_sequence_of_strings instead of _from_sequence to in StringArray.astype(). That required bigger code changes. I'd like to merge this Regression PR and then implement to implement _from_sequence_of_strings as part of #40566

# list of strings
],
)
def test_astype_string_to_extension_dtype_roundtrip(self, data, dtype, request):
if dtype in ("timedelta64[ns]"):
mark = pytest.mark.xfail(reason="TODO fix is_extension_array_dtype GH40478")
request.node.add_marker(mark)
if NaT in data and dtype in ("period[M]", "datetime64[ns]"):
mark = pytest.mark.xfail(
reason="TODO StringArray.astype() None to dtype.na_value conversion"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there an issue for this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I created one here: #40566 to track this

)
request.node.add_marker(mark)
# GH-40351
s = Series(data, dtype=dtype)
tm.assert_series_equal(s, s.astype("string").astype(dtype))


class TestAstypeCategorical:
def test_astype_categorical_to_other(self):
cat = Categorical([f"{i} - {i + 499}" for i in range(0, 10000, 500)])
Expand Down Expand Up @@ -470,6 +503,18 @@ def test_astype_categories_raises(self):
with pytest.raises(TypeError, match="got an unexpected"):
s.astype("category", categories=["a", "b"], ordered=True)

def test_astype_str_to_extension_dtype(self):
# GH-40351
s = Series(["A", np.NaN], dtype="string")
result = s.astype("category")
expected = Series(["A", np.NaN], dtype="category")
tm.assert_series_equal(result, expected)

s = Series(["1/1/2021", "2/1/2021"], dtype="string")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can u add an example for Timedelta, Datetime w/time zone and Interval (all the EA types)

Copy link
Contributor Author

@siboehm siboehm Mar 17, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added test for all ExtensionArray dtypes. BooleanArray and IntervalArray I had to exclude since there's no way to to parse them back from a list of strings. TimedeltaArray xfails due to #40478. For PeriodArrayand DatetimeArray the NaT get converted to NA strings. But converting the NA strings back to NaT fails. I added XFails, unless expecting EA ⇒ StringArray ⇒ EA to roundtrip successfully is still up for debate.

result = s.astype("period[M]")
expected = Series(["1/1/2021", "2/1/2021"], dtype="period[M]")
tm.assert_series_equal(result, expected)

@pytest.mark.parametrize("items", [["a", "b", "c", "a"], [1, 2, 3, 1]])
def test_astype_from_categorical(self, items):
ser = Series(items)
Expand Down