Skip to content

ENH: support any/all for pyarrow numeric and duration dtypes #50717

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jan 24, 2023
Merged

ENH: support any/all for pyarrow numeric and duration dtypes #50717

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c35f0e5
ENH: support any/all for pyarrow numeric and duration dtypes
jbrockmendel Jan 12, 2023
ce6b444
mypy fixup
jbrockmendel Jan 12, 2023
94c191f
Merge branch 'main' into enh-ea-reductions
jbrockmendel Jan 19, 2023
3fcffd5
use pc.not_equal
jbrockmendel Jan 19, 2023
399c167
Merge branch 'main' into enh-ea-reductions
jbrockmendel Jan 20, 2023
4b040ab
use suggested pattern
jbrockmendel Jan 20, 2023
2308f84
fix not_eq
jbrockmendel Jan 21, 2023
4d97849
Merge branch 'main' into enh-ea-reductions
jbrockmendel Jan 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion pandas/core/arrays/arrow/array.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@

import numpy as np

from pandas._libs import lib
from pandas._libs import (
Timedelta,
lib,
)
from pandas._typing import (
ArrayLike,
AxisInt,
Expand Down Expand Up @@ -1008,6 +1011,29 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
------
TypeError : subclass does not define reductions
"""
pa_type = self._data.type

if name in ["any", "all"] and (
pa.types.is_integer(pa_type)
or pa.types.is_floating(pa_type)
or pa.types.is_duration(pa_type)
):
# pyarrow only supports any/all for boolean dtype, we allow
# for other dtypes, matching our non-pyarrow behavior

zero: int | Timedelta
if pa.types.is_duration(pa_type):
zero = Timedelta(0)
else:
zero = 0

not_eq = pc.not_equal(self._data, zero)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah sorry, I had meant

if name in ...:
    ...
    data = pc.not_equal(self._data, zero)
else:
    data = self._data
....
try:
     result = pyarrow_method(data...)

result = not_eq._reduce(name, skipna=skipna, **kwargs)
if isinstance(result, np.bool_):
# need to rule out pd.NA
result = bool(result)
return result

if name == "sem":

def pyarrow_meth(data, skip_nulls, **kwargs):
Expand Down
16 changes: 15 additions & 1 deletion pandas/tests/extension/test_arrow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -566,10 +566,24 @@ def test_reduce_series(
f"pyarrow={pa.__version__} for {pa_dtype}"
),
)
if not pa.types.is_boolean(pa_dtype):
if pa.types.is_string(pa_dtype) or pa.types.is_binary(pa_dtype):
# We *might* want to make this behave like the non-pyarrow cases,
# but have not yet decided.
request.node.add_marker(xfail_mark)

op_name = all_boolean_reductions
ser = pd.Series(data)

if pa.types.is_temporal(pa_dtype) and not pa.types.is_duration(pa_dtype):
# xref GH#34479 we support this in our non-pyarrow datetime64 dtypes,
# but it isn't obvious we _should_. For now, we keep the pyarrow
# behavior which does not support this.

with pytest.raises(TypeError, match="does not support reduction"):
getattr(ser, op_name)(skipna=skipna)

return

result = getattr(ser, op_name)(skipna=skipna)
assert result is (op_name == "any")

Expand Down