From 1843a045e72d8e63e3bb84c862d35c0a35858d32 Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Mon, 6 May 2019 13:34:57 -0700 Subject: [PATCH 1/8] BUG-20629 allow .at accessor with CategoricalIndex --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/core/frame.py | 16 +++++++++++----- pandas/core/indexes/category.py | 2 ++ pandas/tests/indexing/test_categorical.py | 8 ++++++++ 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index a65b92737c220..1a0fca3fa0ec2 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -262,7 +262,7 @@ Bug Fixes Categorical ^^^^^^^^^^^ -- +- Bug in :func:`DataFrame.at` and :func:`Series.at` that would raise exception if the index was a :class:`CategoricalIndex` (:issue:`20629`) - - diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 09f5726e6341f..856614b9bc2f3 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -2710,13 +2710,19 @@ def _get_value(self, index, col, takeable=False): try: return engine.get_value(series._values, index) + except KeyError as e: + if self.index.nlevels > 1: + raise e # partial indexing forbidden + else: + pass # GH 20629 except (TypeError, ValueError): + pass - # we cannot handle direct indexing - # use positional - col = self.columns.get_loc(col) - index = self.index.get_loc(index) - return self._get_value(index, col, takeable=True) + # we cannot handle direct indexing + # use positional + col = self.columns.get_loc(col) + index = self.index.get_loc(index) + return self._get_value(index, col, takeable=True) _get_value.__doc__ = get_value.__doc__ def set_value(self, index, col, value, takeable=False): diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py index 473686a7541a5..0405adebf02c0 100644 --- a/pandas/core/indexes/category.py +++ b/pandas/core/indexes/category.py @@ -498,6 +498,8 @@ def get_value(self, series, key): k = self._convert_scalar_indexer(k, kind='getitem') indexer = self.get_loc(k) return series.iloc[indexer] + except AttributeError: + return super().get_value(series, indexer) except (KeyError, TypeError): pass diff --git a/pandas/tests/indexing/test_categorical.py b/pandas/tests/indexing/test_categorical.py index 36a5f803237a2..fda850222833b 100644 --- a/pandas/tests/indexing/test_categorical.py +++ b/pandas/tests/indexing/test_categorical.py @@ -719,3 +719,11 @@ def test_map_with_dict_or_series(self): output = cur_index.map(mapper) # Order of categories in output can be different tm.assert_index_equal(expected, output) + + def test_at_with_categorical_index(self): + # GH 20629 + s = Series([1, 2, 3], index=pd.CategoricalIndex(["A", "B", "C"])) + assert s.at['A'] == 1 + df = DataFrame([[1, 2], [3, 4], [5, 6]], + index=pd.CategoricalIndex(["A", "B", "C"])) + assert df.at['B', 1] == 4 From 13717db39f177e6f5a5c79d14c3aa3574f691f85 Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Mon, 6 May 2019 22:27:09 -0700 Subject: [PATCH 2/8] make requested changes --- pandas/core/frame.py | 8 ++++---- pandas/tests/indexing/test_categorical.py | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 856614b9bc2f3..a3d67af9008d4 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -2710,11 +2710,11 @@ def _get_value(self, index, col, takeable=False): try: return engine.get_value(series._values, index) - except KeyError as e: + except KeyError: + # GH 20629 if self.index.nlevels > 1: - raise e # partial indexing forbidden - else: - pass # GH 20629 + # partial indexing forbidden + raise except (TypeError, ValueError): pass diff --git a/pandas/tests/indexing/test_categorical.py b/pandas/tests/indexing/test_categorical.py index fda850222833b..1ec89af42a1e1 100644 --- a/pandas/tests/indexing/test_categorical.py +++ b/pandas/tests/indexing/test_categorical.py @@ -638,6 +638,16 @@ def test_loc_slice(self): # expected = df.iloc[[1,2,3,4]] # assert_frame_equal(result, expected) + def test_loc_and_at_with_categorical_index(self): + # GH 20629 + s = Series([1, 2, 3], index=pd.CategoricalIndex(["A", "B", "C"])) + assert s.loc['A'] == 1 + assert s.at['A'] == 1 + df = DataFrame([[1, 2], [3, 4], [5, 6]], + index=pd.CategoricalIndex(["A", "B", "C"])) + assert df.loc['B', 1] == 4 + assert df.at['B', 1] == 4 + def test_boolean_selection(self): df3 = self.df3 @@ -719,11 +729,3 @@ def test_map_with_dict_or_series(self): output = cur_index.map(mapper) # Order of categories in output can be different tm.assert_index_equal(expected, output) - - def test_at_with_categorical_index(self): - # GH 20629 - s = Series([1, 2, 3], index=pd.CategoricalIndex(["A", "B", "C"])) - assert s.at['A'] == 1 - df = DataFrame([[1, 2], [3, 4], [5, 6]], - index=pd.CategoricalIndex(["A", "B", "C"])) - assert df.at['B', 1] == 4 From 188ac73eaf2e7ef923c18217680113033375e2cd Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Sun, 12 May 2019 16:46:27 -0700 Subject: [PATCH 3/8] add type info, use take instead of iloc --- pandas/core/indexes/category.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py index 0405adebf02c0..b1a79a542b696 100644 --- a/pandas/core/indexes/category.py +++ b/pandas/core/indexes/category.py @@ -20,12 +20,15 @@ from pandas.core import accessor from pandas.core.algorithms import take_1d from pandas.core.arrays.categorical import Categorical, contains +from pandas.core.arrays.base import ExtensionArray import pandas.core.common as com import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs import pandas.core.missing as missing from pandas.core.ops import get_op_result_name +from typing import Any, Union + _index_doc_kwargs = dict(ibase._index_doc_kwargs) _index_doc_kwargs.update(dict(target_klass='CategoricalIndex')) @@ -488,18 +491,31 @@ def get_loc(self, key, method=None): except KeyError: raise KeyError(key) - def get_value(self, series, key): + def get_value(self, + series: Union[ABCSeries, ExtensionArray, Index, np.ndarray], + key: Any): """ Fast lookup of value from 1-dimensional ndarray. Only use this if you know what you're doing + + Parameters + ---------- + series : Series, ExtensionArray, Index, or ndarray + 1-dimensional array to take values from + key: : scalar + The value of this index at the position of the desired value, + otherwise the positional index of the desired value + + Returns + ------- + Any + The element of the series at the position indicated by the key """ try: k = com.values_from_object(key) k = self._convert_scalar_indexer(k, kind='getitem') indexer = self.get_loc(k) - return series.iloc[indexer] - except AttributeError: - return super().get_value(series, indexer) + return series.take([indexer])[0] except (KeyError, TypeError): pass From 41bf8543526fd2ebb36d1b77a3887ef82a4304eb Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Sun, 12 May 2019 17:33:48 -0700 Subject: [PATCH 4/8] fix import order --- pandas/core/indexes/category.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py index b1a79a542b696..805ed449bae65 100644 --- a/pandas/core/indexes/category.py +++ b/pandas/core/indexes/category.py @@ -1,4 +1,5 @@ import operator +from typing import Any, Union import warnings import numpy as np @@ -19,16 +20,14 @@ from pandas.core import accessor from pandas.core.algorithms import take_1d -from pandas.core.arrays.categorical import Categorical, contains from pandas.core.arrays.base import ExtensionArray +from pandas.core.arrays.categorical import Categorical, contains import pandas.core.common as com import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs import pandas.core.missing as missing from pandas.core.ops import get_op_result_name -from typing import Any, Union - _index_doc_kwargs = dict(ibase._index_doc_kwargs) _index_doc_kwargs.update(dict(target_klass='CategoricalIndex')) From cb0d4d902c453556a313a5126e04dc425d5cb590 Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Sun, 12 May 2019 17:33:48 -0700 Subject: [PATCH 5/8] fix import order --- pandas/core/indexes/category.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py index b1a79a542b696..805ed449bae65 100644 --- a/pandas/core/indexes/category.py +++ b/pandas/core/indexes/category.py @@ -1,4 +1,5 @@ import operator +from typing import Any, Union import warnings import numpy as np @@ -19,16 +20,14 @@ from pandas.core import accessor from pandas.core.algorithms import take_1d -from pandas.core.arrays.categorical import Categorical, contains from pandas.core.arrays.base import ExtensionArray +from pandas.core.arrays.categorical import Categorical, contains import pandas.core.common as com import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs import pandas.core.missing as missing from pandas.core.ops import get_op_result_name -from typing import Any, Union - _index_doc_kwargs = dict(ibase._index_doc_kwargs) _index_doc_kwargs.update(dict(target_klass='CategoricalIndex')) From 52990a89fac958bf6de68f95a27c2c90191ef01f Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Wed, 15 May 2019 22:02:05 -0700 Subject: [PATCH 6/8] add AnyArrayLike to pandas._typing --- pandas/_typing.py | 12 +++++++++++- pandas/core/indexes/category.py | 6 +++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index 3959e38e6f08c..aafa91191710b 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -8,8 +8,18 @@ from pandas._libs.tslibs.timedeltas import Timedelta from pandas.core.dtypes.dtypes import ExtensionDtype -from pandas.core.dtypes.generic import ABCExtensionArray +from pandas.core.dtypes.generic import (ABCExtensionArray, + ABCIndexClass, + ABCSeries, + ABCSparseArray, + ABCSparseSeries) +AnyArrayLike = Union[ABCExtensionArray, + ABCIndexClass, + ABCSeries, + ABCSparseArray, + ABCSparseSeries, + np.ndarray] ArrayLike = Union[ABCExtensionArray, np.ndarray] DatetimeLikeScalar = Type[Union[Period, Timestamp, Timedelta]] Dtype = Union[str, np.dtype, ExtensionDtype] diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py index 805ed449bae65..866ff3ac07f80 100644 --- a/pandas/core/indexes/category.py +++ b/pandas/core/indexes/category.py @@ -1,5 +1,5 @@ import operator -from typing import Any, Union +from typing import Any import warnings import numpy as np @@ -7,6 +7,7 @@ from pandas._config import get_option from pandas._libs import index as libindex +from pandas._typing import AnyArrayLike import pandas.compat as compat from pandas.compat.numpy import function as nv from pandas.util._decorators import Appender, cache_readonly @@ -20,7 +21,6 @@ from pandas.core import accessor from pandas.core.algorithms import take_1d -from pandas.core.arrays.base import ExtensionArray from pandas.core.arrays.categorical import Categorical, contains import pandas.core.common as com import pandas.core.indexes.base as ibase @@ -491,7 +491,7 @@ def get_loc(self, key, method=None): raise KeyError(key) def get_value(self, - series: Union[ABCSeries, ExtensionArray, Index, np.ndarray], + series: AnyArrayLike, key: Any): """ Fast lookup of value from 1-dimensional ndarray. Only use this if you From 14d016f4310b1953a3e27c09c416cb34325bd441 Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Wed, 15 May 2019 22:04:17 -0700 Subject: [PATCH 7/8] remove unnecessary type --- pandas/_typing.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index aafa91191710b..c70dd58fb2537 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -11,13 +11,11 @@ from pandas.core.dtypes.generic import (ABCExtensionArray, ABCIndexClass, ABCSeries, - ABCSparseArray, ABCSparseSeries) AnyArrayLike = Union[ABCExtensionArray, ABCIndexClass, ABCSeries, - ABCSparseArray, ABCSparseSeries, np.ndarray] ArrayLike = Union[ABCExtensionArray, np.ndarray] From 2831d3d4991e669206ffdb857b610bc3d41bba32 Mon Sep 17 00:00:00 2001 From: JustinZhengBC Date: Wed, 15 May 2019 22:38:53 -0700 Subject: [PATCH 8/8] fix import order --- pandas/_typing.py | 6 ++---- pandas/core/indexes/category.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index c70dd58fb2537..f5bf0dcd3e220 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -8,10 +8,8 @@ from pandas._libs.tslibs.timedeltas import Timedelta from pandas.core.dtypes.dtypes import ExtensionDtype -from pandas.core.dtypes.generic import (ABCExtensionArray, - ABCIndexClass, - ABCSeries, - ABCSparseSeries) +from pandas.core.dtypes.generic import ( + ABCExtensionArray, ABCIndexClass, ABCSeries, ABCSparseSeries) AnyArrayLike = Union[ABCExtensionArray, ABCIndexClass, diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py index 866ff3ac07f80..4fa2706c5140d 100644 --- a/pandas/core/indexes/category.py +++ b/pandas/core/indexes/category.py @@ -7,7 +7,6 @@ from pandas._config import get_option from pandas._libs import index as libindex -from pandas._typing import AnyArrayLike import pandas.compat as compat from pandas.compat.numpy import function as nv from pandas.util._decorators import Appender, cache_readonly @@ -19,6 +18,7 @@ from pandas.core.dtypes.generic import ABCCategorical, ABCSeries from pandas.core.dtypes.missing import isna +from pandas._typing import AnyArrayLike from pandas.core import accessor from pandas.core.algorithms import take_1d from pandas.core.arrays.categorical import Categorical, contains