Skip to content

fix: use public pandas APIs where possible #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jan 26, 2022
Merged

fix: use public pandas APIs where possible #60

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4f96f36
refactor: use public pandas APIs where possible
tswast Jan 24, 2022
eff42ed
no need to override take
tswast Jan 25, 2022
fa81f71
Merge remote-tracking branch 'upstream/main' into issue28-use-public-…
tswast Jan 25, 2022
60da4d0
backport take implementation
tswast Jan 25, 2022
66d5b66
move remaining private pandas methods to backports
tswast Jan 25, 2022
a4e89c2
add note about _validate_scalar to docstring
tswast Jan 25, 2022
5de6407
comment why we can't use public mixin
tswast Jan 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions db_dtypes/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,7 @@
import numpy
import packaging.version
import pandas
import pandas.compat.numpy.function
import pandas.core.algorithms
import pandas.core.arrays
import pandas.core.dtypes.base
import pandas.core.dtypes.dtypes
import pandas.core.dtypes.generic
import pandas.core.nanops
import pandas.api.extensions
import pyarrow
import pyarrow.compute

Expand All @@ -44,7 +38,7 @@
pandas_release = packaging.version.parse(pandas.__version__).release


@pandas.core.dtypes.dtypes.register_extension_dtype
@pandas.api.extensions.register_extension_dtype
class TimeDtype(core.BaseDatetimeDtype):
"""
Extension dtype for time data.
Expand Down Expand Up @@ -113,7 +107,7 @@ def _datetime(
.as_py()
)

if scalar is None:
if pandas.isna(scalar):
return None
if isinstance(scalar, datetime.time):
return pandas.Timestamp(
Expand Down Expand Up @@ -194,7 +188,7 @@ def __arrow_array__(self, type=None):
)


@pandas.core.dtypes.dtypes.register_extension_dtype
@pandas.api.extensions.register_extension_dtype
class DateDtype(core.BaseDatetimeDtype):
"""
Extension dtype for time data.
Expand Down Expand Up @@ -238,7 +232,7 @@ def _datetime(
if isinstance(scalar, (pyarrow.Date32Scalar, pyarrow.Date64Scalar)):
scalar = scalar.as_py()

if scalar is None:
if pandas.isna(scalar):
return None
elif isinstance(scalar, datetime.date):
return pandas.Timestamp(
Expand Down
46 changes: 5 additions & 41 deletions db_dtypes/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Any, Optional, Sequence
from typing import Optional

import numpy
import pandas
from pandas._libs import NaT
from pandas import NaT
import pandas.api.extensions
import pandas.compat.numpy.function
import pandas.core.algorithms
import pandas.core.arrays
import pandas.core.dtypes.base
from pandas.core.dtypes.common import is_dtype_equal, is_list_like, pandas_dtype
import pandas.core.dtypes.dtypes
import pandas.core.dtypes.generic
import pandas.compat.numpy.function # TODO: move to pandas_backports
from pandas.api.types import is_dtype_equal, is_list_like, pandas_dtype
import pandas.core.nanops

from db_dtypes import pandas_backports
Expand Down Expand Up @@ -107,38 +102,7 @@ def isna(self):
return pandas.isna(self._ndarray)

def _validate_scalar(self, value):
if pandas.isna(value):
return None

if not isinstance(value, self.dtype.type):
raise ValueError(value)

return value

def take(
self,
indices: Sequence[int],
*,
allow_fill: bool = False,
fill_value: Any = None,
):
indices = numpy.asarray(indices, dtype=numpy.intp)
data = self._ndarray
if allow_fill:
fill_value = self._validate_scalar(fill_value)
fill_value = (
numpy.datetime64() if fill_value is None else self._datetime(fill_value)
)
if (indices < -1).any():
raise ValueError(
"take called with negative indexes other than -1,"
" when a fill value is provided."
)
out = data.take(indices)
if allow_fill:
out[indices == -1] = fill_value

return self.__class__(out)
return self._datetime(value)

# TODO: provide implementations of dropna, fillna, unique,
# factorize, argsort, searchsoeted for better performance over
Expand Down