CLN: use idiomatic pandas_dtypes in pandas/dtypes/common.py (#24541)

Author: jreback

asv_bench/benchmarks/dtypes.py

@@ -0,0 +1,39 @@
+from pandas.api.types import pandas_dtype
+
+import numpy as np
+from .pandas_vb_common import (
+    numeric_dtypes, datetime_dtypes, string_dtypes, extension_dtypes)
+
+
+_numpy_dtypes = [np.dtype(dtype)
+                 for dtype in (numeric_dtypes +
+                               datetime_dtypes +
+                               string_dtypes)]
+_dtypes = _numpy_dtypes + extension_dtypes
+
+
+class Dtypes(object):
+    params = (_dtypes +
+              list(map(lambda dt: dt.name, _dtypes)))
+    param_names = ['dtype']
+
+    def time_pandas_dtype(self, dtype):
+        pandas_dtype(dtype)
+
+
+class DtypesInvalid(object):
+    param_names = ['dtype']
+    params = ['scalar-string', 'scalar-int', 'list-string', 'array-string']
+    data_dict = {'scalar-string': 'foo',
+                 'scalar-int': 1,
+                 'list-string': ['foo'] * 1000,
+                 'array-string': np.array(['foo'] * 1000)}
+
+    def time_pandas_dtype_invalid(self, dtype):
+        try:
+            pandas_dtype(self.data_dict[dtype])
+        except TypeError:
+            pass
+
+
+from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/pandas_vb_common.py

@@ -2,6 +2,7 @@
 from importlib import import_module
 
 import numpy as np
+import pandas as pd
 
 # Compatibility import for lib
 for imp in ['pandas._libs.lib', 'pandas.lib']:
@@ -14,6 +15,15 @@
 numeric_dtypes = [np.int64, np.int32, np.uint32, np.uint64, np.float32,
                   np.float64, np.int16, np.int8, np.uint16, np.uint8]
 datetime_dtypes = [np.datetime64, np.timedelta64]
+string_dtypes = [np.object]
+extension_dtypes = [pd.Int8Dtype, pd.Int16Dtype,
+                    pd.Int32Dtype, pd.Int64Dtype,
+                    pd.UInt8Dtype, pd.UInt16Dtype,
+                    pd.UInt32Dtype, pd.UInt64Dtype,
+                    pd.CategoricalDtype,
+                    pd.IntervalDtype,
+                    pd.DatetimeTZDtype('ns', 'UTC'),
+                    pd.PeriodDtype('D')]
 
 
 def setup(*args, **kwargs):

doc/source/whatsnew/v0.24.0.rst

@@ -430,7 +430,7 @@ Backwards incompatible API changes
 - The column order of the resultant :class:`DataFrame` from :meth:`MultiIndex.to_frame` is now guaranteed to match the :attr:`MultiIndex.names` order. (:issue:`22420`)
 - Incorrectly passing a :class:`DatetimeIndex` to :meth:`MultiIndex.from_tuples`, rather than a sequence of tuples, now raises a ``TypeError`` rather than a ``ValueError`` (:issue:`24024`)
 - :func:`pd.offsets.generate_range` argument ``time_rule`` has been removed; use ``offset`` instead (:issue:`24157`)
-- In 0.23.x, pandas would raise a ``ValueError`` on a merge of a numeric column (e.g. ``int`` dtyped column) and an ``object`` dtyped column (:issue:`9780`). We have re-enabled the ability to merge ``object`` and other dtypes (:issue:`21681`)
+- In 0.23.x, pandas would raise a ``ValueError`` on a merge of a numeric column (e.g. ``int`` dtyped column) and an ``object`` dtyped column (:issue:`9780`). We have re-enabled the ability to merge ``object`` and other dtypes; pandas will still raise on a merge between a numeric and an ``object`` dtyped column that is composed only of strings (:issue:`21681`)
 
 Percentage change on groupby
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^

pandas/conftest.py

@@ -388,9 +388,14 @@ def tz_aware_fixture(request):
     return request.param
 
 
+# ----------------------------------------------------------------
+# Dtypes
 UNSIGNED_INT_DTYPES = ["uint8", "uint16", "uint32", "uint64"]
+UNSIGNED_EA_INT_DTYPES = ["UInt8", "UInt16", "UInt32", "UInt64"]
 SIGNED_INT_DTYPES = [int, "int8", "int16", "int32", "int64"]
+SIGNED_EA_INT_DTYPES = ["Int8", "Int16", "Int32", "Int64"]
 ALL_INT_DTYPES = UNSIGNED_INT_DTYPES + SIGNED_INT_DTYPES
+ALL_EA_INT_DTYPES = UNSIGNED_EA_INT_DTYPES + SIGNED_EA_INT_DTYPES
 
 FLOAT_DTYPES = [float, "float32", "float64"]
 COMPLEX_DTYPES = [complex, "complex64", "complex128"]

pandas/core/arrays/integer.py

@@ -32,6 +32,7 @@ class _IntegerDtype(ExtensionDtype):
     The attributes name & type are set when these subclasses are created.
     """
     name = None
+    base = None
     type = None
     na_value = np.nan
 
@@ -153,6 +154,7 @@ def coerce_to_array(values, dtype, mask=None, copy=False):
             # Avoid DeprecationWarning from NumPy about np.dtype("Int64")
             # https://github.com/numpy/numpy/pull/7476
             dtype = dtype.lower()
+
         if not issubclass(type(dtype), _IntegerDtype):
             try:
                 dtype = _dtypes[str(np.dtype(dtype))]
@@ -655,7 +657,8 @@ def integer_arithmetic_method(self, other):
     else:
         name = dtype.capitalize()
     classname = "{}Dtype".format(name)
-    attributes_dict = {'type': getattr(np, dtype),
+    numpy_dtype = getattr(np, dtype)
+    attributes_dict = {'type': numpy_dtype,
                        'name': name}
     dtype_type = register_extension_dtype(
         type(classname, (_IntegerDtype, ), attributes_dict)

pandas/core/dtypes/cast.py

@@ -9,9 +9,9 @@
 from pandas.compat import PY3, string_types, text_type, to_str
 
 from .common import (
-    _INT64_DTYPE, _NS_DTYPE, _POSSIBLY_CAST_DTYPES, _TD_DTYPE, _string_dtypes,
-    ensure_int8, ensure_int16, ensure_int32, ensure_int64, ensure_object,
-    is_bool, is_bool_dtype, is_categorical_dtype, is_complex, is_complex_dtype,
+    _INT64_DTYPE, _NS_DTYPE, _POSSIBLY_CAST_DTYPES, _TD_DTYPE, ensure_int8,
+    ensure_int16, ensure_int32, ensure_int64, ensure_object, is_bool,
+    is_bool_dtype, is_categorical_dtype, is_complex, is_complex_dtype,
     is_datetime64_dtype, is_datetime64_ns_dtype, is_datetime64tz_dtype,
     is_datetime_or_timedelta_dtype, is_datetimelike, is_dtype_equal,
     is_extension_array_dtype, is_extension_type, is_float, is_float_dtype,
@@ -544,7 +544,7 @@ def invalidate_string_dtypes(dtype_set):
     """Change string like dtypes to object for
     ``DataFrame.select_dtypes()``.
     """
-    non_string_dtypes = dtype_set - _string_dtypes
+    non_string_dtypes = dtype_set - {np.dtype('S').type, np.dtype('<U').type}
     if non_string_dtypes != dtype_set:
         raise TypeError("string dtypes are not allowed, use 'object' instead")