Merge branch 'master' into add-compression-stata-reading

Author: jreback

asv_bench/benchmarks/series_methods.py

@@ -284,16 +284,29 @@ def time_clip(self, n):
 
 class ValueCounts:
 
-    params = ["int", "uint", "float", "object"]
-    param_names = ["dtype"]
+    params = [[10 ** 3, 10 ** 4, 10 ** 5], ["int", "uint", "float", "object"]]
+    param_names = ["N", "dtype"]
 
-    def setup(self, dtype):
-        self.s = Series(np.random.randint(0, 1000, size=100000)).astype(dtype)
+    def setup(self, N, dtype):
+        self.s = Series(np.random.randint(0, N, size=10 * N)).astype(dtype)
 
-    def time_value_counts(self, dtype):
+    def time_value_counts(self, N, dtype):
         self.s.value_counts()
 
 
+class Mode:
+
+    params = [[10 ** 3, 10 ** 4, 10 ** 5], ["int", "uint", "float", "object"]]
+    param_names = ["N", "dtype"]
+
+    def setup(self, N, dtype):
+        np.random.seed(42)
+        self.s = Series(np.random.randint(0, N, size=10 * N)).astype(dtype)
+
+    def time_mode(self, N, dtype):
+        self.s.mode()
+
+
 class Dir:
     def setup(self):
         self.s = Series(index=tm.makeStringIndex(10000))

doc/source/user_guide/visualization.rst

@@ -552,6 +552,9 @@ These can be specified by the ``x`` and ``y`` keywords.
 .. ipython:: python
 
    df = pd.DataFrame(np.random.rand(50, 4), columns=["a", "b", "c", "d"])
+   df["species"] = pd.Categorical(
+       ["setosa"] * 20 + ["versicolor"] * 20 + ["virginica"] * 10
+   )
 
    @savefig scatter_plot.png
    df.plot.scatter(x="a", y="b");
@@ -579,6 +582,21 @@ each point:
    df.plot.scatter(x="a", y="b", c="c", s=50);
 
 
+.. ipython:: python
+   :suppress:
+
+   plt.close("all")
+
+If a categorical column is passed to ``c``, then a discrete colorbar will be produced:
+
+.. versionadded:: 1.3.0
+
+.. ipython:: python
+
+   @savefig scatter_plot_categorical.png
+   df.plot.scatter(x="a", y="b", c="species", cmap="viridis", s=50);
+
+
 .. ipython:: python
    :suppress:
 

doc/source/whatsnew/v1.3.0.rst

@@ -52,8 +52,10 @@ Other enhancements
 - :meth:`DataFrame.apply` can now accept NumPy unary operators as strings, e.g. ``df.apply("sqrt")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
 - :meth:`DataFrame.apply` can now accept non-callable DataFrame properties as strings, e.g. ``df.apply("size")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
 - :meth:`Series.apply` can now accept list-like or dictionary-like arguments that aren't lists or dictionaries, e.g. ``ser.apply(np.array(["sum", "mean"]))``, which was already the case for :meth:`DataFrame.apply` (:issue:`39140`)
+- :meth:`DataFrame.plot.scatter` can now accept a categorical column as the argument to ``c`` (:issue:`12380`, :issue:`31357`)
 - :meth:`.Styler.set_tooltips` allows on hover tooltips to be added to styled HTML dataframes.
-- :meth:`pandas.read_stata` and :class:`StataReader` support reading data from compressed files.
+\- :meth:`pandas.read_stata` and :class:`StataReader` support reading data from compressed files.
+- :meth:`Series.loc.__getitem__` and :meth:`Series.loc.__setitem__` with :class:`MultiIndex` now raising helpful error message when indexer has too many dimensions (:issue:`35349`)
 
 .. ---------------------------------------------------------------------------
 
@@ -326,6 +328,7 @@ I/O
 - Bug in :func:`to_hdf` raising ``KeyError`` when trying to apply for subclasses of ``DataFrame`` or ``Series`` (:issue:`33748`)
 - Bug in :meth:`~HDFStore.put` raising a wrong ``TypeError`` when saving a DataFrame with non-string dtype (:issue:`34274`)
 - Bug in :func:`json_normalize` resulting in the first element of a generator object not being included in the returned ``DataFrame`` (:issue:`35923`)
+- Bug in :func:`read_csv` apllying thousands separator to date columns when column should be parsed for dates and ``usecols`` is specified for ``engine="python"`` (:issue:`39365`)
 - Bug in :func:`read_excel` forward filling :class:`MultiIndex` names with multiple header and index columns specified (:issue:`34673`)
 - :func:`read_excel` now respects :func:`set_option` (:issue:`34252`)
 - Bug in :func:`read_csv` not switching ``true_values`` and ``false_values`` for nullable ``boolean`` dtype (:issue:`34655`)
@@ -364,7 +367,7 @@ Reshaping
 - Bug in :func:`join` over :class:`MultiIndex` returned wrong result, when one of both indexes had only one level (:issue:`36909`)
 - :meth:`merge_asof` raises ``ValueError`` instead of cryptic ``TypeError`` in case of non-numerical merge columns (:issue:`29130`)
 - Bug in :meth:`DataFrame.join` not assigning values correctly when having :class:`MultiIndex` where at least one dimension is from dtype ``Categorical`` with non-alphabetically sorted categories (:issue:`38502`)
-- :meth:`Series.value_counts` returns keys in original order (:issue:`12679`, :issue:`11227`)
+- :meth:`Series.value_counts` and :meth:`Series.mode` return consistent keys in original order (:issue:`12679`, :issue:`11227` and :issue:`39007`)
 - Bug in :meth:`DataFrame.apply` would give incorrect results when used with a string argument and ``axis=1`` when the axis argument was not supported and now raises a ``ValueError`` instead (:issue:`39211`)
 -
 

pandas/_libs/hashtable_func_helper.pxi.in

@@ -28,52 +28,6 @@ dtypes = [('Complex128', 'complex128', 'complex128',
 {{for name, dtype, ttype, c_type, to_c_type in dtypes}}
 
 
-@cython.wraparound(False)
-@cython.boundscheck(False)
-{{if dtype == 'object'}}
-cdef build_count_table_{{dtype}}(ndarray[{{dtype}}] values,
-                                 kh_{{ttype}}_t *table, bint dropna):
-{{else}}
-cdef build_count_table_{{dtype}}(const {{dtype}}_t[:] values,
-                                 kh_{{ttype}}_t *table, bint dropna):
-{{endif}}
-    cdef:
-        khiter_t k
-        Py_ssize_t i, n = len(values)
-
-        {{c_type}} val
-
-        int ret = 0
-
-    {{if dtype == 'object'}}
-    kh_resize_{{ttype}}(table, n // 10)
-
-    for i in range(n):
-        val = values[i]
-        if not checknull(val) or not dropna:
-            k = kh_get_{{ttype}}(table, <PyObject*>val)
-            if k != table.n_buckets:
-                table.vals[k] += 1
-            else:
-                k = kh_put_{{ttype}}(table, <PyObject*>val, &ret)
-                table.vals[k] = 1
-    {{else}}
-    with nogil:
-        kh_resize_{{ttype}}(table, n)
-
-        for i in range(n):
-            val = {{to_c_type}}(values[i])
-
-            if not is_nan_{{c_type}}(val) or not dropna:
-                k = kh_get_{{ttype}}(table, val)
-                if k != table.n_buckets:
-                    table.vals[k] += 1
-                else:
-                    k = kh_put_{{ttype}}(table, val, &ret)
-                    table.vals[k] = 1
-    {{endif}}
-
-
 @cython.wraparound(False)
 @cython.boundscheck(False)
 {{if dtype == 'object'}}
@@ -84,8 +38,6 @@ cpdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna):
     cdef:
         Py_ssize_t i = 0
         Py_ssize_t n = len(values)
-        size_t unique_key_index = 0
-        size_t unique_key_count = 0
         kh_{{ttype}}_t *table
 
         # Don't use Py_ssize_t, since table.n_buckets is unsigned
@@ -98,12 +50,10 @@ cpdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna):
 
     # we track the order in which keys are first seen (GH39009),
     # khash-map isn't insertion-ordered, thus:
-    #    table maps key to index_of_appearence
-    #    result_keys maps index_of_appearence to key
-    #    result_counts maps index_of_appearence to number of elements
+    #    table maps keys to counts
+    #    result_keys remembers the original order of keys
 
     result_keys = {{name}}Vector()
-    result_counts = Int64Vector()
     table = kh_init_{{ttype}}()
 
     {{if dtype == 'object'}}
@@ -118,14 +68,11 @@ cpdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna):
                 val = navalue
             k = kh_get_{{ttype}}(table, <PyObject*>val)
             if k != table.n_buckets:
-                unique_key_index = table.vals[k]
-                result_counts.data.data[unique_key_index] += 1
+                table.vals[k] += 1
             else:
                 k = kh_put_{{ttype}}(table, <PyObject*>val, &ret)
-                table.vals[k] = unique_key_count
+                table.vals[k] = 1
                 result_keys.append(val)
-                result_counts.append(1)
-                unique_key_count+=1
     {{else}}
     kh_resize_{{ttype}}(table, n)
 
@@ -135,19 +82,26 @@ cpdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna):
         if not is_nan_{{c_type}}(val) or not dropna:
             k = kh_get_{{ttype}}(table, val)
             if k != table.n_buckets:
-                unique_key_index = table.vals[k]
-                result_counts.data.data[unique_key_index] += 1
+                table.vals[k] += 1
             else:
                 k = kh_put_{{ttype}}(table, val, &ret)
-                table.vals[k] = unique_key_count
+                table.vals[k] = 1
                 result_keys.append(val)
-                result_counts.append(1)
-                unique_key_count+=1
     {{endif}}
 
+    # collect counts in the order corresponding to result_keys:
+    cdef int64_t[:] result_counts = np.empty(table.size, dtype=np.int64)
+    for i in range(table.size):
+        {{if dtype == 'object'}}
+        k = kh_get_{{ttype}}(table, result_keys.data[i])
+        {{else}}
+        k = kh_get_{{ttype}}(table, result_keys.data.data[i])
+        {{endif}}
+        result_counts[i] = table.vals[k]
+
     kh_destroy_{{ttype}}(table)
 
-    return result_keys.to_array(), result_counts.to_array()
+    return result_keys.to_array(), result_counts.base
 
 
 @cython.wraparound(False)
@@ -294,78 +248,42 @@ def ismember_{{dtype}}(const {{dtype}}_t[:] arr, const {{dtype}}_t[:] values):
     kh_destroy_{{ttype}}(table)
     return result.view(np.bool_)
 
-{{endfor}}
-
-
 # ----------------------------------------------------------------------
 # Mode Computations
 # ----------------------------------------------------------------------
 
-{{py:
-
-# dtype, ctype, table_type, npy_dtype
-dtypes = [('complex128', 'khcomplex128_t', 'complex128', 'complex128'),
-          ('complex64', 'khcomplex64_t', 'complex64', 'complex64'),
-          ('float64', 'float64_t', 'float64', 'float64'),
-          ('float32', 'float32_t', 'float32', 'float32'),
-          ('int64', 'int64_t', 'int64', 'int64'),
-          ('int32', 'int32_t', 'int32', 'int32'),
-          ('int16', 'int16_t', 'int16', 'int16'),
-          ('int8', 'int8_t', 'int8', 'int8'),
-          ('uint64', 'uint64_t', 'uint64', 'uint64'),
-          ('uint32', 'uint32_t', 'uint32', 'uint32'),
-          ('uint16', 'uint16_t', 'uint16', 'uint16'),
-          ('uint8', 'uint8_t', 'uint8', 'uint8'),
-          ('object', 'object', 'pymap', 'object_')]
-}}
-
-{{for dtype, ctype, table_type, npy_dtype in dtypes}}
-
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-
 {{if dtype == 'object'}}
-
-
-def mode_{{dtype}}(ndarray[{{ctype}}] values, bint dropna):
+def mode_{{dtype}}(ndarray[{{dtype}}] values, bint dropna):
 {{else}}
-
-
 def mode_{{dtype}}(const {{dtype}}_t[:] values, bint dropna):
 {{endif}}
     cdef:
-        int count, max_count = 1
-        int j = -1  # so you can do +=
-        # Don't use Py_ssize_t, since table.n_buckets is unsigned
-        khiter_t k
-        kh_{{table_type}}_t *table
-        ndarray[{{ctype}}] modes
+        {{if dtype == 'object'}}
+        ndarray[{{dtype}}] keys
+        ndarray[{{dtype}}] modes
+        {{else}}
+        {{dtype}}_t[:] keys
+        ndarray[{{dtype}}_t] modes
+        {{endif}}
+        int64_t[:] counts
+        int64_t count, max_count = -1
+        Py_ssize_t k, j = 0
 
-    table = kh_init_{{table_type}}()
-    build_count_table_{{dtype}}(values, table, dropna)
+    keys, counts = value_count_{{dtype}}(values, dropna)
 
-    modes = np.empty(table.n_buckets, dtype=np.{{npy_dtype}})
+    {{if dtype == 'object'}}
+    modes = np.empty(len(keys), dtype=np.object_)
+    {{else}}
+    modes = np.empty(len(keys), dtype=np.{{dtype}})
+    {{endif}}
 
     {{if dtype != 'object'}}
     with nogil:
-        for k in range(table.n_buckets):
-            if kh_exist_{{table_type}}(table, k):
-                count = table.vals[k]
-                if count == max_count:
-                    j += 1
-                elif count > max_count:
-                    max_count = count
-                    j = 0
-                else:
-                    continue
-
-                modes[j] = table.keys[k]
-    {{else}}
-    for k in range(table.n_buckets):
-        if kh_exist_{{table_type}}(table, k):
-            count = table.vals[k]
-
+        for k in range(len(keys)):
+            count = counts[k]
             if count == max_count:
                 j += 1
             elif count > max_count:
@@ -374,11 +292,21 @@ def mode_{{dtype}}(const {{dtype}}_t[:] values, bint dropna):
             else:
                 continue
 
-            modes[j] = <object>table.keys[k]
+            modes[j] = keys[k]
+    {{else}}
+    for k in range(len(keys)):
+        count = counts[k]
+        if count == max_count:
+            j += 1
+        elif count > max_count:
+            max_count = count
+            j = 0
+        else:
+            continue
+
+        modes[j] = keys[k]
     {{endif}}
 
-    kh_destroy_{{table_type}}(table)
-
     return modes[:j + 1]
 
 {{endfor}}

pandas/core/dtypes/cast.py

@@ -1952,4 +1952,7 @@ def can_hold_element(dtype: np.dtype, element: Any) -> bool:
             return tipo.kind == "b"
         return lib.is_bool(element)
 
+    elif dtype == object:
+        return True
+
     raise NotImplementedError(dtype)

pandas/core/dtypes/common.py

@@ -1677,7 +1677,7 @@ def _is_dtype_type(arr_or_dtype, condition) -> bool:
     return condition(tipo)
 
 
-def infer_dtype_from_object(dtype):
+def infer_dtype_from_object(dtype) -> DtypeObj:
     """
     Get a numpy dtype.type-style object for a dtype object.