Merge branch 'master' into cleanup/matplotlib-style

Author: ivanovmg

.pre-commit-config.yaml

@@ -62,6 +62,11 @@ repos:
             |math|module|note|raw|seealso|toctree|versionadded
             |versionchanged|warning):[^:]
         files: \.(py|pyx|rst)$
+    -   id: incorrect-code-directives
+        name: Check for incorrect code block or IPython directives
+        language: pygrep
+        entry: (\.\. code-block ::|\.\. ipython ::)
+        files: \.(py|pyx|rst)$
 -   repo: https://github.com/asottile/yesqa
     rev: v1.2.2
     hooks:

ci/code_checks.sh

@@ -207,18 +207,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -r -E --include '*.py' '(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)' pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Check for wrong space after code-block directive and before colon (".. code-block ::" instead of ".. code-block::")' ; echo $MSG
-    invgrep -R --include="*.rst" ".. code-block ::" doc/source
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for wrong space after ipython directive and before colon (".. ipython ::" instead of ".. ipython::")' ; echo $MSG
-    invgrep -R --include="*.rst" ".. ipython ::" doc/source
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for extra blank lines after the class definition' ; echo $MSG
-    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' .
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     MSG='Check for use of {foo!r} instead of {repr(foo)}' ; echo $MSG
     invgrep -R --include=*.{py,pyx} '!r}' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
@@ -243,12 +231,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include=*.{py,pyx} '\.__class__' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG
-    INVGREP_APPEND=" <- trailing whitespaces found"
-    invgrep -RI --exclude=\*.{svg,c,cpp,html,js} --exclude-dir=env "\s$" *
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-    unset INVGREP_APPEND
-
     MSG='Check code for instances of os.remove' ; echo $MSG
     invgrep -R --include="*.py*" --exclude "common.py" --exclude "test_writers.py" --exclude "test_store.py" -E "os\.remove" pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"

doc/source/ecosystem.rst

@@ -230,7 +230,7 @@ allows users to view, manipulate and edit pandas ``Index``, ``Series``,
 and ``DataFrame`` objects like a "spreadsheet", including copying and modifying
 values, sorting, displaying a "heatmap", converting data types and more.
 pandas objects can also be renamed, duplicated, new columns added,
-copyed/pasted to/from the clipboard (as TSV), and saved/loaded to/from a file.
+copied/pasted to/from the clipboard (as TSV), and saved/loaded to/from a file.
 Spyder can also import data from a variety of plain text and binary files
 or the clipboard into a new pandas DataFrame via a sophisticated import wizard.
 
@@ -376,6 +376,23 @@ Dask-ML enables parallel and distributed machine learning using Dask alongside e
 
 Koalas provides a familiar pandas DataFrame interface on top of Apache Spark. It enables users to leverage multi-cores on one machine or a cluster of machines to speed up or scale their DataFrame code.
 
+`Modin <https://github.com/modin-project/modin>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``modin.pandas`` DataFrame is a parallel and distributed drop-in replacement
+for pandas. This means that you can use Modin with existing pandas code or write
+new code with the existing pandas API. Modin can leverage your entire machine or
+cluster to speed up and scale your pandas workloads, including traditionally
+time-consuming tasks like ingesting data (``read_csv``, ``read_excel``,
+``read_parquet``, etc.).
+
+.. code:: python
+
+    # import pandas as pd
+    import modin.pandas as pd
+
+    df = pd.read_csv("big.csv")  # use all your cores!
+
 `Odo <http://odo.pydata.org>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -400,16 +417,6 @@ If also displays progress bars.
     # df.apply(func)
     df.parallel_apply(func)
 
-`Ray <https://ray.readthedocs.io/en/latest/pandas_on_ray.html>`__
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-pandas on Ray is an early stage DataFrame library that wraps pandas and transparently distributes the data and computation. The user does not need to know how many cores their system has, nor do they need to specify how to distribute the data. In fact, users can continue using their previous pandas notebooks while experiencing a considerable speedup from pandas on Ray, even on a single machine. Only a modification of the import statement is needed, as we demonstrate below. Once you’ve changed your import statement, you’re ready to use pandas on Ray just like you would pandas.
-
-.. code:: python
-
-    # import pandas as pd
-    import ray.dataframe as pd
-
 
 `Vaex <https://docs.vaex.io/>`__
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/whatsnew/v0.16.2.rst

@@ -147,7 +147,7 @@ Bug fixes
 - Bug in ``setitem`` where type promotion is applied to the entire block (:issue:`10280`)
 - Bug in ``Series`` arithmetic methods may incorrectly hold names (:issue:`10068`)
 - Bug in ``GroupBy.get_group`` when grouping on multiple keys, one of which is categorical. (:issue:`10132`)
-- Bug in ``DatetimeIndex`` and ``TimedeltaIndex`` names are lost after timedelta arithmetics ( :issue:`9926`)
+- Bug in ``DatetimeIndex`` and ``TimedeltaIndex`` names are lost after timedelta arithmetic ( :issue:`9926`)
 - Bug in ``DataFrame`` construction from nested ``dict`` with ``datetime64`` (:issue:`10160`)
 - Bug in ``Series`` construction from ``dict`` with ``datetime64`` keys (:issue:`9456`)
 - Bug in ``Series.plot(label="LABEL")`` not correctly setting the label (:issue:`10119`)

doc/source/whatsnew/v0.24.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0241:
 
-Whats new in 0.24.1 (February 3, 2019)
---------------------------------------
+What's new in 0.24.1 (February 3, 2019)
+---------------------------------------
 
 .. warning::
 

doc/source/whatsnew/v0.24.2.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0242:
 
-Whats new in 0.24.2 (March 12, 2019)
-------------------------------------
+What's new in 0.24.2 (March 12, 2019)
+-------------------------------------
 
 .. warning::
 

doc/source/whatsnew/v1.1.4.rst

@@ -20,6 +20,7 @@ Fixed regressions
 - Fixed regression in :class:`RollingGroupby` with ``sort=False`` not being respected (:issue:`36889`)
 - Fixed regression in :meth:`Series.astype` converting ``None`` to ``"nan"`` when casting to string (:issue:`36904`)
 - Fixed regression in :class:`RollingGroupby` causing a segmentation fault with Index of dtype object (:issue:`36727`)
+- Fixed regression in :meth:`DataFrame.resample(...).apply(...)` raised ``AttributeError`` when input was a :class:`DataFrame` and only a :class:`Series` was evaluated (:issue:`36951`)
 
 .. ---------------------------------------------------------------------------
 
@@ -30,6 +31,7 @@ Bug fixes
 - Bug causing ``groupby(...).sum()`` and similar to not preserve metadata (:issue:`29442`)
 - Bug in :meth:`Series.isin` and :meth:`DataFrame.isin` raising a ``ValueError`` when the target was read-only (:issue:`37174`)
 - Bug in :meth:`GroupBy.fillna` that introduced a performance regression after 1.0.5 (:issue:`36757`)
+- Bug in :meth:`DataFrame.info` was raising a ``KeyError`` when the DataFrame has integer column names (:issue:`37245`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.2.0.rst

@@ -180,7 +180,7 @@ Alternatively, you can also use the dtype object:
 .. warning::
 
    Experimental: the new floating data types are currently experimental, and its
-   behaviour or API may still change without warning. Expecially the behaviour
+   behaviour or API may still change without warning. Especially the behaviour
    regarding NaN (distinct from NA missing values) is subject to change.
 
 .. _whatsnew_120.index_name_preservation:
@@ -523,6 +523,7 @@ Other
 
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` incorrectly raising ``AssertionError`` instead of ``ValueError`` when invalid parameter combinations are passed (:issue:`36045`)
 - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` with numeric values and string ``to_replace`` (:issue:`34789`)
+- Fixed bug in metadata propagation incorrectly copying DataFrame columns as metadata when the column name overlaps with the metadata name (:issue:`37037`)
 - Fixed metadata propagation in the :class:`Series.dt` and :class:`Series.str` accessors (:issue:`28283`)
 - Bug in :meth:`Index.union` behaving differently depending on whether operand is a :class:`Index` or other list-like (:issue:`36384`)
 - Passing an array with 2 or more dimensions to the :class:`Series` constructor now raises the more specific ``ValueError``, from a bare ``Exception`` previously (:issue:`35744`)

pandas/conftest.py

@@ -361,6 +361,19 @@ def multiindex_year_month_day_dataframe_random_data():
     return ymd
 
 
+@pytest.fixture
+def multiindex_dataframe_random_data():
+    """DataFrame with 2 level MultiIndex with random data"""
+    index = MultiIndex(
+        levels=[["foo", "bar", "baz", "qux"], ["one", "two", "three"]],
+        codes=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3], [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
+        names=["first", "second"],
+    )
+    return DataFrame(
+        np.random.randn(10, 3), index=index, columns=Index(["A", "B", "C"], name="exp")
+    )
+
+
 def _create_multiindex():
     """
     MultiIndex used to test the general functionality of this object

pandas/core/arrays/base.py

@@ -507,7 +507,12 @@ def _values_for_argsort(self) -> np.ndarray:
         return np.array(self)
 
     def argsort(
-        self, ascending: bool = True, kind: str = "quicksort", *args, **kwargs
+        self,
+        ascending: bool = True,
+        kind: str = "quicksort",
+        na_position: str = "last",
+        *args,
+        **kwargs,
     ) -> np.ndarray:
         """
         Return the indices that would sort this array.
@@ -538,7 +543,14 @@ def argsort(
         # 2. argsort : total control over sorting.
         ascending = nv.validate_argsort_with_ascending(ascending, args, kwargs)
 
-        result = nargsort(self, kind=kind, ascending=ascending, na_position="last")
+        values = self._values_for_argsort()
+        result = nargsort(
+            values,
+            kind=kind,
+            ascending=ascending,
+            na_position=na_position,
+            mask=np.asarray(self.isna()),
+        )
         return result
 
     def argmin(self):

pandas/core/generic.py

@@ -5364,7 +5364,7 @@ def __finalize__(
 
             self.flags.allows_duplicate_labels = other.flags.allows_duplicate_labels
             # For subclasses using _metadata.
-            for name in self._metadata:
+            for name in set(self._metadata) & set(other._metadata):
                 assert isinstance(name, str)
                 object.__setattr__(self, name, getattr(other, name, None))
 

pandas/core/indexes/base.py

@@ -4994,7 +4994,7 @@ def isin(self, values, level=None):
             self._validate_index_level(level)
         return algos.isin(self, values)
 
-    def _get_string_slice(self, key: str_t, use_lhs: bool = True, use_rhs: bool = True):
+    def _get_string_slice(self, key: str_t):
         # this is for partial string indexing,
         # overridden in DatetimeIndex, TimedeltaIndex and PeriodIndex
         raise NotImplementedError

pandas/core/indexes/datetimelike.py

@@ -398,16 +398,12 @@ def _partial_date_slice(
         self,
         reso: Resolution,
         parsed: datetime,
-        use_lhs: bool = True,
-        use_rhs: bool = True,
     ):
         """
         Parameters
         ----------
         reso : Resolution
         parsed : datetime
-        use_lhs : bool, default True
-        use_rhs : bool, default True
 
         Returns
         -------
@@ -422,8 +418,7 @@ def _partial_date_slice(
         if self.is_monotonic:
 
             if len(self) and (
-                (use_lhs and t1 < self[0] and t2 < self[0])
-                or (use_rhs and t1 > self[-1] and t2 > self[-1])
+                (t1 < self[0] and t2 < self[0]) or (t1 > self[-1] and t2 > self[-1])
             ):
                 # we are out of range
                 raise KeyError
@@ -432,13 +427,13 @@ def _partial_date_slice(
 
             # a monotonic (sorted) series can be sliced
             # Use asi8.searchsorted to avoid re-validating Periods/Timestamps
-            left = i8vals.searchsorted(unbox(t1), side="left") if use_lhs else None
-            right = i8vals.searchsorted(unbox(t2), side="right") if use_rhs else None
+            left = i8vals.searchsorted(unbox(t1), side="left")
+            right = i8vals.searchsorted(unbox(t2), side="right")
             return slice(left, right)
 
         else:
-            lhs_mask = (i8vals >= unbox(t1)) if use_lhs else True
-            rhs_mask = (i8vals <= unbox(t2)) if use_rhs else True
+            lhs_mask = i8vals >= unbox(t1)
+            rhs_mask = i8vals <= unbox(t2)
 
             # try to find the dates
             return (lhs_mask & rhs_mask).nonzero()[0]

pandas/core/indexes/datetimes.py

@@ -729,11 +729,11 @@ def _maybe_cast_slice_bound(self, label, side: str, kind):
             self._deprecate_mismatched_indexing(label)
         return self._maybe_cast_for_get_loc(label)
 
-    def _get_string_slice(self, key: str, use_lhs: bool = True, use_rhs: bool = True):
+    def _get_string_slice(self, key: str):
         freq = getattr(self, "freqstr", getattr(self, "inferred_freq", None))
         parsed, reso = parsing.parse_time_string(key, freq)
         reso = Resolution.from_attrname(reso)
-        loc = self._partial_date_slice(reso, parsed, use_lhs=use_lhs, use_rhs=use_rhs)
+        loc = self._partial_date_slice(reso, parsed)
         return loc
 
     def slice_indexer(self, start=None, end=None, step=None, kind=None):

pandas/core/indexes/period.py

@@ -622,12 +622,11 @@ def _validate_partial_date_slice(self, reso: Resolution):
             #  why is that check not needed?
             raise ValueError
 
-    def _get_string_slice(self, key: str, use_lhs: bool = True, use_rhs: bool = True):
-        # TODO: Check for non-True use_lhs/use_rhs
+    def _get_string_slice(self, key: str):
         parsed, reso = parse_time_string(key, self.freq)
         reso = Resolution.from_attrname(reso)
         try:
-            return self._partial_date_slice(reso, parsed, use_lhs, use_rhs)
+            return self._partial_date_slice(reso, parsed)
         except KeyError as err:
             raise KeyError(key) from err
 

pandas/core/internals/concat.py

@@ -217,9 +217,7 @@ def is_na(self) -> bool:
         # a block is NOT null, chunks should help in such cases.  1000 value
         # was chosen rather arbitrarily.
         values = self.block.values
-        if self.block.is_categorical:
-            values_flat = values.categories
-        elif is_sparse(self.block.values.dtype):
+        if is_sparse(self.block.values.dtype):
             return False
         elif self.block.is_extension:
             # TODO(EA2D): no need for special case with 2D EAs

pandas/core/ops/__init__.py

@@ -14,7 +14,7 @@
 from pandas.util._decorators import Appender
 
 from pandas.core.dtypes.common import is_array_like, is_list_like
-from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import algorithms
@@ -25,7 +25,10 @@
     get_array_op,
     logical_op,
 )
-from pandas.core.ops.common import unpack_zerodim_and_defer  # noqa:F401
+from pandas.core.ops.common import (  # noqa:F401
+    get_op_result_name,
+    unpack_zerodim_and_defer,
+)
 from pandas.core.ops.docstrings import (
     _flex_comp_doc_FRAME,
     _op_descriptions,
@@ -76,67 +79,6 @@
 
 COMPARISON_BINOPS: Set[str] = {"eq", "ne", "lt", "gt", "le", "ge"}
 
-# -----------------------------------------------------------------------------
-# Ops Wrapping Utilities
-
-
-def get_op_result_name(left, right):
-    """
-    Find the appropriate name to pin to an operation result.  This result
-    should always be either an Index or a Series.
-
-    Parameters
-    ----------
-    left : {Series, Index}
-    right : object
-
-    Returns
-    -------
-    name : object
-        Usually a string
-    """
-    # `left` is always a Series when called from within ops
-    if isinstance(right, (ABCSeries, ABCIndexClass)):
-        name = _maybe_match_name(left, right)
-    else:
-        name = left.name
-    return name
-
-
-def _maybe_match_name(a, b):
-    """
-    Try to find a name to attach to the result of an operation between
-    a and b.  If only one of these has a `name` attribute, return that
-    name.  Otherwise return a consensus name if they match of None if
-    they have different names.
-
-    Parameters
-    ----------
-    a : object
-    b : object
-
-    Returns
-    -------
-    name : str or None
-
-    See Also
-    --------
-    pandas.core.common.consensus_name_attr
-    """
-    a_has = hasattr(a, "name")
-    b_has = hasattr(b, "name")
-    if a_has and b_has:
-        if a.name == b.name:
-            return a.name
-        else:
-            # TODO: what if they both have np.nan for their names?
-            return None
-    elif a_has:
-        return a.name
-    elif b_has:
-        return b.name
-    return None
-
 
 # -----------------------------------------------------------------------------
 # Masking NA values and fallbacks for operations numpy does not support