Merge remote-tracking branch 'upstream/master'

Author: r-toroxel

.travis.yml

@@ -58,12 +58,6 @@ matrix:
       services:
         - mysql
         - postgresql
-
-    - env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
-      services:
-        - mysql
-        - postgresql
   allow_failures:
     - arch: arm64
       env:

asv_bench/benchmarks/rolling.py

@@ -91,11 +91,18 @@ class EWMMethods:
     def setup(self, constructor, window, dtype, method):
         N = 10 ** 5
         arr = (100 * np.random.random(N)).astype(dtype)
+        times = pd.date_range("1900", periods=N, freq="23s")
         self.ewm = getattr(pd, constructor)(arr).ewm(halflife=window)
+        self.ewm_times = getattr(pd, constructor)(arr).ewm(
+            halflife="1 Day", times=times
+        )
 
     def time_ewm(self, constructor, window, dtype, method):
         getattr(self.ewm, method)()
 
+    def time_ewm_times(self, constructor, window, dtype, method):
+        self.ewm.mean()
+
 
 class VariableWindowMethods(Methods):
     params = (

asv_bench/benchmarks/tslibs/tz_convert.py

@@ -0,0 +1,30 @@
+import numpy as np
+from pytz import UTC
+
+from pandas._libs.tslibs.tzconversion import tz_convert, tz_localize_to_utc
+
+from .tslib import _sizes, _tzs
+
+
+class TimeTZConvert:
+    params = (
+        _sizes,
+        [x for x in _tzs if x is not None],
+    )
+    param_names = ["size", "tz"]
+
+    def setup(self, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_tz_convert_from_utc(self, size, tz):
+        # effectively:
+        #  dti = DatetimeIndex(self.i8data, tz=tz)
+        #  dti.tz_localize(None)
+        tz_convert(self.i8data, UTC, tz)
+
+    def time_tz_localize_to_utc(self, size, tz):
+        # effectively:
+        #  dti = DatetimeIndex(self.i8data)
+        #  dti.tz_localize(tz, ambiguous="NaT", nonexistent="NaT")
+        tz_localize_to_utc(self.i8data, tz, ambiguous="NaT", nonexistent="NaT")

ci/azure/posix.yml

@@ -30,6 +30,11 @@ jobs:
           LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans"
 
+        py36_slow:
+          ENV_FILE: ci/deps/azure-36-slow.yaml
+          CONDA_PY: "36"
+          PATTERN: "slow"
+
         py36_locale:
           ENV_FILE: ci/deps/azure-36-locale.yaml
           CONDA_PY: "36"

ci/deps/travis-36-slow.yaml renamed to ci/deps/azure-36-slow.yaml

@@ -166,5 +166,4 @@ if [[ -n ${SQL:0} ]]; then
 else
    echo "not using dbs on non-linux Travis builds or Azure Pipelines"
 fi
-
 echo "done"

ci/setup_env.sh

@@ -1095,6 +1095,25 @@ and **alpha** to the EW functions:
   one half.
 * **Alpha** specifies the smoothing factor directly.
 
+.. versionadded:: 1.1.0
+
+You can also specify ``halflife`` in terms of a timedelta convertible unit to specify the amount of
+time it takes for an observation to decay to half its value when also specifying a sequence
+of ``times``.
+
+.. ipython:: python
+
+    df = pd.DataFrame({'B': [0, 1, 2, np.nan, 4]})
+    df
+    times = ['2020-01-01', '2020-01-03', '2020-01-10', '2020-01-15', '2020-01-17']
+    df.ewm(halflife='4 days', times=pd.DatetimeIndex(times)).mean()
+
+The following formula is used to compute exponentially weighted mean with an input vector of times:
+
+.. math::
+
+    y_t = \frac{\sum_{i=0}^t 0.5^\frac{t_{t} - t_{i}}{\lambda} x_{t-i}}{0.5^\frac{t_{t} - t_{i}}{\lambda}},
+
 Here is an example for a univariate time series:
 
 .. ipython:: python

doc/source/user_guide/computation.rst

@@ -329,6 +329,7 @@ Other enhancements
 - :meth:`DataFrame.to_excel` can now also write OpenOffice spreadsheet (.ods) files (:issue:`27222`)
 - :meth:`~Series.explode` now accepts ``ignore_index`` to reset the index, similarly to :meth:`pd.concat` or :meth:`DataFrame.sort_values` (:issue:`34932`).
 - :meth:`read_csv` now accepts string values like "0", "0.0", "1", "1.0" as convertible to the nullable boolean dtype (:issue:`34859`)
+- :class:`pandas.core.window.ExponentialMovingWindow` now supports a ``times`` argument that allows ``mean`` to be calculated with observations spaced by the timestamps in ``times`` (:issue:`34839`)
 
 .. ---------------------------------------------------------------------------
 
@@ -788,6 +789,7 @@ Deprecations
 - :meth:`Categorical.to_dense` is deprecated and will be removed in a future version, use ``np.asarray(cat)`` instead (:issue:`32639`)
 - The ``fastpath`` keyword in the ``SingleBlockManager`` constructor is deprecated and will be removed in a future version (:issue:`33092`)
 - Providing ``suffixes`` as a ``set`` in :func:`pandas.merge` is deprecated. Provide a tuple instead (:issue:`33740`, :issue:`34741`).
+- Indexing a series with a multi-dimensional indexer like ``[:, None]`` to return an ndarray now raises a ``FutureWarning``. Convert to a NumPy array before indexing instead (:issue:`27837`)
 - :meth:`Index.is_mixed` is deprecated and will be removed in a future version, check ``index.inferred_type`` directly instead (:issue:`32922`)
 
 - Passing any arguments but the first one to  :func:`read_html` as
@@ -976,6 +978,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc` with dictionary of values changes columns with dtype of ``int`` to ``float`` (:issue:`34573`)
 - Bug in :meth:`Series.loc` when used with a :class:`MultiIndex` would raise an IndexingError when accessing a None value (:issue:`34318`)
 - Bug in :meth:`DataFrame.reset_index` and :meth:`Series.reset_index` would not preserve data types on an empty :class:`DataFrame` or :class:`Series` with a :class:`MultiIndex` (:issue:`19602`)
+- Bug in :class:`Series` and :class:`DataFrame` indexing with a ``time`` key on a :class:`DatetimeIndex` with ``NaT`` entries (:issue:`35114`)
 
 Missing
 ^^^^^^^
@@ -1121,6 +1124,7 @@ Sparse
 - Bug where :class:`DataFrame` containing :class:`SparseArray` filled with ``NaN`` when indexed by a list-like (:issue:`27781`, :issue:`29563`)
 - The repr of :class:`SparseDtype` now includes the repr of its ``fill_value`` attribute. Previously it used ``fill_value``'s  string representation (:issue:`34352`)
 - Bug where empty :class:`DataFrame` could not be cast to :class:`SparseDtype` (:issue:`33113`)
+- Bug in :meth:`arrays.SparseArray` was returning the incorrect type when indexing a sparse dataframe with an iterable (:issue:`34526`, :issue:`34540`)
 
 ExtensionArray
 ^^^^^^^^^^^^^^

doc/source/whatsnew/v1.1.0.rst

@@ -20,14 +20,14 @@ dependencies:
   - flake8<3.8.0  # temporary pin, GH#34150
   - flake8-comprehensions>=3.1.0  # used by flake8, linting of unnecessary comprehensions
   - flake8-rst>=0.6.0,<=0.7.0  # linting of code blocks in rst files
-  - isort  # check that imports are in the right order
+  - isort=4.3.21  # check that imports are in the right order
   - mypy=0.730
   - pycodestyle  # used by flake8
 
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
   - gitdb2=2.0.6  # GH-32060
-  - sphinx
+  - sphinx<=3.1.1
 
   # documentation (jupyter notebooks)
   - nbconvert>=5.4.1

environment.yml

@@ -15,7 +15,7 @@ PyDateTime_IMPORT
 
 
 cimport numpy as cnp
-from numpy cimport float64_t, int64_t, ndarray, uint8_t
+from numpy cimport float64_t, int64_t, ndarray, uint8_t, intp_t
 import numpy as np
 cnp.import_array()
 
@@ -46,7 +46,6 @@ from pandas._libs.tslibs.timezones cimport (
     get_dst_info,
     is_utc,
     is_tzlocal,
-    utc_pytz as UTC,
 )
 from pandas._libs.tslibs.conversion cimport (
     _TSObject,
@@ -67,8 +66,8 @@ from pandas._libs.tslibs.timestamps cimport create_timestamp_from_ts, _Timestamp
 from pandas._libs.tslibs.timestamps import Timestamp
 
 from pandas._libs.tslibs.tzconversion cimport (
-    tz_convert_single,
     tz_convert_utc_to_tzlocal,
+    tz_localize_to_utc_single,
 )
 
 # Note: this is the only non-tslibs intra-pandas dependency here
@@ -157,13 +156,15 @@ def ints_to_pydatetime(
         Py_ssize_t i, n = len(arr)
         ndarray[int64_t] trans
         int64_t[:] deltas
-        Py_ssize_t pos
+        intp_t[:] pos
         npy_datetimestruct dts
         object dt, new_tz
         str typ
-        int64_t value, delta, local_value
+        int64_t value, local_value, delta = NPY_NAT  # dummy for delta
         ndarray[object] result = np.empty(n, dtype=object)
         object (*func_create)(int64_t, npy_datetimestruct, tzinfo, object, bint)
+        bint use_utc = False, use_tzlocal = False, use_fixed = False
+        bint use_pytz = False
 
     if box == "date":
         assert (tz is None), "tz should be None when converting to date"
@@ -184,66 +185,45 @@ def ints_to_pydatetime(
         )
 
     if is_utc(tz) or tz is None:
-        for i in range(n):
-            value = arr[i]
-            if value == NPY_NAT:
-                result[i] = <object>NaT
-            else:
-                dt64_to_dtstruct(value, &dts)
-                result[i] = func_create(value, dts, tz, freq, fold)
+        use_utc = True
     elif is_tzlocal(tz):
-        for i in range(n):
-            value = arr[i]
-            if value == NPY_NAT:
-                result[i] = <object>NaT
-            else:
-                # Python datetime objects do not support nanosecond
-                # resolution (yet, PEP 564). Need to compute new value
-                # using the i8 representation.
-                local_value = tz_convert_utc_to_tzlocal(value, tz)
-                dt64_to_dtstruct(local_value, &dts)
-                result[i] = func_create(value, dts, tz, freq, fold)
+        use_tzlocal = True
     else:
         trans, deltas, typ = get_dst_info(tz)
-
-        if typ not in ['pytz', 'dateutil']:
+        if typ not in ["pytz", "dateutil"]:
             # static/fixed; in this case we know that len(delta) == 1
+            use_fixed = True
             delta = deltas[0]
-            for i in range(n):
-                value = arr[i]
-                if value == NPY_NAT:
-                    result[i] = <object>NaT
-                else:
-                    # Adjust datetime64 timestamp, recompute datetimestruct
-                    dt64_to_dtstruct(value + delta, &dts)
-                    result[i] = func_create(value, dts, tz, freq, fold)
+        else:
+            pos = trans.searchsorted(arr, side="right") - 1
+            use_pytz = typ == "pytz"
 
-        elif typ == 'dateutil':
-            # no zone-name change for dateutil tzs - dst etc
-            # represented in single object.
-            for i in range(n):
-                value = arr[i]
-                if value == NPY_NAT:
-                    result[i] = <object>NaT
-                else:
-                    # Adjust datetime64 timestamp, recompute datetimestruct
-                    pos = trans.searchsorted(value, side='right') - 1
-                    dt64_to_dtstruct(value + deltas[pos], &dts)
-                    result[i] = func_create(value, dts, tz, freq, fold)
+    for i in range(n):
+        new_tz = tz
+        value = arr[i]
+
+        if value == NPY_NAT:
+            result[i] = <object>NaT
         else:
-            # pytz
-            for i in range(n):
-                value = arr[i]
-                if value == NPY_NAT:
-                    result[i] = <object>NaT
-                else:
-                    # Adjust datetime64 timestamp, recompute datetimestruct
-                    pos = trans.searchsorted(value, side='right') - 1
-                    # find right representation of dst etc in pytz timezone
-                    new_tz = tz._tzinfos[tz._transition_info[pos]]
+            if use_utc:
+                local_value = value
+            elif use_tzlocal:
+                local_value = tz_convert_utc_to_tzlocal(value, tz)
+            elif use_fixed:
+                local_value = value + delta
+            elif not use_pytz:
+                # i.e. dateutil
+                # no zone-name change for dateutil tzs - dst etc
+                # represented in single object.
+                local_value = value + deltas[pos[i]]
+            else:
+                # pytz
+                # find right representation of dst etc in pytz timezone
+                new_tz = tz._tzinfos[tz._transition_info[pos[i]]]
+                local_value = value + deltas[pos[i]]
 
-                    dt64_to_dtstruct(value + deltas[pos], &dts)
-                    result[i] = func_create(value, dts, new_tz, freq, fold)
+            dt64_to_dtstruct(local_value, &dts)
+            result[i] = func_create(value, dts, new_tz, freq, fold)
 
     return result
 
@@ -269,7 +249,7 @@ def _test_parse_iso8601(ts: str):
     check_dts_bounds(&obj.dts)
     if out_local == 1:
         obj.tzinfo = pytz.FixedOffset(out_tzoffset)
-        obj.value = tz_convert_single(obj.value, obj.tzinfo, UTC)
+        obj.value = tz_localize_to_utc_single(obj.value, obj.tzinfo)
         return Timestamp(obj.value, tz=obj.tzinfo)
     else:
         return Timestamp(obj.value)
@@ -363,8 +343,8 @@ def format_array_from_datetime(
 
 def array_with_unit_to_datetime(
     ndarray values,
-    object unit,
-    str errors='coerce'
+    str unit,
+    str errors="coerce"
 ):
     """
     Convert the ndarray to datetime according to the time unit.
@@ -384,7 +364,7 @@ def array_with_unit_to_datetime(
     ----------
     values : ndarray of object
          Date-like objects to convert.
-    unit : object
+    unit : str
          Time unit to use during conversion.
     errors : str, default 'raise'
          Error behavior when parsing.
@@ -727,7 +707,7 @@ cpdef array_to_datetime(
                             # dateutil.tz.tzoffset objects
                             out_tzoffset_vals.add(out_tzoffset * 60.)
                             tz = pytz.FixedOffset(out_tzoffset)
-                            value = tz_convert_single(value, tz, UTC)
+                            value = tz_localize_to_utc_single(value, tz)
                             out_local = 0
                             out_tzoffset = 0
                         else: