Merge remote-tracking branch 'upstream/master' into ea-repr

Author: TomAugspurger

doc/source/api.rst

@@ -2482,6 +2482,7 @@ Style Application
    Styler.set_properties
    Styler.set_uuid
    Styler.clear
+   Styler.pipe
 
 Builtin Styles
 ~~~~~~~~~~~~~~

doc/source/whatsnew/v0.24.0.rst

@@ -184,6 +184,30 @@ array, but rather an ``ExtensionArray``:
 This is the same behavior as ``Series.values`` for categorical data. See
 :ref:`whatsnew_0240.api_breaking.interval_values` for more.
 
+
+.. _whatsnew_0240.enhancements.styler_pipe:
+
+New ``Styler.pipe()`` method
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The :class:`~pandas.io.formats.style.Styler` class has gained a
+:meth:`~pandas.io.formats.style.Styler.pipe` method (:issue:`23229`).  This provides a
+convenient way to apply users' predefined styling functions, and can help reduce
+"boilerplate" when using DataFrame styling functionality repeatedly within a notebook.
+
+.. ipython:: python
+
+    df = pandas.DataFrame({'N': [1250, 1500, 1750], 'X': [0.25, 0.35, 0.50]})
+
+    def format_and_align(styler):
+        return (styler.format({'N': '{:,}', 'X': '{:.1%}'})
+                      .set_properties(**{'text-align': 'right'}))
+
+    df.style.pipe(format_and_align).set_caption('Summary of results.')
+
+Similar methods already exist for other classes in pandas, including :meth:`DataFrame.pipe`,
+:meth:`Groupby.pipe`, and :meth:`Resampler.pipe`.
+
+
 .. _whatsnew_0240.enhancements.join_with_two_multiindexes:
 
 Joining with two multi-indexes
@@ -225,6 +249,7 @@ For earlier versions this can be done using the following.
    pd.merge(left.reset_index(), right.reset_index(),
             on=['key'], how='inner').set_index(['key', 'X', 'Y'])
 
+
 .. _whatsnew_0240.enhancements.rename_axis:
 
 Renaming names in a MultiIndex
@@ -248,6 +273,7 @@ Example:
 
 See the :ref:`advanced docs on renaming<advanced.index_names>` for more details.
 
+
 .. _whatsnew_0240.enhancements.other:
 
 Other Enhancements
@@ -1405,7 +1431,8 @@ Notice how we now instead output ``np.nan`` itself instead of a stringified form
 - Bug in :meth:`read_csv()` in which unnecessary warnings were being raised when the dialect's values conflicted with the default arguments (:issue:`23761`)
 - Bug in :meth:`read_html()` in which the error message was not displaying the valid flavors when an invalid one was provided (:issue:`23549`)
 - Bug in :meth:`read_excel()` in which extraneous header names were extracted, even though none were specified (:issue:`11733`)
-- Bug in :meth:`read_excel()` in which ``index_col=None`` was not being respected and parsing index columns anyway (:issue:`20480`)
+- Bug in :meth:`read_excel()` in which column names were not being properly converted to string sometimes in Python 2.x (:issue:`23874`)
+- Bug in :meth:`read_excel()` in which ``index_col=None`` was not being respected and parsing index columns anyway (:issue:`18792`, :issue:`20480`)
 - Bug in :meth:`read_excel()` in which ``usecols`` was not being validated for proper column names when passed in as a string (:issue:`20480`)
 - :func:`DataFrame.to_string()`, :func:`DataFrame.to_html()`, :func:`DataFrame.to_latex()` will correctly format output when a string is passed as the ``float_format`` argument (:issue:`21625`, :issue:`22270`)
 

pandas/_libs/tslib.pyx

@@ -76,7 +76,8 @@ cdef inline object create_time_from_ts(
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def ints_to_pydatetime(int64_t[:] arr, tz=None, freq=None, box="datetime"):
+def ints_to_pydatetime(int64_t[:] arr, object tz=None, object freq=None,
+                       str box="datetime"):
     """
     Convert an i8 repr to an ndarray of datetimes, date, time or Timestamp
 
@@ -104,8 +105,9 @@ def ints_to_pydatetime(int64_t[:] arr, tz=None, freq=None, box="datetime"):
         int64_t[:] deltas
         Py_ssize_t pos
         npy_datetimestruct dts
-        object dt
-        int64_t value, delta
+        object dt, new_tz
+        str typ
+        int64_t value, delta, local_value
         ndarray[object] result = np.empty(n, dtype=object)
         object (*func_create)(int64_t, npy_datetimestruct, object, object)
 
@@ -303,7 +305,8 @@ def format_array_from_datetime(ndarray[int64_t] values, object tz=None,
     return result
 
 
-def array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
+def array_with_unit_to_datetime(ndarray values, object unit,
+                                str errors='coerce'):
     """
     convert the ndarray according to the unit
     if errors:
@@ -458,10 +461,10 @@ def array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-cpdef array_to_datetime(ndarray[object] values, errors='raise',
-                        dayfirst=False, yearfirst=False,
-                        format=None, utc=None,
-                        require_iso8601=False):
+cpdef array_to_datetime(ndarray[object] values, str errors='raise',
+                        bint dayfirst=False, bint yearfirst=False,
+                        object format=None, object utc=None,
+                        bint require_iso8601=False):
     """
     Converts a 1D array of date-like values to a numpy array of either:
         1) datetime64[ns] data
@@ -510,9 +513,11 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
         bint is_raise = errors=='raise'
         bint is_ignore = errors=='ignore'
         bint is_coerce = errors=='coerce'
+        bint is_same_offsets
         _TSObject _ts
+        int64_t value
         int out_local=0, out_tzoffset=0
-        float offset_seconds
+        float offset_seconds, tz_offset
         set out_tzoffset_vals = set()
 
     # specify error conditions
@@ -764,7 +769,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise',
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cdef array_to_datetime_object(ndarray[object] values, bint is_raise,
-                              dayfirst=False, yearfirst=False):
+                              bint dayfirst=False, bint yearfirst=False):
     """
     Fall back function for array_to_datetime
 

pandas/_libs/tslibs/conversion.pyx

@@ -99,10 +99,13 @@ def ensure_datetime64ns(arr: ndarray, copy: bool=True):
 
     ivalues = arr.view(np.int64).ravel()
 
-    result = np.empty(shape, dtype='M8[ns]')
+    result = np.empty(shape, dtype=NS_DTYPE)
     iresult = result.ravel().view(np.int64)
 
     if len(iresult) == 0:
+        result = arr.view(NS_DTYPE)
+        if copy:
+            result = result.copy()
         return result
 
     unit = get_datetime64_unit(arr.flat[0])

pandas/conftest.py

@@ -1,3 +1,5 @@
+from datetime import date, time, timedelta
+from decimal import Decimal
 import importlib
 import os
 
@@ -8,7 +10,7 @@
 import pytest
 from pytz import FixedOffset, utc
 
-from pandas.compat import PY3
+from pandas.compat import PY3, u
 import pandas.util._test_decorators as td
 
 import pandas as pd
@@ -514,6 +516,84 @@ def any_numpy_dtype(request):
     return request.param
 
 
+# categoricals are handled separately
+_any_skipna_inferred_dtype = [
+    ('string', ['a', np.nan, 'c']),
+    ('unicode' if not PY3 else 'string', [u('a'), np.nan, u('c')]),
+    ('bytes' if PY3 else 'string', [b'a', np.nan, b'c']),
+    ('empty', [np.nan, np.nan, np.nan]),
+    ('empty', []),
+    ('mixed-integer', ['a', np.nan, 2]),
+    ('mixed', ['a', np.nan, 2.0]),
+    ('floating', [1.0, np.nan, 2.0]),
+    ('integer', [1, np.nan, 2]),
+    ('mixed-integer-float', [1, np.nan, 2.0]),
+    ('decimal', [Decimal(1), np.nan, Decimal(2)]),
+    ('boolean', [True, np.nan, False]),
+    ('datetime64', [np.datetime64('2013-01-01'), np.nan,
+                    np.datetime64('2018-01-01')]),
+    ('datetime', [pd.Timestamp('20130101'), np.nan, pd.Timestamp('20180101')]),
+    ('date', [date(2013, 1, 1), np.nan, date(2018, 1, 1)]),
+    # The following two dtypes are commented out due to GH 23554
+    # ('complex', [1 + 1j, np.nan, 2 + 2j]),
+    # ('timedelta64', [np.timedelta64(1, 'D'),
+    #                  np.nan, np.timedelta64(2, 'D')]),
+    ('timedelta', [timedelta(1), np.nan, timedelta(2)]),
+    ('time', [time(1), np.nan, time(2)]),
+    ('period', [pd.Period(2013), pd.NaT, pd.Period(2018)]),
+    ('interval', [pd.Interval(0, 1), np.nan, pd.Interval(0, 2)])]
+ids, _ = zip(*_any_skipna_inferred_dtype)  # use inferred type as fixture-id
+
+
+@pytest.fixture(params=_any_skipna_inferred_dtype, ids=ids)
+def any_skipna_inferred_dtype(request):
+    """
+    Fixture for all inferred dtypes from _libs.lib.infer_dtype
+
+    The covered (inferred) types are:
+    * 'string'
+    * 'unicode' (if PY2)
+    * 'empty'
+    * 'bytes' (if PY3)
+    * 'mixed'
+    * 'mixed-integer'
+    * 'mixed-integer-float'
+    * 'floating'
+    * 'integer'
+    * 'decimal'
+    * 'boolean'
+    * 'datetime64'
+    * 'datetime'
+    * 'date'
+    * 'timedelta'
+    * 'time'
+    * 'period'
+    * 'interval'
+
+    Returns
+    -------
+    inferred_dtype : str
+        The string for the inferred dtype from _libs.lib.infer_dtype
+    values : np.ndarray
+        An array of object dtype that will be inferred to have
+        `inferred_dtype`
+
+    Examples
+    --------
+    >>> import pandas._libs.lib as lib
+    >>>
+    >>> def test_something(any_skipna_inferred_dtype):
+    ...     inferred_dtype, values = any_skipna_inferred_dtype
+    ...     # will pass
+    ...     assert lib.infer_dtype(values, skipna=True) == inferred_dtype
+    """
+    inferred_dtype, values = request.param
+    values = np.array(values, dtype=object)  # object dtype to avoid casting
+
+    # correctness of inference tested in tests/dtypes/test_inference.py
+    return inferred_dtype, values
+
+
 @pytest.fixture
 def mock():
     """