Merge for update

Author: makbigc

Makefile

@@ -1,5 +1,7 @@
 .PHONY : develop build clean clean_pyc doc lint-diff black
 
+all: develop
+
 clean:
 	-python setup.py clean
 

asv_bench/benchmarks/index_cached_properties.py

@@ -0,0 +1,75 @@
+import pandas as pd
+
+
+class IndexCache:
+    number = 1
+    repeat = (3, 100, 20)
+
+    params = [
+        [
+            "DatetimeIndex",
+            "Float64Index",
+            "IntervalIndex",
+            "Int64Index",
+            "MultiIndex",
+            "PeriodIndex",
+            "RangeIndex",
+            "TimedeltaIndex",
+            "UInt64Index",
+        ]
+    ]
+    param_names = ["index_type"]
+
+    def setup(self, index_type):
+        N = 10 ** 5
+        if index_type == "MultiIndex":
+            self.idx = pd.MultiIndex.from_product(
+                [pd.date_range("1/1/2000", freq="T", periods=N // 2), ["a", "b"]]
+            )
+        elif index_type == "DatetimeIndex":
+            self.idx = pd.date_range("1/1/2000", freq="T", periods=N)
+        elif index_type == "Int64Index":
+            self.idx = pd.Index(range(N))
+        elif index_type == "PeriodIndex":
+            self.idx = pd.period_range("1/1/2000", freq="T", periods=N)
+        elif index_type == "RangeIndex":
+            self.idx = pd.RangeIndex(start=0, stop=N)
+        elif index_type == "IntervalIndex":
+            self.idx = pd.IntervalIndex.from_arrays(range(N), range(1, N + 1))
+        elif index_type == "TimedeltaIndex":
+            self.idx = pd.TimedeltaIndex(range(N))
+        elif index_type == "Float64Index":
+            self.idx = pd.Float64Index(range(N))
+        elif index_type == "UInt64Index":
+            self.idx = pd.UInt64Index(range(N))
+        else:
+            raise ValueError
+        assert len(self.idx) == N
+        self.idx._cache = {}
+
+    def time_values(self, index_type):
+        self.idx._values
+
+    def time_shape(self, index_type):
+        self.idx.shape
+
+    def time_is_monotonic(self, index_type):
+        self.idx.is_monotonic
+
+    def time_is_monotonic_decreasing(self, index_type):
+        self.idx.is_monotonic_decreasing
+
+    def time_is_monotonic_increasing(self, index_type):
+        self.idx.is_monotonic_increasing
+
+    def time_is_unique(self, index_type):
+        self.idx.is_unique
+
+    def time_engine(self, index_type):
+        self.idx._engine
+
+    def time_inferred_type(self, index_type):
+        self.idx.inferred_type
+
+    def time_is_all_dates(self, index_type):
+        self.idx.is_all_dates

asv_bench/benchmarks/io/parsers.py

@@ -10,7 +10,7 @@
     pass
 
 
-class DoesStringLookLikeDatetime(object):
+class DoesStringLookLikeDatetime:
 
     params = (["2Q2005", "0.0", "10000"],)
     param_names = ["value"]
@@ -23,7 +23,7 @@ def time_check_datetimes(self, value):
             _does_string_look_like_datetime(obj)
 
 
-class ConcatDateCols(object):
+class ConcatDateCols:
 
     params = ([1234567890, "AAAA"], [1, 2])
     param_names = ["value", "dim"]

asv_bench/benchmarks/reshape.py

@@ -240,4 +240,17 @@ def time_qcut_datetime(self, bins):
         pd.qcut(self.datetime_series, bins)
 
 
+class Explode:
+    param_names = ["n_rows", "max_list_length"]
+    params = [[100, 1000, 10000], [3, 5, 10]]
+
+    def setup(self, n_rows, max_list_length):
+
+        data = [np.arange(np.random.randint(max_list_length)) for _ in range(n_rows)]
+        self.series = pd.Series(data)
+
+    def time_explode(self, n_rows, max_list_length):
+        self.series.explode()
+
+
 from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/series_methods.py

@@ -219,7 +219,7 @@ def time_series_datetimeindex_repr(self):
         getattr(self.s, "a", None)
 
 
-class All(object):
+class All:
 
     params = [[10 ** 3, 10 ** 6], ["fast", "slow"]]
     param_names = ["N", "case"]
@@ -232,7 +232,7 @@ def time_all(self, N, case):
         self.s.all()
 
 
-class Any(object):
+class Any:
 
     params = [[10 ** 3, 10 ** 6], ["fast", "slow"]]
     param_names = ["N", "case"]
@@ -245,7 +245,7 @@ def time_any(self, N, case):
         self.s.any()
 
 
-class NanOps(object):
+class NanOps:
 
     params = [
         [

asv_bench/benchmarks/timeseries.py

@@ -293,7 +293,7 @@ def time_format_YYYYMMDD(self):
         to_datetime(self.stringsD, format="%Y%m%d")
 
 
-class ToDatetimeCacheSmallCount(object):
+class ToDatetimeCacheSmallCount:
 
     params = ([True, False], [50, 500, 5000, 100000])
     param_names = ["cache", "count"]

ci/azure/posix.yml

@@ -33,6 +33,12 @@ jobs:
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "it_IT.UTF-8"
 
+        py36_32bit:
+          ENV_FILE: ci/deps/azure-36-32bit.yaml
+          CONDA_PY: "36"
+          PATTERN: "not slow and not network"
+          BITS32: "yes"
+
         py37_locale:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"

ci/code_checks.sh

@@ -156,7 +156,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for python2 new-style classes and for empty parentheses' ; echo $MSG
-    invgrep -R --include="*.py" --include="*.pyx" -E "class\s\S*\((object)?\):" pandas scripts
+    invgrep -R --include="*.py" --include="*.pyx" -E "class\s\S*\((object)?\):" pandas asv_bench/benchmarks scripts
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for backticks incorrectly rendering because of missing spaces' ; echo $MSG

ci/deps/azure-36-32bit.yaml

@@ -0,0 +1,20 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - gcc_linux-32
+  - gcc_linux-32
+  - gxx_linux-32
+  - cython=0.28.2
+  - numpy=1.14.*
+  - python-dateutil
+  - python=3.6.*
+  - pytz=2017.2
+  # universal
+  - pytest>=4.0.2,<5.0.0
+  - pytest-xdist
+  - pytest-mock
+  - pytest-azurepipelines
+  - hypothesis>=3.58.0
+  - pip

ci/deps/azure-37-numpydev.yaml

@@ -17,4 +17,5 @@ dependencies:
     - "--pre"
     - "numpy"
     - "scipy"
-    - pytest-azurepipelines
+    # https://github.com/pandas-dev/pandas/issues/27421
+    - pytest-azurepipelines<1.0.0

ci/deps/azure-macos-35.yaml

@@ -29,4 +29,6 @@ dependencies:
     - pytest-xdist
     - pytest-mock
     - hypothesis>=3.58.0
-    - pytest-azurepipelines
+    # https://github.com/pandas-dev/pandas/issues/27421
+    - pytest-azurepipelines<1.0.0
+

ci/setup_env.sh

@@ -94,6 +94,12 @@ echo
 echo "conda env create -q --file=${ENV_FILE}"
 time conda env create -q --file="${ENV_FILE}"
 
+
+if [[ "$BITS32" == "yes" ]]; then
+    # activate 32-bit compiler
+    export CONDA_BUILD=1
+fi
+
 echo "activate pandas-dev"
 source activate pandas-dev
 

doc/source/development/contributing.rst

@@ -288,7 +288,7 @@ complex changes to the documentation as well.
 Some other important things to know about the docs:
 
 * The *pandas* documentation consists of two parts: the docstrings in the code
-  itself and the docs in this folder ``pandas/doc/``.
+  itself and the docs in this folder ``doc/``.
 
   The docstrings provide a clear explanation of the usage of the individual
   functions, while the documentation in this folder consists of tutorial-like
@@ -404,11 +404,11 @@ Building the documentation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 So how do you build the docs? Navigate to your local
-``pandas/doc/`` directory in the console and run::
+``doc/`` directory in the console and run::
 
     python make.py html
 
-Then you can find the HTML output in the folder ``pandas/doc/build/html/``.
+Then you can find the HTML output in the folder ``doc/build/html/``.
 
 The first time you build the docs, it will take quite a while because it has to run
 all the code examples and build all the generated docstring pages. In subsequent
@@ -448,7 +448,7 @@ You can also specify to use multiple cores to speed up the documentation build::
 Open the following file in a web browser to see the full documentation you
 just built::
 
-    pandas/docs/build/html/index.html
+    doc/build/html/index.html
 
 And you'll have the satisfaction of seeing your new and improved documentation!
 

doc/source/reference/extensions.rst

@@ -18,10 +18,44 @@ objects.
    api.extensions.register_series_accessor
    api.extensions.register_index_accessor
    api.extensions.ExtensionDtype
-   api.extensions.ExtensionArray
 
 .. autosummary::
    :toctree: api/
    :template: autosummary/class_without_autosummary.rst
 
+   api.extensions.ExtensionArray
    arrays.PandasArray
+
+.. We need this autosummary so that methods and attributes are generated.
+.. Separate block, since they aren't classes.
+
+   .. autosummary::
+      :toctree: api/
+
+      api.extensions.ExtensionArray._concat_same_type
+      api.extensions.ExtensionArray._formatter
+      api.extensions.ExtensionArray._formatting_values
+      api.extensions.ExtensionArray._from_factorized
+      api.extensions.ExtensionArray._from_sequence
+      api.extensions.ExtensionArray._from_sequence_of_strings
+      api.extensions.ExtensionArray._ndarray_values
+      api.extensions.ExtensionArray._reduce
+      api.extensions.ExtensionArray._values_for_argsort
+      api.extensions.ExtensionArray._values_for_factorize
+      api.extensions.ExtensionArray.argsort
+      api.extensions.ExtensionArray.astype
+      api.extensions.ExtensionArray.copy
+      api.extensions.ExtensionArray.dropna
+      api.extensions.ExtensionArray.factorize
+      api.extensions.ExtensionArray.fillna
+      api.extensions.ExtensionArray.isna
+      api.extensions.ExtensionArray.ravel
+      api.extensions.ExtensionArray.repeat
+      api.extensions.ExtensionArray.searchsorted
+      api.extensions.ExtensionArray.shift
+      api.extensions.ExtensionArray.take
+      api.extensions.ExtensionArray.unique
+      api.extensions.ExtensionArray.dtype
+      api.extensions.ExtensionArray.nbytes
+      api.extensions.ExtensionArray.ndim
+      api.extensions.ExtensionArray.shape

doc/source/reference/frame.rst

@@ -239,6 +239,7 @@ Reshaping, sorting, transposing
    DataFrame.unstack
    DataFrame.swapaxes
    DataFrame.melt
+   DataFrame.explode
    DataFrame.squeeze
    DataFrame.to_xarray
    DataFrame.T

doc/source/reference/series.rst

@@ -245,6 +245,7 @@ Reshaping, sorting
    Series.sort_index
    Series.swaplevel
    Series.unstack
+   Series.explode
    Series.searchsorted
    Series.ravel
    Series.repeat
@@ -590,4 +591,3 @@ Sparse
 
    SparseSeries.to_coo
    SparseSeries.from_coo
-

doc/source/user_guide/reshaping.rst

@@ -801,3 +801,53 @@ Note to subdivide over multiple columns we can pass in a list to the
 
    df.pivot_table(
        values=['val0'], index='row', columns=['item', 'col'], aggfunc=['mean'])
+
+.. _reshaping.explode:
+
+Exploding a list-like column
+----------------------------
+
+.. versionadded:: 0.25.0
+
+Sometimes the values in a column are list-like.
+
+.. ipython:: python
+
+   keys = ['panda1', 'panda2', 'panda3']
+   values = [['eats', 'shoots'], ['shoots', 'leaves'], ['eats', 'leaves']]
+   df = pd.DataFrame({'keys': keys, 'values': values})
+   df
+
+We can 'explode' the ``values`` column, transforming each list-like to a separate row, by using :meth:`~Series.explode`. This will replicate the index values from the original row:
+
+.. ipython:: python
+
+   df['values'].explode()
+
+You can also explode the column in the ``DataFrame``.
+
+.. ipython:: python
+
+   df.explode('values')
+
+:meth:`Series.explode` will replace empty lists with ``np.nan`` and preserve scalar entries. The dtype of the resulting ``Series`` is always ``object``.
+
+.. ipython:: python
+
+   s = pd.Series([[1, 2, 3], 'foo', [], ['a', 'b']])
+   s
+   s.explode()
+
+Here is a typical usecase. You have comma separated strings in a column and want to expand this.
+
+.. ipython:: python
+
+    df = pd.DataFrame([{'var1': 'a,b,c', 'var2': 1},
+                       {'var1': 'd,e,f', 'var2': 2}])
+    df
+
+Creating a long form DataFrame is now straightforward using explode and chained operations
+
+.. ipython:: python
+
+   df.assign(var1=df.var1.str.split(',')).explode('var1')