Merge branch 'master' into bug-fix-23020

Author: makbigc

.pep8speaks.yml

@@ -15,7 +15,6 @@ pycodestyle:
         - E402,  # module level import not at top of file
         - E722,  # do not use bare except
         - E731,  # do not assign a lambda expression, use a def
-        - E741,  # ambiguous variable name 'l'
         - C406,  # Unnecessary list literal - rewrite as a dict literal.
         - C408,  # Unnecessary dict call - rewrite as a literal.
         - C409   # Unnecessary list passed to tuple() - rewrite as a tuple literal.

asv_bench/benchmarks/indexing.py

@@ -2,16 +2,17 @@
 
 import numpy as np
 import pandas.util.testing as tm
-from pandas import (Series, DataFrame, Panel, MultiIndex, Int64Index,
-                    Float64Index, IntervalIndex, CategoricalIndex,
+from pandas import (Series, DataFrame, Panel, MultiIndex,
+                    Int64Index, UInt64Index, Float64Index,
+                    IntervalIndex, CategoricalIndex,
                     IndexSlice, concat, date_range)
 
 
 class NumericSeriesIndexing(object):
 
     goal_time = 0.2
     params = [
-        (Int64Index, Float64Index),
+        (Int64Index, UInt64Index, Float64Index),
         ('unique_monotonic_inc', 'nonunique_monotonic_inc'),
     ]
     param_names = ['index_dtype', 'index_structure']

asv_bench/benchmarks/indexing_engines.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+from pandas._libs.index import (Int64Engine, UInt64Engine, Float64Engine,
+                                ObjectEngine)
+
+
+class NumericEngineIndexing(object):
+
+    goal_time = 0.2
+    params = [[Int64Engine, UInt64Engine, Float64Engine],
+              [np.int64, np.uint64, np.float64],
+              ['monotonic_incr', 'monotonic_decr', 'non_monotonic'],
+              ]
+    param_names = ['engine', 'dtype', 'index_type']
+
+    def setup(self, engine, dtype, index_type):
+        N = 10**5
+        values = list([1] * N + [2] * N + [3] * N)
+        arr = {
+            'monotonic_incr': np.array(values, dtype=dtype),
+            'monotonic_decr': np.array(list(reversed(values)),
+                                       dtype=dtype),
+            'non_monotonic': np.array([1, 2, 3] * N, dtype=dtype),
+        }[index_type]
+
+        self.data = engine(lambda: arr, len(arr))
+        # code belows avoids populating the mapping etc. while timing.
+        self.data.get_loc(2)
+
+    def time_get_loc(self, engine, dtype, index_type):
+        self.data.get_loc(2)
+
+
+class ObjectEngineIndexing(object):
+
+    goal_time = 0.2
+    params = [('monotonic_incr', 'monotonic_decr', 'non_monotonic')]
+    param_names = ['index_type']
+
+    def setup(self, index_type):
+        N = 10**5
+        values = list('a' * N + 'b' * N + 'c' * N)
+        arr = {
+            'monotonic_incr': np.array(values, dtype=object),
+            'monotonic_decr': np.array(list(reversed(values)), dtype=object),
+            'non_monotonic': np.array(list('abc') * N, dtype=object),
+        }[index_type]
+
+        self.data = ObjectEngine(lambda: arr, len(arr))
+        # code belows avoids populating the mapping etc. while timing.
+        self.data.get_loc('b')
+
+    def time_get_loc(self, index_type):
+        self.data.get_loc('b')

ci/azure-macos-35.yaml

@@ -8,10 +8,10 @@ dependencies:
   - html5lib
   - jinja2
   - lxml
-  - matplotlib
+  - matplotlib=2.2.0
   - nomkl
   - numexpr
-  - numpy=1.10.4
+  - numpy=1.12.0
   - openpyxl=2.5.5
   - pytables
   - python=3.5*

ci/azure-windows-27.yaml

@@ -10,7 +10,7 @@ dependencies:
   - html5lib
   - jinja2=2.8
   - lxml
-  - matplotlib
+  - matplotlib=2.0.1
   - numexpr
   - numpy=1.12*
   - openpyxl=2.5.5

ci/azure-windows-36.yaml

@@ -5,12 +5,14 @@ channels:
 dependencies:
   - blosc
   - bottleneck
+  - boost-cpp<1.67
   - fastparquet
   - feather-format
   - matplotlib
   - numexpr
   - numpy=1.14*
   - openpyxl=2.5.5
+  - parquet-cpp
   - pyarrow
   - pytables
   - python-dateutil

ci/azure/macos.yml

@@ -9,7 +9,7 @@ jobs:
   strategy:
     maxParallel: 11
     matrix:
-      py35_np_110:
+      py35_np_120:
         ENV_FILE: ci/azure-macos-35.yaml
         CONDA_PY: "35"
         CONDA_ENV: pandas

ci/azure/windows-py27.yml

@@ -9,7 +9,7 @@ jobs:
   strategy:
     maxParallel: 11
     matrix:
-      py36_np14:
+      py36_np121:
         ENV_FILE: ci/azure-windows-27.yaml
         CONDA_PY: "27"
         CONDA_ENV: pandas

ci/circle-27-compat.yaml

@@ -3,18 +3,18 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - bottleneck=1.0.0
+  - bottleneck=1.2.0
   - cython=0.28.2
   - jinja2=2.8
-  - numexpr=2.4.4 # we test that we correctly don't use an unsupported numexpr
-  - numpy=1.9.3
+  - numexpr=2.6.1
+  - numpy=1.12.0
   - openpyxl=2.5.5
   - psycopg2
-  - pytables=3.2.2
+  - pytables=3.4.2
   - python-dateutil=2.5.0
   - python=2.7*
   - pytz=2013b
-  - scipy=0.14.0
+  - scipy=0.18.1
   - sqlalchemy=0.7.8
   - xlrd=0.9.2
   - xlsxwriter=0.5.2

ci/code_checks.sh

@@ -16,7 +16,7 @@
 
 echo "inside $0"
 [[ $LINT ]] || { echo "NOT Linting. To lint use: LINT=true $0 $1"; exit 0; }
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" ]] || { echo "Unkown command $1. Usage: $0 [lint|patterns|doctests]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" ]] || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests]"; exit 9999; }
 
 source activate pandas
 RET=0
@@ -56,6 +56,11 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    # Imports - Check formatting using isort see setup.cfg for settings
+    MSG='Check import format using isort ' ; echo $MSG
+    isort --recursive --check-only pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 ### PATTERNS ###
@@ -117,22 +122,22 @@ fi
 if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Doctests frame.py' ; echo $MSG
-    pytest --doctest-modules -v pandas/core/frame.py \
-        -k"-axes -combine -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_stata"
+    pytest -q --doctest-modules pandas/core/frame.py \
+        -k"-axes -combine -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_stata"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests series.py' ; echo $MSG
-    pytest --doctest-modules -v pandas/core/series.py \
+    pytest -q --doctest-modules pandas/core/series.py \
         -k"-nonzero -reindex -searchsorted -to_dict"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests generic.py' ; echo $MSG
-    pytest --doctest-modules -v pandas/core/generic.py \
+    pytest -q --doctest-modules pandas/core/generic.py \
         -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -resample -to_json -transpose -values -xs"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests top-level reshaping functions' ; echo $MSG
-    pytest --doctest-modules -v \
+    pytest -q --doctest-modules \
         pandas/core/reshape/concat.py \
         pandas/core/reshape/pivot.py \
         pandas/core/reshape/reshape.py \

ci/environment-dev.yaml

@@ -8,6 +8,7 @@ dependencies:
   - flake8
   - flake8-comprehensions
   - hypothesis>=3.58.0
+  - isort
   - moto
   - pytest>=3.6
   - python-dateutil>=2.5.0

ci/requirements-optional-conda.txt

@@ -1,6 +1,6 @@
 beautifulsoup4>=4.2.1
 blosc
-bottleneck
+bottleneck>=1.2.0
 fastparquet
 feather-format
 gcsfs
@@ -9,17 +9,17 @@ ipython>=5.6.0
 ipykernel
 jinja2
 lxml
-matplotlib
+matplotlib>=2.0.0
 nbsphinx
-numexpr
+numexpr>=2.6.1
 openpyxl=2.5.5
 pyarrow
 pymysql
-pytables
+pytables>=3.4.2
 pytest-cov
 pytest-xdist
 s3fs
-scipy
+scipy>=0.18.1
 seaborn
 sqlalchemy
 statsmodels

ci/requirements-optional-pip.txt

@@ -2,7 +2,7 @@
 # Do not modify directly
 beautifulsoup4>=4.2.1
 blosc
-bottleneck
+bottleneck>=1.2.0
 fastparquet
 feather-format
 gcsfs
@@ -11,17 +11,17 @@ ipython>=5.6.0
 ipykernel
 jinja2
 lxml
-matplotlib
+matplotlib>=2.0.0
 nbsphinx
-numexpr
+numexpr>=2.6.1
 openpyxl==2.5.5
 pyarrow
 pymysql
 tables
 pytest-cov
 pytest-xdist
 s3fs
-scipy
+scipy>=0.18.1
 seaborn
 sqlalchemy
 statsmodels

ci/requirements_dev.txt

@@ -5,6 +5,7 @@ NumPy
 flake8
 flake8-comprehensions
 hypothesis>=3.58.0
+isort
 moto
 pytest>=3.6
 python-dateutil>=2.5.0

ci/travis-27-locale.yaml

@@ -3,11 +3,11 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - bottleneck=1.0.0
+  - bottleneck=1.2.0
   - cython=0.28.2
   - lxml
-  - matplotlib=1.4.3
-  - numpy=1.9.3
+  - matplotlib=2.0.0
+  - numpy=1.12.0
   - openpyxl=2.4.0
   - python-dateutil
   - python-blosc

ci/travis-27.yaml

@@ -14,7 +14,7 @@ dependencies:
   - jemalloc=4.5.0.post
   - jinja2=2.8
   - lxml
-  - matplotlib
+  - matplotlib=2.2.2
   - mock
   - nomkl
   - numexpr

ci/travis-36.yaml

@@ -14,6 +14,7 @@ dependencies:
   - geopandas
   - html5lib
   - ipython
+  - isort
   - jinja2
   - lxml
   - matplotlib

ci/travis-37-numpydev.yaml

@@ -13,5 +13,5 @@ dependencies:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
     - "--pre"
-    - "numpy"
+    - "numpy<=1.16.0.dev0+20181015190246"
     - "scipy"

doc/source/cookbook.rst

@@ -1228,36 +1228,40 @@ Correlation
 
 The `method` argument within `DataFrame.corr` can accept a callable in addition to the named correlation types.  Here we compute the `distance correlation <https://en.wikipedia.org/wiki/Distance_correlation>`__ matrix for a `DataFrame` object.
 
-.. ipython:: python
-
-    def distcorr(x, y):
-        n = len(x)
-        a = np.zeros(shape=(n, n))
-        b = np.zeros(shape=(n, n))
-
-        for i in range(n):
-            for j in range(i + 1, n):
-                a[i, j] = abs(x[i] - x[j])
-                b[i, j] = abs(y[i] - y[j])
-
-        a += a.T
-        b += b.T
-
-        a_bar = np.vstack([np.nanmean(a, axis=0)] * n)
-        b_bar = np.vstack([np.nanmean(b, axis=0)] * n)
-
-        A = a - a_bar - a_bar.T + np.full(shape=(n, n), fill_value=a_bar.mean())
-        B = b - b_bar - b_bar.T + np.full(shape=(n, n), fill_value=b_bar.mean())
-
-        cov_ab = np.sqrt(np.nansum(A * B)) / n
-        std_a = np.sqrt(np.sqrt(np.nansum(A**2)) / n)
-        std_b = np.sqrt(np.sqrt(np.nansum(B**2)) / n)
-
-        return cov_ab / std_a / std_b
-
-    df = pd.DataFrame(np.random.normal(size=(100, 3)))
+.. code-block:: python
 
-    df.corr(method=distcorr)
+    >>> def distcorr(x, y):
+    ...     n = len(x)
+    ...     a = np.zeros(shape=(n, n))
+    ...     b = np.zeros(shape=(n, n))
+    ...
+    ...     for i in range(n):
+    ...         for j in range(i + 1, n):
+    ...             a[i, j] = abs(x[i] - x[j])
+    ...             b[i, j] = abs(y[i] - y[j])
+    ...
+    ...     a += a.T
+    ...     b += b.T
+    ...
+    ...     a_bar = np.vstack([np.nanmean(a, axis=0)] * n)
+    ...     b_bar = np.vstack([np.nanmean(b, axis=0)] * n)
+    ...
+    ...     A = a - a_bar - a_bar.T + np.full(shape=(n, n), fill_value=a_bar.mean())
+    ...     B = b - b_bar - b_bar.T + np.full(shape=(n, n), fill_value=b_bar.mean())
+    ...
+    ...     cov_ab = np.sqrt(np.nansum(A * B)) / n
+    ...     std_a = np.sqrt(np.sqrt(np.nansum(A**2)) / n)
+    ...     std_b = np.sqrt(np.sqrt(np.nansum(B**2)) / n)
+    ...
+    ...     return cov_ab / std_a / std_b
+    ...
+    >>> df = pd.DataFrame(np.random.normal(size=(100, 3)))
+    ...
+    >>> df.corr(method=distcorr)
+              0         1         2
+    0  1.000000  0.171368  0.145302
+    1  0.171368  1.000000  0.189919
+    2  0.145302  0.189919  1.000000
 
 Timedeltas
 ----------