Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-arith

Author: jbrockmendel

.travis.yml

@@ -41,10 +41,10 @@ matrix:
         - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
-
-    - env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
+      - JOB="3.8, slow" ENV_FILE="ci/deps/travis-38-slow.yaml" PATTERN="slow" SQL="1"
+      services:
+        - mysql
+        - postgresql
 
     - env:
         - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"

ci/azure/posix.yml

@@ -20,39 +20,35 @@ jobs:
           CONDA_PY: "37"
           PATTERN: "not slow and not network and not clipboard"
 
+        py37:
+          ENV_FILE: ci/deps/azure-37.yaml
+          CONDA_PY: "37"
+          PATTERN: "not slow and not network and not clipboard"
+
         py37_locale_slow:
           ENV_FILE: ci/deps/azure-37-locale_slow.yaml
           CONDA_PY: "37"
           PATTERN: "slow"
-          # pandas does not use the language (zh_CN), but should support different encodings (utf8)
-          # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
-          LANG: "zh_CN.utf8"
-          LC_ALL: "zh_CN.utf8"
-          EXTRA_APT: "language-pack-zh-hans"
+          LANG: "it_IT.utf8"
+          LC_ALL: "it_IT.utf8"
+          EXTRA_APT: "language-pack-it xsel"
 
         py37_slow:
           ENV_FILE: ci/deps/azure-37-slow.yaml
           CONDA_PY: "37"
           PATTERN: "slow"
 
-        py37_locale:
-          ENV_FILE: ci/deps/azure-37-locale.yaml
-          CONDA_PY: "37"
-          PATTERN: "not slow and not network"
-          LANG: "it_IT.utf8"
-          LC_ALL: "it_IT.utf8"
-          EXTRA_APT: "language-pack-it xsel"
-
-#        py37_32bit:
-#          ENV_FILE: ci/deps/azure-37-32bit.yaml
-#          CONDA_PY: "37"
-#          PATTERN: "not slow and not network and not clipboard"
-#          BITS32: "yes"
+        py38:
+          ENV_FILE: ci/deps/azure-38.yaml
+          CONDA_PY: "38"
+          PATTERN: "not slow and not network and not clipboard"
 
         py38_locale:
           ENV_FILE: ci/deps/azure-38-locale.yaml
           CONDA_PY: "38"
           PATTERN: "not slow and not network"
+          # pandas does not use the language (zh_CN), but should support different encodings (utf8)
+          # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
           LANG: "zh_CN.utf8"
           LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans xsel"

ci/deps/azure-37-32bit.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
+  - pytest-azurepipelines
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/azure-37-slow.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
+  - pytest-azurepipelines
 
   # pandas dependencies
   - botocore>=1.11

ci/deps/travis-37.yaml renamed to ci/deps/azure-37.yaml

@@ -10,11 +10,11 @@ dependencies:
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
+  - pytest-azurepipelines
 
   # pandas dependencies
   - numpy
   - python-dateutil
   - nomkl
   - pytz
-  - pip
   - tabulate==0.8.3

ci/deps/travis-38.yaml renamed to ci/deps/azure-38.yaml

@@ -11,7 +11,12 @@ dependencies:
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
 
-  # pandas dependencies
+  # required
+  - numpy
+  - python-dateutil
+  - pytz
+
+  # optional
   - beautifulsoup4
   - blosc=1.15.0
   - python-blosc
@@ -20,22 +25,23 @@ dependencies:
   - ipython
   - jinja2
   - lxml=4.3.0
-  - matplotlib=3.0.*
+  - matplotlib
   - nomkl
   - numexpr
-  - numpy
   - openpyxl
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
   - pyarrow>=0.17
-  - psycopg2=2.7
-  - pymysql=0.7.11
   - pytables>=3.5.1
-  - python-dateutil
-  - pytz
   - scipy
-  - sqlalchemy=1.3.0
   - xarray=0.12.0
   - xlrd
   - xlsxwriter
   - xlwt
+  - moto
+  - flask
+
+  # sql
+  - psycopg2=2.7
+  - pymysql=0.7.11
+  - sqlalchemy=1.3.0

ci/deps/travis-37-locale.yaml

@@ -3,35 +3,35 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - python=3.7.*
+  - python=3.8.*
 
   # tools
   - cython>=0.29.21
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-asyncio
   - hypothesis>=3.58.0
-  - pytest-azurepipelines
 
   # pandas dependencies
   - beautifulsoup4
+  - fsspec>=0.7.4
   - html5lib
-  - ipython
-  - jinja2
   - lxml
-  - matplotlib>=3.3.0
-  - moto
-  - flask
-  - nomkl
+  - matplotlib
   - numexpr
-  - numpy=1.16.*
+  - numpy
   - openpyxl
+  - patsy
+  - psycopg2
+  - pymysql
   - pytables
   - python-dateutil
   - pytz
+  - s3fs>=0.4.0
+  - moto>=1.3.14
   - scipy
-  - xarray
+  - sqlalchemy
   - xlrd
   - xlsxwriter
   - xlwt
   - moto
+  - flask

ci/deps/azure-37-locale.yaml renamed to ci/deps/travis-38-slow.yaml

@@ -123,7 +123,10 @@ aggregating statistics for given columns can be defined using the
 .. ipython:: python
 
     titanic.agg(
-        {"Age": ["min", "max", "median", "skew"], "Fare": ["min", "max", "median", "mean"]}
+        {
+            "Age": ["min", "max", "median", "skew"],
+            "Fare": ["min", "max", "median", "mean"],
+        }
     )
 
 .. raw:: html

doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst

@@ -304,7 +304,8 @@ whereas a tuple of lists refer to several values within a level:
 .. ipython:: python
 
    s = pd.Series(
-       [1, 2, 3, 4, 5, 6], index=pd.MultiIndex.from_product([["A", "B"], ["c", "d", "e"]])
+       [1, 2, 3, 4, 5, 6],
+       index=pd.MultiIndex.from_product([["A", "B"], ["c", "d", "e"]]),
    )
    s.loc[[("A", "c"), ("B", "d")]]  # list of tuples
    s.loc[(["A", "B"], ["c", "d"])]  # tuple of lists
@@ -819,7 +820,9 @@ values **not** in the categories, similarly to how you can reindex **any** panda
 
 .. ipython:: python
 
-   df3 = pd.DataFrame({"A": np.arange(3), "B": pd.Series(list("abc")).astype("category")})
+   df3 = pd.DataFrame(
+       {"A": np.arange(3), "B": pd.Series(list("abc")).astype("category")}
+   )
    df3 = df3.set_index("B")
    df3
 
@@ -934,7 +937,9 @@ example, be millisecond offsets.
                np.random.randn(5, 2), index=np.arange(5) * 250.0, columns=list("AB")
            ),
            pd.DataFrame(
-               np.random.randn(6, 2), index=np.arange(4, 10) * 250.1, columns=list("AB")
+               np.random.randn(6, 2),
+               index=np.arange(4, 10) * 250.1,
+               columns=list("AB"),
            ),
        ]
    )

doc/source/user_guide/advanced.rst

@@ -464,7 +464,10 @@ which we illustrate:
        {"A": [1.0, np.nan, 3.0, 5.0, np.nan], "B": [np.nan, 2.0, 3.0, np.nan, 6.0]}
    )
    df2 = pd.DataFrame(
-       {"A": [5.0, 2.0, 4.0, np.nan, 3.0, 7.0], "B": [np.nan, np.nan, 3.0, 4.0, 6.0, 8.0]}
+       {
+           "A": [5.0, 2.0, 4.0, np.nan, 3.0, 7.0],
+           "B": [np.nan, np.nan, 3.0, 4.0, 6.0, 8.0],
+       }
    )
    df1
    df2
@@ -712,7 +715,10 @@ Similarly, you can get the most frequently occurring value(s), i.e. the mode, of
     s5 = pd.Series([1, 1, 3, 3, 3, 5, 5, 7, 7, 7])
     s5.mode()
     df5 = pd.DataFrame(
-        {"A": np.random.randint(0, 7, size=50), "B": np.random.randint(-10, 15, size=50)}
+        {
+            "A": np.random.randint(0, 7, size=50),
+            "B": np.random.randint(-10, 15, size=50),
+        }
     )
     df5.mode()
 
@@ -1192,7 +1198,9 @@ to :ref:`merging/joining functionality <merging>`:
 
 .. ipython:: python
 
-   s = pd.Series(["six", "seven", "six", "seven", "six"], index=["a", "b", "c", "d", "e"])
+   s = pd.Series(
+       ["six", "seven", "six", "seven", "six"], index=["a", "b", "c", "d", "e"]
+   )
    t = pd.Series({"six": 6.0, "seven": 7.0})
    s
    s.map(t)
@@ -1494,7 +1502,9 @@ labels).
 
    df = pd.DataFrame(
        {"x": [1, 2, 3, 4, 5, 6], "y": [10, 20, 30, 40, 50, 60]},
-       index=pd.MultiIndex.from_product([["a", "b", "c"], [1, 2]], names=["let", "num"]),
+       index=pd.MultiIndex.from_product(
+           [["a", "b", "c"], [1, 2]], names=["let", "num"]
+       ),
    )
    df
    df.rename_axis(index={"let": "abc"})
@@ -1803,7 +1813,9 @@ used to sort a pandas object by its index levels.
        }
    )
 
-   unsorted_df = df.reindex(index=["a", "d", "c", "b"], columns=["three", "two", "one"])
+   unsorted_df = df.reindex(
+       index=["a", "d", "c", "b"], columns=["three", "two", "one"]
+   )
    unsorted_df
 
    # DataFrame
@@ -1849,7 +1861,9 @@ to use to determine the sorted order.
 
 .. ipython:: python
 
-   df1 = pd.DataFrame({"one": [2, 1, 1, 1], "two": [1, 3, 2, 4], "three": [5, 4, 3, 2]})
+   df1 = pd.DataFrame(
+       {"one": [2, 1, 1, 1], "two": [1, 3, 2, 4], "three": [5, 4, 3, 2]}
+   )
    df1.sort_values(by="two")
 
 The ``by`` parameter can take a list of column names, e.g.:
@@ -1994,7 +2008,9 @@ all levels to ``by``.
 
 .. ipython:: python
 
-   df1.columns = pd.MultiIndex.from_tuples([("a", "one"), ("a", "two"), ("b", "three")])
+   df1.columns = pd.MultiIndex.from_tuples(
+       [("a", "one"), ("a", "two"), ("b", "three")]
+   )
    df1.sort_values(by=("a", "two"))
 
 
@@ -2245,7 +2261,11 @@ to the correct type.
      import datetime
 
      df = pd.DataFrame(
-         [[1, 2], ["a", "b"], [datetime.datetime(2016, 3, 2), datetime.datetime(2016, 3, 2)]]
+         [
+             [1, 2],
+             ["a", "b"],
+             [datetime.datetime(2016, 3, 2), datetime.datetime(2016, 3, 2)],
+         ]
      )
      df = df.T
      df

doc/source/user_guide/basics.rst

@@ -513,7 +513,11 @@ The ordering of the categorical is determined by the ``categories`` of that colu
 
    dfs = pd.DataFrame(
        {
-           "A": pd.Categorical(list("bbeebbaa"), categories=["e", "a", "b"], ordered=True),
+           "A": pd.Categorical(
+               list("bbeebbaa"),
+               categories=["e", "a", "b"],
+               ordered=True,
+           ),
            "B": [1, 2, 1, 2, 2, 1, 2, 1],
        }
    )
@@ -642,7 +646,13 @@ Groupby will also show "unused" categories:
     df.groupby("cats").mean()
 
     cats2 = pd.Categorical(["a", "a", "b", "b"], categories=["a", "b", "c"])
-    df2 = pd.DataFrame({"cats": cats2, "B": ["c", "d", "c", "d"], "values": [1, 2, 3, 4]})
+    df2 = pd.DataFrame(
+        {
+            "cats": cats2,
+            "B": ["c", "d", "c", "d"],
+            "values": [1, 2, 3, 4],
+        }
+    )
     df2.groupby(["cats", "B"]).mean()
 
 
@@ -1115,7 +1125,11 @@ You can use ``fillna`` to handle missing values before applying a function.
 .. ipython:: python
 
     df = pd.DataFrame(
-        {"a": [1, 2, 3, 4], "b": ["a", "b", "c", "d"], "cats": pd.Categorical([1, 2, 3, 2])}
+        {
+            "a": [1, 2, 3, 4],
+            "b": ["a", "b", "c", "d"],
+            "cats": pd.Categorical([1, 2, 3, 2]),
+        }
     )
     df.apply(lambda row: type(row["cats"]), axis=1)
     df.apply(lambda col: col.dtype, axis=0)