Merge branch 'master' into get_loc-nan

Author: proost

.github/FUNDING.yml

@@ -1,2 +1,3 @@
 custom: https://pandas.pydata.org/donate.html
+github: [numfocus]
 tidelift: pypi/pandas

.github/workflows/assign.yml

@@ -0,0 +1,15 @@
+name: Assign
+on:
+  issue_comment:
+    types: created
+
+jobs:
+  one:
+    runs-on: ubuntu-latest
+    steps:
+      - name:
+        run: |
+            if [[ "${{ github.event.comment.body }}" == "take" ]]; then
+                echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
+                curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+            fi

.github/workflows/ci.yml

@@ -0,0 +1,97 @@
+name: CI
+
+on:
+  push:
+    branches: master
+  pull_request:
+    branches: master
+
+env:
+  ENV_FILE: environment.yml
+
+jobs:
+  checks:
+    name: Checks
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Setting conda path
+      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+
+    - name: Looking for unwanted patterns
+      run: ci/code_checks.sh patterns
+      if: true
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+      if: true
+
+    - name: Linting
+      run: |
+        source activate pandas-dev
+        ci/code_checks.sh lint
+      if: true
+
+    - name: Dependencies consistency
+      run: |
+        source activate pandas-dev
+        ci/code_checks.sh dependencies
+      if: true
+
+    - name: Checks on imported code
+      run: |
+        source activate pandas-dev
+        ci/code_checks.sh code
+      if: true
+
+    - name: Running doctests
+      run: |
+        source activate pandas-dev
+        ci/code_checks.sh doctests
+      if: true
+
+    - name: Docstring validation
+      run: |
+        source activate pandas-dev
+        ci/code_checks.sh docstrings
+      if: true
+
+    - name: Typing validation
+      run: |
+        source activate pandas-dev
+        ci/code_checks.sh typing
+      if: true
+
+    - name: Testing docstring validation script
+      run: |
+        source activate pandas-dev
+        pytest --capture=no --strict scripts
+      if: true
+
+    - name: Running benchmarks
+      run: |
+        source activate pandas-dev
+        cd asv_bench
+        asv check -E existing
+        git remote add upstream https://github.com/pandas-dev/pandas.git
+        git fetch upstream
+        if git diff upstream/master --name-only | grep -q "^asv_bench/"; then
+            asv machine --yes
+            asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
+            if grep "failed" benchmarks.log > /dev/null ; then
+                exit 1
+            fi
+        else
+            echo "Benchmarks did not run, no changes detected"
+        fi
+      if: true
+
+    - name: Publish benchmarks artifact
+      uses: actions/upload-artifact@master
+      with:
+        name: Benchmarks log
+        path: asv_bench/benchmarks.log
+      if: failure()

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/python/black
-    rev: stable
+    rev: 19.10b0
     hooks:
     -   id: black
         language_version: python3.7
@@ -9,7 +9,7 @@ repos:
     hooks:
     -   id: flake8
         language: python_venv
-        additional_dependencies: [flake8-comprehensions]
+        additional_dependencies: [flake8-comprehensions>=3.1.0]
 -   repo: https://github.com/pre-commit/mirrors-isort
     rev: v4.3.20
     hooks:

.travis.yml

@@ -30,6 +30,10 @@ matrix:
       - python: 3.5
 
     include:
+    - dist: trusty
+      env:
+        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)"
+
     - dist: trusty
       env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
@@ -71,23 +75,17 @@ before_install:
   # This overrides travis and tells it to look nowhere.
   - export BOTO_CONFIG=/dev/null
 
+
 install:
   - echo "install start"
   - ci/prep_cython_cache.sh
   - ci/setup_env.sh
   - ci/submit_cython_cache.sh
   - echo "install done"
 
-before_script:
-  # display server (for clipboard functionality) needs to be started here,
-  # does not work if done in install:setup_env.sh (GH-26103)
-  - export DISPLAY=":99.0"
-  - echo "sh -e /etc/init.d/xvfb start"
-  - sh -e /etc/init.d/xvfb start
-  - sleep 3
-
 script:
   - echo "script start"
+  - echo "$JOB"
   - source activate pandas-dev
   - ci/run_tests.sh
 

Makefile

@@ -15,7 +15,7 @@ lint-diff:
 	git diff upstream/master --name-only -- "*.py" | xargs flake8
 
 black:
-	black . --exclude '(asv_bench/env|\.egg|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist|setup.py)'
+	black .
 
 develop: build
 	python -m pip install --no-build-isolation -e .

README.md

@@ -1,5 +1,5 @@
 <div align="center">
-  <img src="https://github.com/pandas-dev/pandas/blob/master/doc/logo/pandas_logo.png"><br>
+  <img src="https://dev.pandas.io/static/img/pandas.svg"><br>
 </div>
 
 -----------------
@@ -164,12 +164,11 @@ pip install pandas
 ```
 
 ## Dependencies
-- [NumPy](https://www.numpy.org): 1.13.3 or higher
-- [python-dateutil](https://labix.org/python-dateutil): 2.5.0 or higher
-- [pytz](https://pythonhosted.org/pytz): 2015.4 or higher
+- [NumPy](https://www.numpy.org)
+- [python-dateutil](https://labix.org/python-dateutil)
+- [pytz](https://pythonhosted.org/pytz)
 
-See the [full installation instructions](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies)
-for recommended and optional dependencies.
+See the [full installation instructions](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies) for minimum supported versions of required, recommended and optional dependencies.
 
 ## Installation from sources
 To install pandas from source you need Cython in addition to the normal
@@ -190,7 +189,7 @@ or for installing in [development mode](https://pip.pypa.io/en/latest/reference/
 
 
 ```sh
-python -m pip install --no-build-isolation -e .
+python -m pip install -e . --no-build-isolation --no-use-pep517
 ```
 
 If you have `make`, you can also use `make develop` to run the same command.

asv_bench/benchmarks/categoricals.py

@@ -14,21 +14,6 @@
         pass
 
 
-class Concat:
-    def setup(self):
-        N = 10 ** 5
-        self.s = pd.Series(list("aabbcd") * N).astype("category")
-
-        self.a = pd.Categorical(list("aabbcd") * N)
-        self.b = pd.Categorical(list("bbcdjk") * N)
-
-    def time_concat(self):
-        pd.concat([self.s, self.s])
-
-    def time_union(self):
-        union_categoricals([self.a, self.b])
-
-
 class Constructor:
     def setup(self):
         N = 10 ** 5
@@ -77,14 +62,41 @@ def time_existing_series(self):
         pd.Categorical(self.series)
 
 
+class CategoricalOps:
+    params = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"]
+    param_names = ["op"]
+
+    def setup(self, op):
+        N = 10 ** 5
+        self.cat = pd.Categorical(list("aabbcd") * N, ordered=True)
+
+    def time_categorical_op(self, op):
+        getattr(self.cat, op)("b")
+
+
+class Concat:
+    def setup(self):
+        N = 10 ** 5
+        self.s = pd.Series(list("aabbcd") * N).astype("category")
+
+        self.a = pd.Categorical(list("aabbcd") * N)
+        self.b = pd.Categorical(list("bbcdjk") * N)
+
+    def time_concat(self):
+        pd.concat([self.s, self.s])
+
+    def time_union(self):
+        union_categoricals([self.a, self.b])
+
+
 class ValueCounts:
 
     params = [True, False]
     param_names = ["dropna"]
 
     def setup(self, dropna):
         n = 5 * 10 ** 5
-        arr = ["s{:04d}".format(i) for i in np.random.randint(0, n // 10, size=n)]
+        arr = [f"s{i:04d}" for i in np.random.randint(0, n // 10, size=n)]
         self.ts = pd.Series(arr).astype("category")
 
     def time_value_counts(self, dropna):
@@ -102,7 +114,7 @@ def time_rendering(self):
 class SetCategories:
     def setup(self):
         n = 5 * 10 ** 5
-        arr = ["s{:04d}".format(i) for i in np.random.randint(0, n // 10, size=n)]
+        arr = [f"s{i:04d}" for i in np.random.randint(0, n // 10, size=n)]
         self.ts = pd.Series(arr).astype("category")
 
     def time_set_categories(self):
@@ -112,7 +124,7 @@ def time_set_categories(self):
 class RemoveCategories:
     def setup(self):
         n = 5 * 10 ** 5
-        arr = ["s{:04d}".format(i) for i in np.random.randint(0, n // 10, size=n)]
+        arr = [f"s{i:04d}" for i in np.random.randint(0, n // 10, size=n)]
         self.ts = pd.Series(arr).astype("category")
 
     def time_remove_categories(self):
@@ -164,9 +176,9 @@ def setup(self, dtype):
         np.random.seed(1234)
         n = 5 * 10 ** 5
         sample_size = 100
-        arr = [i for i in np.random.randint(0, n // 10, size=n)]
+        arr = list(np.random.randint(0, n // 10, size=n))
         if dtype == "object":
-            arr = ["s{:04d}".format(i) for i in arr]
+            arr = [f"s{i:04d}" for i in arr]
         self.sample = np.random.choice(arr, sample_size)
         self.series = pd.Series(arr).astype("category")
 
@@ -225,7 +237,7 @@ def setup(self, index):
         elif index == "non_monotonic":
             self.data = pd.Categorical.from_codes([0, 1, 2] * N, categories=categories)
         else:
-            raise ValueError("Invalid index param: {}".format(index))
+            raise ValueError(f"Invalid index param: {index}")
 
         self.scalar = 10000
         self.list = list(range(10000))
@@ -282,4 +294,18 @@ def time_sort_values(self):
         self.index.sort_values(ascending=False)
 
 
+class SearchSorted:
+    def setup(self):
+        N = 10 ** 5
+        self.ci = tm.makeCategoricalIndex(N).sort_values()
+        self.c = self.ci.values
+        self.key = self.ci.categories[1]
+
+    def time_categorical_index_contains(self):
+        self.ci.searchsorted(self.key)
+
+    def time_categorical_contains(self):
+        self.c.searchsorted(self.key)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/ctors.py

@@ -67,7 +67,7 @@ class SeriesConstructors:
     def setup(self, data_fmt, with_index, dtype):
         if data_fmt in (gen_of_str, gen_of_tuples) and with_index:
             raise NotImplementedError(
-                "Series constructors do not support " "using generators with indexes"
+                "Series constructors do not support using generators with indexes"
             )
         N = 10 ** 4
         if dtype == "float":

asv_bench/benchmarks/eval.py

@@ -27,7 +27,7 @@ def time_add(self, engine, threads):
 
     def time_and(self, engine, threads):
         pd.eval(
-            "(self.df > 0) & (self.df2 > 0) & " "(self.df3 > 0) & (self.df4 > 0)",
+            "(self.df > 0) & (self.df2 > 0) & (self.df3 > 0) & (self.df4 > 0)",
             engine=engine,
         )
 

asv_bench/benchmarks/frame_ctor.py

@@ -99,7 +99,7 @@ class FromLists:
     def setup(self):
         N = 1000
         M = 100
-        self.data = [[j for j in range(M)] for i in range(N)]
+        self.data = [list(range(M)) for i in range(N)]
 
     def time_frame_from_lists(self):
         self.df = DataFrame(self.data)

asv_bench/benchmarks/frame_methods.py

@@ -565,7 +565,7 @@ def setup(self):
 
     def time_frame_get_dtype_counts(self):
         with warnings.catch_warnings(record=True):
-            self.df.get_dtype_counts()
+            self.df._data.get_dtype_counts()
 
     def time_info(self):
         self.df.info()

asv_bench/benchmarks/gil.py

@@ -37,7 +37,7 @@ def wrapper(fname):
         return wrapper
 
 
-from .pandas_vb_common import BaseIO  # noqa: E402 isort:skip
+from .pandas_vb_common import BaseIO  # isort:skip
 
 
 class ParallelGroupbyMethods:
@@ -250,13 +250,11 @@ def setup(self, dtype):
                 np.random.randn(rows, cols), index=date_range("1/1/2000", periods=rows)
             ),
             "object": DataFrame(
-                "foo",
-                index=range(rows),
-                columns=["object%03d".format(i) for i in range(5)],
+                "foo", index=range(rows), columns=["object%03d" for _ in range(5)]
             ),
         }
 
-        self.fname = "__test_{}__.csv".format(dtype)
+        self.fname = f"__test_{dtype}__.csv"
         df = data[dtype]
         df.to_csv(self.fname)