Merge remote-tracking branch 'upstream/master' into tests/test_rolling_max_datetimeindex

Author: ganevgv

.github/workflows/ci.yml

@@ -8,8 +8,6 @@ on:
 
 env:
   ENV_FILE: environment.yml
-  # TODO: remove export PATH=... in each step once this works
-  # PATH: $HOME/miniconda3/bin:$PATH
 
 jobs:
   checks:
@@ -20,84 +18,80 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v1
 
+    - name: Setting conda path
+      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
       if: true
 
     - name: Setup environment and build pandas
-      run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
-        ci/setup_env.sh
+      run: ci/setup_env.sh
       if: true
 
     - name: Linting
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/code_checks.sh lint
       if: true
 
     - name: Dependencies consistency
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/code_checks.sh dependencies
       if: true
 
     - name: Checks on imported code
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/code_checks.sh code
       if: true
 
     - name: Running doctests
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/code_checks.sh doctests
       if: true
 
     - name: Docstring validation
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/code_checks.sh docstrings
       if: true
 
     - name: Typing validation
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/code_checks.sh typing
       if: true
 
     - name: Testing docstring validation script
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         pytest --capture=no --strict scripts
       if: true
 
     - name: Running benchmarks
       run: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         cd asv_bench
         asv check -E existing
         git remote add upstream https://github.com/pandas-dev/pandas.git
         git fetch upstream
         if git diff upstream/master --name-only | grep -q "^asv_bench/"; then
             asv machine --yes
-            ASV_OUTPUT="$(asv dev)"
-            if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then
-                echo "##vso[task.logissue type=error]Benchmarks run with errors"
-                echo "$ASV_OUTPUT"
+            asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
+            if grep "failed" benchmarks.log > /dev/null ; then
                 exit 1
-            else
-                echo "Benchmarks run without errors"
             fi
         else
             echo "Benchmarks did not run, no changes detected"
         fi
       if: true
+
+    - name: Publish benchmarks artifact
+      uses: actions/upload-artifact@master
+      with:
+        name: Benchmarks log
+        path: asv_bench/benchmarks.log
+      if: failure()

README.md

@@ -164,12 +164,11 @@ pip install pandas
 ```
 
 ## Dependencies
-- [NumPy](https://www.numpy.org): 1.13.3 or higher
-- [python-dateutil](https://labix.org/python-dateutil): 2.5.0 or higher
-- [pytz](https://pythonhosted.org/pytz): 2015.4 or higher
+- [NumPy](https://www.numpy.org)
+- [python-dateutil](https://labix.org/python-dateutil)
+- [pytz](https://pythonhosted.org/pytz)
 
-See the [full installation instructions](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies)
-for recommended and optional dependencies.
+See the [full installation instructions](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies) for minimum supported versions of required, recommended and optional dependencies.
 
 ## Installation from sources
 To install pandas from source you need Cython in addition to the normal

asv_bench/benchmarks/categoricals.py

@@ -14,21 +14,6 @@
         pass
 
 
-class Concat:
-    def setup(self):
-        N = 10 ** 5
-        self.s = pd.Series(list("aabbcd") * N).astype("category")
-
-        self.a = pd.Categorical(list("aabbcd") * N)
-        self.b = pd.Categorical(list("bbcdjk") * N)
-
-    def time_concat(self):
-        pd.concat([self.s, self.s])
-
-    def time_union(self):
-        union_categoricals([self.a, self.b])
-
-
 class Constructor:
     def setup(self):
         N = 10 ** 5
@@ -77,6 +62,33 @@ def time_existing_series(self):
         pd.Categorical(self.series)
 
 
+class CategoricalOps:
+    params = ["__lt__", "__le__", "__eq__", "__ne__", "__ge__", "__gt__"]
+    param_names = ["op"]
+
+    def setup(self, op):
+        N = 10 ** 5
+        self.cat = pd.Categorical(list("aabbcd") * N, ordered=True)
+
+    def time_categorical_op(self, op):
+        getattr(self.cat, op)("b")
+
+
+class Concat:
+    def setup(self):
+        N = 10 ** 5
+        self.s = pd.Series(list("aabbcd") * N).astype("category")
+
+        self.a = pd.Categorical(list("aabbcd") * N)
+        self.b = pd.Categorical(list("bbcdjk") * N)
+
+    def time_concat(self):
+        pd.concat([self.s, self.s])
+
+    def time_union(self):
+        union_categoricals([self.a, self.b])
+
+
 class ValueCounts:
 
     params = [True, False]

asv_bench/benchmarks/timeseries.py

@@ -113,7 +113,7 @@ class InferFreq:
     def setup(self, freq):
         if freq is None:
             self.idx = date_range(start="1/1/1700", freq="D", periods=10000)
-            self.idx.freq = None
+            self.idx._data._freq = None
         else:
             self.idx = date_range(start="1/1/1700", freq=freq, periods=10000)
 

azure-pipelines.yml

@@ -16,95 +16,6 @@ jobs:
     name: Windows
     vmImage: vs2017-win2016
 
-- job: 'Checks'
-  pool:
-    vmImage: ubuntu-16.04
-  timeoutInMinutes: 90
-  steps:
-  - script: |
-      echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
-      echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
-      echo '##vso[task.setvariable variable=AZURE]true'
-    displayName: 'Setting environment variables'
-
-  # Do not require a conda environment
-  - script: ci/code_checks.sh patterns
-    displayName: 'Looking for unwanted patterns'
-    condition: true
-
-  - script: |
-      sudo apt-get update
-      sudo apt-get install -y libc6-dev-i386
-      ci/setup_env.sh
-    displayName: 'Setup environment and build pandas'
-    condition: true
-
-  # Do not require pandas
-  - script: |
-      source activate pandas-dev
-      ci/code_checks.sh lint
-    displayName: 'Linting'
-    condition: true
-
-  - script: |
-      source activate pandas-dev
-      ci/code_checks.sh dependencies
-    displayName: 'Dependencies consistency'
-    condition: true
-
-  # Require pandas
-  - script: |
-      source activate pandas-dev
-      ci/code_checks.sh code
-    displayName: 'Checks on imported code'
-    condition: true
-
-  - script: |
-      source activate pandas-dev
-      ci/code_checks.sh doctests
-    displayName: 'Running doctests'
-    condition: true
-
-  - script: |
-      source activate pandas-dev
-      ci/code_checks.sh docstrings
-    displayName: 'Docstring validation'
-    condition: true
-
-  - script: |
-      source activate pandas-dev
-      ci/code_checks.sh typing
-    displayName: 'Typing validation'
-    condition: true
-
-  - script: |
-      source activate pandas-dev
-      pytest --capture=no --strict scripts
-    displayName: 'Testing docstring validation script'
-    condition: true
-
-  - script: |
-      source activate pandas-dev
-      cd asv_bench
-      asv check -E existing
-      git remote add upstream https://github.com/pandas-dev/pandas.git
-      git fetch upstream
-      if git diff upstream/master --name-only | grep -q "^asv_bench/"; then
-          asv machine --yes
-          ASV_OUTPUT="$(asv dev)"
-          if [[ $(echo "$ASV_OUTPUT" | grep "failed") ]]; then
-              echo "##vso[task.logissue type=error]Benchmarks run with errors"
-              echo "$ASV_OUTPUT"
-              exit 1
-          else
-              echo "Benchmarks run without errors"
-          fi
-      else
-          echo "Benchmarks did not run, no changes detected"
-      fi
-    displayName: 'Running benchmarks'
-    condition: true
-
 - job: 'Web_and_Docs'
   pool:
     vmImage: ubuntu-16.04

ci/azure/posix.yml

@@ -44,16 +44,13 @@ jobs:
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
-        # https://github.com/pandas-dev/pandas/issues/29432
-        # py37_np_dev:
-        #   ENV_FILE: ci/deps/azure-37-numpydev.yaml
-        #   CONDA_PY: "37"
-        #   PATTERN: "not slow and not network"
-        #   TEST_ARGS: "-W error"
-        #   PANDAS_TESTING_MODE: "deprecate"
-        #   EXTRA_APT: "xsel"
-        #   # TODO:
-        #   continueOnError: true
+        py37_np_dev:
+          ENV_FILE: ci/deps/azure-37-numpydev.yaml
+          CONDA_PY: "37"
+          PATTERN: "not slow and not network"
+          TEST_ARGS: "-W error"
+          PANDAS_TESTING_MODE: "deprecate"
+          EXTRA_APT: "xsel"
 
   steps:
     - script: |

ci/code_checks.sh

@@ -34,17 +34,13 @@ function invgrep {
     #
     # This is useful for the CI, as we want to fail if one of the patterns
     # that we want to avoid is found by grep.
-    if [[ "$AZURE" == "true" ]]; then
-        set -o pipefail
-        grep -n "$@" | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Found unwanted pattern: " $3}'
-    else
-        grep "$@"
-    fi
-    return $((! $?))
+    grep -n "$@" | sed "s/^/$INVGREP_PREPEND/" | sed "s/$/$INVGREP_APPEND/" ; EXIT_STATUS=${PIPESTATUS[0]}
+    return $((! $EXIT_STATUS))
 }
 
-if [[ "$AZURE" == "true" ]]; then
-    FLAKE8_FORMAT="##vso[task.logissue type=error;sourcepath=%(path)s;linenumber=%(row)s;columnnumber=%(col)s;code=%(code)s;]%(text)s"
+if [[ "$GITHUB_ACTIONS" == "true" ]]; then
+    FLAKE8_FORMAT="##[error]%(path)s:%(row)s:%(col)s:%(code):%(text)s"
+    INVGREP_PREPEND="##[error]"
 else
     FLAKE8_FORMAT="default"
 fi
@@ -194,15 +190,15 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' .
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for use of comment-based annotation syntax' ; echo $MSG
+    invgrep -R --include="*.py" -P '# type: (?!ignore)' pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG
-    set -o pipefail
-    if [[ "$AZURE" == "true" ]]; then
-        # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
-        ! grep -n '--exclude=*.'{svg,c,cpp,html,js} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
-    else
-        ! grep -n '--exclude=*.'{svg,c,cpp,html,js} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
-    fi
+    INVGREP_APPEND=" <- trailing whitespaces found"
+    invgrep -RI --exclude=\*.{svg,c,cpp,html,js} --exclude-dir=env "\s$" *
     RET=$(($RET + $?)) ; echo $MSG "DONE"
+    unset INVGREP_APPEND
 fi
 
 ### CODE ###

ci/deps/azure-36-32bit.yaml

@@ -3,21 +3,24 @@ channels:
   - defaults
   - conda-forge
 dependencies:
+  - python=3.6.*
+
+  # tools
+  ### Cython 0.29.13 and pytest 5.0.1 for 32 bits are not available with conda, installing below with pip instead
+  - pytest-xdist>=1.21
+  - hypothesis>=3.58.0
+  - pytest-azurepipelines
+
+  # pandas dependencies
   - attrs=19.1.0
   - gcc_linux-32
-  - gcc_linux-32
   - gxx_linux-32
   - numpy=1.14.*
   - python-dateutil
-  - python=3.6.*
   - pytz=2017.2
-  # universal
-  - pytest
-  - pytest-xdist
-  - pytest-mock
-  - pytest-azurepipelines
-  - hypothesis>=3.58.0
+
+  # see comment above
   - pip
   - pip:
-    # Anaconda doesn't build a new enough Cython
     - cython>=0.29.13
+    - pytest>=5.0.1