From 5eaf56386e653087c9d541414bda23f8711279f6 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 8 Jan 2021 16:13:54 -0600 Subject: [PATCH 01/14] CI: use conda incubator in doc and benchmarks --- .github/workflows/ci.yml | 98 ++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8834bd509bf0..0d8e3f6639191 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: master + branches: [master] pull_request: branches: - master @@ -16,10 +16,6 @@ jobs: name: Checks runs-on: ubuntu-latest steps: - - - name: Setting conda path - run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - - name: Checkout uses: actions/checkout@v1 @@ -27,49 +23,58 @@ jobs: run: ci/code_checks.sh patterns if: always() - - name: Setup environment and build pandas - run: ci/setup_env.sh - if: always() + - name: Cache conda + uses: actions/cache@v1 + env: + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('${{ env.ENV_FILE }}') }} - - name: Linting + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Environment Detail run: | - source activate pandas-dev - ci/code_checks.sh lint + conda info + conda list + + - name: Build Pandas + run: | + python setup.py build_ext -j 2 + python -m pip install -e . --no-build-isolation --no-use-pep517 + + - name: Linting + run: ci/code_checks.sh lint if: always() - name: Checks on imported code - run: | - source activate pandas-dev - ci/code_checks.sh code + run: ci/code_checks.sh code if: always() - name: Running doctests - run: | - source activate pandas-dev - ci/code_checks.sh doctests + run: ci/code_checks.sh doctests if: always() - name: Docstring validation - run: | - source activate pandas-dev - ci/code_checks.sh docstrings + run: ci/code_checks.sh docstrings if: always() - name: Typing validation - run: | - source activate pandas-dev - ci/code_checks.sh typing + run: ci/code_checks.sh typing if: always() - name: Testing docstring validation script - run: | - source activate pandas-dev - pytest --capture=no --strict-markers scripts + run: pytest --capture=no --strict-markers scripts if: always() - name: Running benchmarks run: | - source activate pandas-dev cd asv_bench asv check -E existing git remote add upstream https://github.com/pandas-dev/pandas.git @@ -92,25 +97,40 @@ jobs: name: Web and docs runs-on: ubuntu-latest steps: - - - name: Setting conda path - run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - - name: Checkout uses: actions/checkout@v1 - - name: Setup environment and build pandas - run: ci/setup_env.sh + - name: Cache conda + uses: actions/cache@v1 + env: + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('${{ env.ENV_FILE }}') }} - - name: Build website + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Environment Detail run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build + conda info + conda list - - name: Build documentation + - name: Build Pandas run: | - source activate pandas-dev - doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} + python setup.py build_ext -j 2 + python -m pip install -e . --no-build-isolation --no-use-pep517 + + - name: Build website + run: python web/pandas_web.py web/pandas --target-path=web/build + + - name: Build documentation + run: doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) From cb3f9a07e37b086e8a8da7b3ae8f74153d9b5edc Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 8 Jan 2021 16:29:23 -0600 Subject: [PATCH 02/14] CI: use conda incubator in doc and benchmarks --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d8e3f6639191..5589d464da6e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,10 @@ jobs: checks: name: Checks runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: - name: Checkout uses: actions/checkout@v1 @@ -96,6 +100,10 @@ jobs: web_and_docs: name: Web and docs runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: - name: Checkout uses: actions/checkout@v1 From 6493a5a8eba3fd072ebba2ccb21d1307638bd830 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 10 Jan 2021 20:27:48 -0600 Subject: [PATCH 03/14] fix doc usage error --- doc/source/user_guide/enhancingperf.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/user_guide/enhancingperf.rst b/doc/source/user_guide/enhancingperf.rst index 42621c032416d..49bb923dbec39 100644 --- a/doc/source/user_guide/enhancingperf.rst +++ b/doc/source/user_guide/enhancingperf.rst @@ -247,7 +247,7 @@ We've gotten another big improvement. Let's check again where the time is spent: .. ipython:: python - %%prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy()) + %prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy()) As one might expect, the majority of the time is now spent in ``apply_integrate_f``, so if we wanted to make anymore efficiencies we must continue to concentrate our From 8a534db45be560c459b2135070e47c93e1357a36 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 08:52:59 -0600 Subject: [PATCH 04/14] cache --- .github/workflows/ci.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17e320b3b2a0b..fbb0500207d31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,13 +28,10 @@ jobs: if: always() - name: Cache conda - uses: actions/cache@v1 - env: - CACHE_NUMBER: 0 + uses: actions/cache@v2 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} + key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - uses: conda-incubator/setup-miniconda@v2 with: @@ -109,13 +106,10 @@ jobs: uses: actions/checkout@v1 - name: Cache conda - uses: actions/cache@v1 - env: - CACHE_NUMBER: 0 + uses: actions/cache@v2 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} + key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - uses: conda-incubator/setup-miniconda@v2 with: @@ -138,7 +132,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} + run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log ; exit ${PIPESTATUS[0]} # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) From 323daeb28bb6e041ccb1bc31edb19c2881b9dcf1 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 09:33:57 -0600 Subject: [PATCH 05/14] debug --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbb0500207d31..e3af4964cd478 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,9 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log ; exit ${PIPESTATUS[0]} + run: | + doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log + echo ${PIPESTATUS} # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) From 7e7987367093d7d9d371ff8307828d508c0deb24 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 10:16:57 -0600 Subject: [PATCH 06/14] debug --- .github/workflows/ci.yml | 4 +--- doc/source/whatsnew/v0.8.0.rst | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3af4964cd478..29f791542374e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,9 +132,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: | - doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log - echo ${PIPESTATUS} + run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index 781054fc4de7c..dc4dcdc8a7f9a 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -165,7 +165,7 @@ New plotting methods ``Series.plot`` now supports a ``secondary_y`` option: -.. code-block:: python +.. ipython:: python plt.figure() From d462f03d7618b15757945c4321774c76d2b399d3 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 10:41:41 -0600 Subject: [PATCH 07/14] debug --- doc/source/whatsnew/v0.8.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index dc4dcdc8a7f9a..d9f52b73ca10e 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -156,7 +156,7 @@ Other new features New plotting methods ~~~~~~~~~~~~~~~~~~~~ -.. code-block:: python +.. ipython:: python import pandas as pd From bcd7943fbf1cec61ba0d353788d6234abf17f960 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 16:05:04 -0600 Subject: [PATCH 08/14] revert --- doc/source/whatsnew/v0.8.0.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index d9f52b73ca10e..781054fc4de7c 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -156,7 +156,7 @@ Other new features New plotting methods ~~~~~~~~~~~~~~~~~~~~ -.. ipython:: python +.. code-block:: python import pandas as pd @@ -165,7 +165,7 @@ New plotting methods ``Series.plot`` now supports a ``secondary_y`` option: -.. ipython:: python +.. code-block:: python plt.figure() From 24bb7c171e4de6f5660b8b849b1088d3fb8948f7 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 19:41:52 -0600 Subject: [PATCH 09/14] revert --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29f791542374e..3b53ce859e932 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,7 @@ on: env: ENV_FILE: environment.yml + PANDAS_CI: 1 jobs: checks: @@ -132,7 +133,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log + run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log ; exit ${PIPESTATUS[0]} # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) From 089b8baa2eced16a686241f2e85ed9d3273371a5 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 22:56:54 -0600 Subject: [PATCH 10/14] change ipython to code-block --- doc/source/whatsnew/v0.8.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index 781054fc4de7c..490175914cef1 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -176,7 +176,7 @@ New plotting methods Vytautas Jancauskas, the 2012 GSOC participant, has added many new plot types. For example, ``'kde'`` is a new option: -.. ipython:: python +.. code-block:: python s = pd.Series( np.concatenate((np.random.randn(1000), np.random.randn(1000) * 0.5 + 3)) From d788560a4b1a7970bdcf9278d0052d6f5f09ebcf Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 13 Jan 2021 23:35:08 -0600 Subject: [PATCH 11/14] debug --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b53ce859e932..972139f2d96ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log ; exit ${PIPESTATUS[0]} + run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) From 776cb83765a4ad83582c6195d224278a04ed1f8d Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 14 Jan 2021 00:11:41 -0600 Subject: [PATCH 12/14] debug --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 972139f2d96ae..49cab85ab6daa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -l -e {0} steps: - name: Checkout @@ -133,7 +133,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log + run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log ; exit ${PIPESTATUS[0]} # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) From 8551328a4958e1dd2e8091cbf61ea7bb501514e8 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 14 Jan 2021 00:36:19 -0600 Subject: [PATCH 13/14] debug --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49cab85ab6daa..770856d12b217 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors --num-jobs 2 | tee sphinx.log ; exit ${PIPESTATUS[0]} + run: doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) From a60be156d8d3a7acf8b5ece9516ad3b49d71eff4 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 14 Jan 2021 11:18:19 -0600 Subject: [PATCH 14/14] revert doc ci --- .github/workflows/ci.yml | 41 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 770856d12b217..b551e7ded0178 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,42 +98,25 @@ jobs: web_and_docs: name: Web and docs runs-on: ubuntu-latest - defaults: - run: - shell: bash -l -e {0} - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Setting conda path + run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - - name: Environment Detail - run: | - conda info - conda list + - name: Checkout + uses: actions/checkout@v1 - - name: Build Pandas - run: | - python setup.py build_ext -j 2 - python -m pip install -e . --no-build-isolation --no-use-pep517 + - name: Setup environment and build pandas + run: ci/setup_env.sh - name: Build website - run: python web/pandas_web.py web/pandas --target-path=web/build - + run: | + source activate pandas-dev + python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} + run: | + source activate pandas-dev + doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)