From f48827df949bc97a0ed04dd115a881899f21da27 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Thu, 21 Apr 2016 16:07:58 +0100 Subject: [PATCH 01/52] Use sudo: required to stop travis building on container-based infrastructure --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1f2940404eed0..92ee884d6db1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ - +sudo: required language: python env: From 712e5926d9a6078d92cfd990a57ae63bd4b8110e Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 8 May 2016 22:29:01 +0100 Subject: [PATCH 02/52] First go to try locale settings on travis --- .travis.yml | 15 ++++++++++++++- ci/install_travis.sh | 29 +++++++++++++++-------------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 92ee884d6db1a..615fd12daa75e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: required +sudo: false language: python env: @@ -36,6 +36,11 @@ matrix: - LOCALE_OVERRIDE="zh_CN.GB18030" - FULL_DEPS=true - JOB_TAG=_LOCALE + addons: + apt: + packages: + - language-pack-zh + - python: 2.7 env: - JOB_NAME: "27_nslow" @@ -89,6 +94,10 @@ matrix: - LOCALE_OVERRIDE="it_IT.UTF-8" - INSTALL_TEST=true - JOB_TAG=_COMPAT + addons: + apt: + packages: + - language-pack-it - python: 2.7 env: - JOB_NAME: "doc_build" @@ -129,6 +138,10 @@ matrix: - LOCALE_OVERRIDE="it_IT.UTF-8" - INSTALL_TEST=true - JOB_TAG=_COMPAT + addons: + apt: + packages: + - language-pack-it - python: 2.7 env: - JOB_NAME: "doc_build" diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 07d33f56beba8..5e60dfe0cbdd8 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -34,20 +34,21 @@ python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" home_dir=$(pwd) echo "home_dir: [$home_dir]" -if [ -n "$LOCALE_OVERRIDE" ]; then - # make sure the locale is available - # probably useless, since you would need to relogin - time sudo locale-gen "$LOCALE_OVERRIDE" - - # Need to enable for locale testing. The location of the locale file(s) is - # distro specific. For example, on Arch Linux all of the locales are in a - # commented file--/etc/locale.gen--that must be commented in to be used - # whereas Ubuntu looks in /var/lib/locales/supported.d/* and generates locales - # based on what's in the files in that folder - time echo 'it_CH.UTF-8 UTF-8' | sudo tee -a /var/lib/locales/supported.d/it - time sudo locale-gen - -fi +# Not going to work in containers +#if [ -n "$LOCALE_OVERRIDE" ]; then +# # make sure the locale is available +# # probably useless, since you would need to relogin +# time sudo locale-gen "$LOCALE_OVERRIDE" +# +# # Need to enable for locale testing. The location of the locale file(s) is +# # distro specific. For example, on Arch Linux all of the locales are in a +# # commented file--/etc/locale.gen--that must be commented in to be used +# # whereas Ubuntu looks in /var/lib/locales/supported.d/* and generates locales +# # based on what's in the files in that folder +# time echo 'it_CH.UTF-8 UTF-8' | sudo tee -a /var/lib/locales/supported.d/it +# time sudo locale-gen +# +#fi # install gui for clipboard testing if [ -n "$CLIPBOARD_GUI" ]; then From 01bfe3f3aa1280595f78e74c9612bc7fea9756b1 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 8 May 2016 22:37:43 +0100 Subject: [PATCH 03/52] lanuage-pack-zh is not allowed try -chan --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 615fd12daa75e..7f252ccc15153 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: addons: apt: packages: - - language-pack-zh + - language-pack-zh-chan - python: 2.7 env: From 65707fc32ced25ffc706f6712c533b3e7fc339a0 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 8 May 2016 23:38:14 +0100 Subject: [PATCH 04/52] Fix spelling --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7f252ccc15153..56aeb94f1463d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: addons: apt: packages: - - language-pack-zh-chan + - language-pack-zh-hans - python: 2.7 env: From 70c429b7f9c48c40099b7f9353bf95f65d6f7202 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 8 May 2016 23:56:18 +0100 Subject: [PATCH 05/52] Test zh_CN.UTF-8 as that's allowed to be installed --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 56aeb94f1463d..25e19ec3d5944 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,8 @@ matrix: env: - JOB_NAME: "27_slow_nnet_LOCALE" - NOSE_ARGS="slow and not network and not disabled" - - LOCALE_OVERRIDE="zh_CN.GB18030" +# - LOCALE_OVERRIDE="zh_CN.GB18030" + - LOCALE_OVERRIDE="zh_CN.UTF-8" - FULL_DEPS=true - JOB_TAG=_LOCALE addons: From 09457f3ddf45a463db4b748fbac2baf79efab028 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Mon, 9 May 2016 22:37:02 +0100 Subject: [PATCH 06/52] Remove more sudo --- .travis.yml | 32 ++++++++++++++++++++++++++++++-- ci/before_install_travis.sh | 3 ++- ci/install-3.5_NUMPY_DEV.sh | 3 ++- ci/install_travis.sh | 23 ++++++++++++----------- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25e19ec3d5944..80db1710eaa31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,6 @@ matrix: env: - JOB_NAME: "27_slow_nnet_LOCALE" - NOSE_ARGS="slow and not network and not disabled" -# - LOCALE_OVERRIDE="zh_CN.GB18030" - LOCALE_OVERRIDE="zh_CN.UTF-8" - FULL_DEPS=true - JOB_TAG=_LOCALE @@ -41,7 +40,6 @@ matrix: apt: packages: - language-pack-zh-hans - - python: 2.7 env: - JOB_NAME: "27_nslow" @@ -49,12 +47,20 @@ matrix: - FULL_DEPS=true - CLIPBOARD_GUI=gtk2 - LINT=true + addons: + apt: + packages: + - python-gtk2 - python: 3.4 env: - JOB_NAME: "34_nslow" - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - CLIPBOARD=xsel + addons: + apt: + packages: + - xsel - python: 3.5 env: - JOB_NAME: "35_nslow" @@ -62,6 +68,10 @@ matrix: - FULL_DEPS=true - CLIPBOARD=xsel - COVERAGE=true + addons: + apt: + packages: + - xsel - python: 2.7 env: - JOB_NAME: "27_slow" @@ -75,6 +85,10 @@ matrix: - NOSE_ARGS="slow and not network and not disabled" - FULL_DEPS=true - CLIPBOARD=xsel + addons: + apt: + packages: + - xsel - python: 2.7 env: - JOB_NAME: "27_build_test_conda" @@ -88,6 +102,11 @@ matrix: - JOB_TAG=_NUMPY_DEV - NOSE_ARGS="not slow and not network and not disabled" - PANDAS_TESTING_MODE="deprecate" + addons: + apt: + packages: + - libatlas-base-dev + - gfortran - python: 2.7 env: - JOB_NAME: "27_nslow_nnet_COMPAT" @@ -119,6 +138,10 @@ matrix: - NOSE_ARGS="slow and not network and not disabled" - FULL_DEPS=true - CLIPBOARD=xsel + addons: + apt: + packages: + - xsel - python: 2.7 env: - JOB_NAME: "27_build_test_conda" @@ -132,6 +155,11 @@ matrix: - JOB_TAG=_NUMPY_DEV - NOSE_ARGS="not slow and not network and not disabled" - PANDAS_TESTING_MODE="deprecate" + addons: + apt: + packages: + - libatlas-base-dev + - gfortran - python: 2.7 env: - JOB_NAME: "27_nslow_nnet_COMPAT" diff --git a/ci/before_install_travis.sh b/ci/before_install_travis.sh index 76775ecbc78f0..55ca38b22f206 100755 --- a/ci/before_install_travis.sh +++ b/ci/before_install_travis.sh @@ -9,7 +9,8 @@ echo "inside $0" # overview if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - sudo apt-get update $APT_ARGS # run apt-get update for all versions +# Not going to work in containers +# sudo apt-get update $APT_ARGS # run apt-get update for all versions sh -e /etc/init.d/xvfb start fi diff --git a/ci/install-3.5_NUMPY_DEV.sh b/ci/install-3.5_NUMPY_DEV.sh index ecb07ca23c667..99ec0387b9d72 100644 --- a/ci/install-3.5_NUMPY_DEV.sh +++ b/ci/install-3.5_NUMPY_DEV.sh @@ -12,7 +12,8 @@ pip uninstall numpy -y # these wheels don't play nice with the conda libgfortran / openblas # time conda install -n pandas libgfortran openblas || exit 1 -time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran +# Not going to work in containers +#time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran # install numpy wheel from master pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy scipy diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 5e60dfe0cbdd8..1d912ac6b0c6f 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -50,20 +50,21 @@ echo "home_dir: [$home_dir]" # #fi +# Not going to work in containers # install gui for clipboard testing -if [ -n "$CLIPBOARD_GUI" ]; then - echo "Using CLIPBOARD_GUI: $CLIPBOARD_GUI" - [ -n "$python_major_version" ] && py="py" - python_cb_gui_pkg=python${python_major_version}-${py}${CLIPBOARD_GUI} - time sudo apt-get $APT_ARGS install $python_cb_gui_pkg -fi - +#if [ -n "$CLIPBOARD_GUI" ]; then +# echo "Using CLIPBOARD_GUI: $CLIPBOARD_GUI" +# [ -n "$python_major_version" ] && py="py" +# python_cb_gui_pkg=python${python_major_version}-${py}${CLIPBOARD_GUI} +# time sudo apt-get $APT_ARGS install $python_cb_gui_pkg +#fi +# Not going to work in containers # install a clipboard if $CLIPBOARD is not empty -if [ -n "$CLIPBOARD" ]; then - echo "Using clipboard: $CLIPBOARD" - time sudo apt-get $APT_ARGS install $CLIPBOARD -fi +#if [ -n "$CLIPBOARD" ]; then +# echo "Using clipboard: $CLIPBOARD" +# time sudo apt-get $APT_ARGS install $CLIPBOARD +#fi python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" [ "$python_major_version" == "2" ] && python_major_version="" From 483c989d2af02083ca6589e5d19c81a755698d25 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Fri, 20 May 2016 16:19:25 +0100 Subject: [PATCH 07/52] Delete sudo lines from script files --- ci/before_install_travis.sh | 3 --- ci/install-3.5_NUMPY_DEV.sh | 3 --- ci/install_travis.sh | 32 -------------------------------- 3 files changed, 38 deletions(-) diff --git a/ci/before_install_travis.sh b/ci/before_install_travis.sh index 55ca38b22f206..f90427f97d3b7 100755 --- a/ci/before_install_travis.sh +++ b/ci/before_install_travis.sh @@ -9,9 +9,6 @@ echo "inside $0" # overview if [ "${TRAVIS_OS_NAME}" == "linux" ]; then -# Not going to work in containers -# sudo apt-get update $APT_ARGS # run apt-get update for all versions - sh -e /etc/init.d/xvfb start fi diff --git a/ci/install-3.5_NUMPY_DEV.sh b/ci/install-3.5_NUMPY_DEV.sh index 99ec0387b9d72..946ec43ad9f1a 100644 --- a/ci/install-3.5_NUMPY_DEV.sh +++ b/ci/install-3.5_NUMPY_DEV.sh @@ -12,9 +12,6 @@ pip uninstall numpy -y # these wheels don't play nice with the conda libgfortran / openblas # time conda install -n pandas libgfortran openblas || exit 1 -# Not going to work in containers -#time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran - # install numpy wheel from master pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy scipy diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 1d912ac6b0c6f..46ce1164981bf 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -34,38 +34,6 @@ python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" home_dir=$(pwd) echo "home_dir: [$home_dir]" -# Not going to work in containers -#if [ -n "$LOCALE_OVERRIDE" ]; then -# # make sure the locale is available -# # probably useless, since you would need to relogin -# time sudo locale-gen "$LOCALE_OVERRIDE" -# -# # Need to enable for locale testing. The location of the locale file(s) is -# # distro specific. For example, on Arch Linux all of the locales are in a -# # commented file--/etc/locale.gen--that must be commented in to be used -# # whereas Ubuntu looks in /var/lib/locales/supported.d/* and generates locales -# # based on what's in the files in that folder -# time echo 'it_CH.UTF-8 UTF-8' | sudo tee -a /var/lib/locales/supported.d/it -# time sudo locale-gen -# -#fi - -# Not going to work in containers -# install gui for clipboard testing -#if [ -n "$CLIPBOARD_GUI" ]; then -# echo "Using CLIPBOARD_GUI: $CLIPBOARD_GUI" -# [ -n "$python_major_version" ] && py="py" -# python_cb_gui_pkg=python${python_major_version}-${py}${CLIPBOARD_GUI} -# time sudo apt-get $APT_ARGS install $python_cb_gui_pkg -#fi - -# Not going to work in containers -# install a clipboard if $CLIPBOARD is not empty -#if [ -n "$CLIPBOARD" ]; then -# echo "Using clipboard: $CLIPBOARD" -# time sudo apt-get $APT_ARGS install $CLIPBOARD -#fi - python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" [ "$python_major_version" == "2" ] && python_major_version="" From 91cc6b389a77aa1c3c106730186fa55163e30ad8 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Fri, 20 May 2016 16:20:07 +0100 Subject: [PATCH 08/52] Fix indent bug and add comments to builds which are also in allow_failures --- .travis.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80db1710eaa31..91ca04df6d644 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,12 +72,14 @@ matrix: apt: packages: - xsel +# In allow_failures - python: 2.7 env: - JOB_NAME: "27_slow" - JOB_TAG=_SLOW - NOSE_ARGS="slow and not network and not disabled" - FULL_DEPS=true +# In allow_failures - python: 3.4 env: - JOB_NAME: "34_slow" @@ -89,6 +91,7 @@ matrix: apt: packages: - xsel +# In allow_failures - python: 2.7 env: - JOB_NAME: "27_build_test_conda" @@ -96,6 +99,7 @@ matrix: - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - BUILD_TEST=true +# In allow_failures - python: 3.5 env: - JOB_NAME: "35_numpy_dev" @@ -107,6 +111,7 @@ matrix: packages: - libatlas-base-dev - gfortran +# In allow_failures - python: 2.7 env: - JOB_NAME: "27_nslow_nnet_COMPAT" @@ -118,6 +123,7 @@ matrix: apt: packages: - language-pack-it +# In allow_failures - python: 2.7 env: - JOB_NAME: "doc_build" @@ -139,9 +145,9 @@ matrix: - FULL_DEPS=true - CLIPBOARD=xsel addons: - apt: - packages: - - xsel + apt: + packages: + - xsel - python: 2.7 env: - JOB_NAME: "27_build_test_conda" @@ -156,10 +162,10 @@ matrix: - NOSE_ARGS="not slow and not network and not disabled" - PANDAS_TESTING_MODE="deprecate" addons: - apt: - packages: - - libatlas-base-dev - - gfortran + apt: + packages: + - libatlas-base-dev + - gfortran - python: 2.7 env: - JOB_NAME: "27_nslow_nnet_COMPAT" @@ -168,9 +174,9 @@ matrix: - INSTALL_TEST=true - JOB_TAG=_COMPAT addons: - apt: - packages: - - language-pack-it + apt: + packages: + - language-pack-it - python: 2.7 env: - JOB_NAME: "doc_build" From 2b191bc8e2b7362ded3699b4445bae9e07afd2dd Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 24 May 2016 12:14:31 +0100 Subject: [PATCH 09/52] Add LOCALE to print versions --- pandas/util/print_versions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/util/print_versions.py b/pandas/util/print_versions.py index 115423f3e3e22..3089968d70fbb 100644 --- a/pandas/util/print_versions.py +++ b/pandas/util/print_versions.py @@ -4,6 +4,7 @@ import struct import subprocess import codecs +import locale def get_sys_info(): @@ -46,6 +47,7 @@ def get_sys_info(): ("byteorder", "%s" % sys.byteorder), ("LC_ALL", "%s" % os.environ.get('LC_ALL', "None")), ("LANG", "%s" % os.environ.get('LANG', "None")), + ("LOCALE", "%s.%s" % locale.getlocale()), ]) except: From a7ccaee082066960cdc4844406753ebeb12dbc6a Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 24 May 2016 14:12:56 +0100 Subject: [PATCH 10/52] Print version info during test script --- ci/script.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/script.sh b/ci/script.sh index e2ba883b81883..b5971c2312547 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -17,6 +17,10 @@ if [ -n "$LOCALE_OVERRIDE" ]; then python -c "$pycmd" fi +echo "Version info:" +pycmd="import pandas; pandas.show_versions();" +python -c "$pycmd" + if [ "$BUILD_TEST" ]; then echo "We are not running nosetests as this is simply a build test." elif [ "$COVERAGE" ]; then From 5eca9ce26a1a246f04565ae5db2eab956b6b09a3 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 24 May 2016 22:55:30 +0100 Subject: [PATCH 11/52] Test blosc memory Backtrace --- pandas/util/print_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/util/print_versions.py b/pandas/util/print_versions.py index 3089968d70fbb..49cad153687da 100644 --- a/pandas/util/print_versions.py +++ b/pandas/util/print_versions.py @@ -76,7 +76,7 @@ def show_versions(as_json=False): ("patsy", lambda mod: mod.__version__), ("dateutil", lambda mod: mod.__version__), ("pytz", lambda mod: mod.VERSION), - ("blosc", lambda mod: mod.__version__), + # ("blosc", lambda mod: mod.__version__), ("bottleneck", lambda mod: mod.__version__), ("tables", lambda mod: mod.__version__), ("numexpr", lambda mod: mod.__version__), From 70f583dfaaec2d0902d221fc5b3253e06312d483 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Wed, 25 May 2016 10:41:27 +0100 Subject: [PATCH 12/52] Revert back previous change to fix in another PR --- pandas/util/print_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/util/print_versions.py b/pandas/util/print_versions.py index 49cad153687da..3089968d70fbb 100644 --- a/pandas/util/print_versions.py +++ b/pandas/util/print_versions.py @@ -76,7 +76,7 @@ def show_versions(as_json=False): ("patsy", lambda mod: mod.__version__), ("dateutil", lambda mod: mod.__version__), ("pytz", lambda mod: mod.VERSION), - # ("blosc", lambda mod: mod.__version__), + ("blosc", lambda mod: mod.__version__), ("bottleneck", lambda mod: mod.__version__), ("tables", lambda mod: mod.__version__), ("numexpr", lambda mod: mod.__version__), From 815444f580622bb28789b346c09594491086ef08 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Thu, 26 May 2016 22:45:43 +0100 Subject: [PATCH 13/52] Experiment adding miniconda to cache --- .travis.yml | 5 ++++- ci/install_travis.sh | 32 +++++++++++++++++++------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 91ca04df6d644..a0ca6b6b44cb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,11 @@ sudo: false language: python -env: +cache: + directories: + - $HOME/miniconda +env: global: # scatterci API key #- secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ=" diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 46ce1164981bf..aea7ada5c7d1d 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -37,22 +37,28 @@ echo "home_dir: [$home_dir]" python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" [ "$python_major_version" == "2" ] && python_major_version="" -# install miniconda -if [ "${TRAVIS_OS_NAME}" == "osx" ]; then - wget http://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh || exit 1 -else - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh || exit 1 -fi -bash miniconda.sh -b -p $HOME/miniconda || exit 1 +MINICONDA_DIR="$HOME/miniconda" -conda config --set always_yes yes --set changeps1 no || exit 1 -conda update -q conda || exit 1 -conda config --add channels http://conda.anaconda.org/pandas || exit 1 -conda config --set ssl_verify false || exit 1 +if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ]; then + echo "Miniconda install already present from cache: $MINICONDA_DIR" +else + rm -rf "$MINICONDA_DIR" + # install miniconda + if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + wget http://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh || exit 1 + else + wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh || exit 1 + fi + bash miniconda.sh -b -p MINICONDA_DIR || exit 1 -# Useful for debugging any issues with conda -conda info -a || exit 1 + conda config --set always_yes yes --set changeps1 no || exit 1 + conda update -q conda || exit 1 + conda config --add channels http://conda.anaconda.org/pandas || exit 1 + conda config --set ssl_verify false || exit 1 + # Useful for debugging any issues with conda + conda info -a || exit 1 +fi # build deps REQ="ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.build" time conda create -n pandas python=$TRAVIS_PYTHON_VERSION nose coverage flake8 || exit 1 From 3b191dc4d38398564b9789bda568a5db6af4f6b2 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Thu, 26 May 2016 22:50:42 +0100 Subject: [PATCH 14/52] Fix missing $ --- ci/install_travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index aea7ada5c7d1d..70fbe9a1b005a 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -49,7 +49,7 @@ else else wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh || exit 1 fi - bash miniconda.sh -b -p MINICONDA_DIR || exit 1 + bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1 conda config --set always_yes yes --set changeps1 no || exit 1 conda update -q conda || exit 1 From e9874ec95939ffe941455b831b7ce556bf93a94d Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Fri, 27 May 2016 10:01:55 +0100 Subject: [PATCH 15/52] Experiment with caching --- .travis.yml | 110 ++++++++++++++++++++++++++------------------ ci/prep_ccache.sh | 59 +++++++++--------------- ci/submit_ccache.sh | 34 +++----------- 3 files changed, 94 insertions(+), 109 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0ca6b6b44cb4..aedfac0430a0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ language: python cache: directories: - $HOME/miniconda + - $HOME/.cache env: global: @@ -32,45 +33,50 @@ matrix: - BUILD_TYPE=conda - JOB_TAG=_OSX - TRAVIS_PYTHON_VERSION=3.5 + - CACHE_NAME="35_osx" - python: 2.7 env: - - JOB_NAME: "27_slow_nnet_LOCALE" - - NOSE_ARGS="slow and not network and not disabled" - - LOCALE_OVERRIDE="zh_CN.UTF-8" - - FULL_DEPS=true - - JOB_TAG=_LOCALE + - JOB_NAME: "27_slow_nnet_LOCALE" + - NOSE_ARGS="slow and not network and not disabled" + - LOCALE_OVERRIDE="zh_CN.UTF-8" + - FULL_DEPS=true + - JOB_TAG=_LOCALE + - CACHE_NAME="27_slow_nnet_LOCALE" addons: apt: packages: - language-pack-zh-hans - python: 2.7 env: - - JOB_NAME: "27_nslow" - - NOSE_ARGS="not slow and not disabled" - - FULL_DEPS=true - - CLIPBOARD_GUI=gtk2 - - LINT=true + - JOB_NAME: "27_nslow" + - NOSE_ARGS="not slow and not disabled" + - FULL_DEPS=true + - CLIPBOARD_GUI=gtk2 + - LINT=true + - CACHE_NAME="27_nslow" addons: apt: packages: - python-gtk2 - python: 3.4 env: - - JOB_NAME: "34_nslow" - - NOSE_ARGS="not slow and not disabled" - - FULL_DEPS=true - - CLIPBOARD=xsel + - JOB_NAME: "34_nslow" + - NOSE_ARGS="not slow and not disabled" + - FULL_DEPS=true + - CLIPBOARD=xsel + - CACHE_NAME="34_nslow" addons: apt: packages: - xsel - python: 3.5 env: - - JOB_NAME: "35_nslow" - - NOSE_ARGS="not slow and not network and not disabled" - - FULL_DEPS=true - - CLIPBOARD=xsel - - COVERAGE=true + - JOB_NAME: "35_nslow" + - NOSE_ARGS="not slow and not network and not disabled" + - FULL_DEPS=true + - CLIPBOARD=xsel + - COVERAGE=true + - CACHE_NAME="35_nslow" addons: apt: packages: @@ -78,18 +84,20 @@ matrix: # In allow_failures - python: 2.7 env: - - JOB_NAME: "27_slow" - - JOB_TAG=_SLOW - - NOSE_ARGS="slow and not network and not disabled" - - FULL_DEPS=true + - JOB_NAME: "27_slow" + - JOB_TAG=_SLOW + - NOSE_ARGS="slow and not network and not disabled" + - FULL_DEPS=true + - CACHE_NAME="27_slow" # In allow_failures - python: 3.4 env: - - JOB_NAME: "34_slow" - - JOB_TAG=_SLOW - - NOSE_ARGS="slow and not network and not disabled" - - FULL_DEPS=true - - CLIPBOARD=xsel + - JOB_NAME: "34_slow" + - JOB_TAG=_SLOW + - NOSE_ARGS="slow and not network and not disabled" + - FULL_DEPS=true + - CLIPBOARD=xsel + - CACHE_NAME="34_slow" addons: apt: packages: @@ -97,18 +105,20 @@ matrix: # In allow_failures - python: 2.7 env: - - JOB_NAME: "27_build_test_conda" - - JOB_TAG=_BUILD_TEST - - NOSE_ARGS="not slow and not disabled" - - FULL_DEPS=true - - BUILD_TEST=true + - JOB_NAME: "27_build_test_conda" + - JOB_TAG=_BUILD_TEST + - NOSE_ARGS="not slow and not disabled" + - FULL_DEPS=true + - BUILD_TEST=true + - CACHE_NAME="27_build_test_conda" # In allow_failures - python: 3.5 env: - - JOB_NAME: "35_numpy_dev" - - JOB_TAG=_NUMPY_DEV - - NOSE_ARGS="not slow and not network and not disabled" - - PANDAS_TESTING_MODE="deprecate" + - JOB_NAME: "35_numpy_dev" + - JOB_TAG=_NUMPY_DEV + - NOSE_ARGS="not slow and not network and not disabled" + - PANDAS_TESTING_MODE="deprecate" + - CACHE_NAME="35_numpy_dev" addons: apt: packages: @@ -117,11 +127,12 @@ matrix: # In allow_failures - python: 2.7 env: - - JOB_NAME: "27_nslow_nnet_COMPAT" - - NOSE_ARGS="not slow and not network and not disabled" - - LOCALE_OVERRIDE="it_IT.UTF-8" - - INSTALL_TEST=true - - JOB_TAG=_COMPAT + - JOB_NAME: "27_nslow_nnet_COMPAT" + - NOSE_ARGS="not slow and not network and not disabled" + - LOCALE_OVERRIDE="it_IT.UTF-8" + - INSTALL_TEST=true + - JOB_TAG=_COMPAT + - CACHE_NAME="27_nslow_nnet_COMPAT" addons: apt: packages: @@ -129,10 +140,11 @@ matrix: # In allow_failures - python: 2.7 env: - - JOB_NAME: "doc_build" - - FULL_DEPS=true - - DOC_BUILD=true # if rst files were changed, build docs in parallel with tests - - JOB_TAG=_DOC_BUILD + - JOB_NAME: "doc_build" + - FULL_DEPS=true + - DOC_BUILD=true # if rst files were changed, build docs in parallel with tests + - JOB_TAG=_DOC_BUILD + - CACHE_NAME="doc_build" allow_failures: - python: 2.7 env: @@ -140,6 +152,7 @@ matrix: - JOB_TAG=_SLOW - NOSE_ARGS="slow and not network and not disabled" - FULL_DEPS=true + - CACHE_NAME="27_slow" - python: 3.4 env: - JOB_NAME: "34_slow" @@ -147,6 +160,7 @@ matrix: - NOSE_ARGS="slow and not network and not disabled" - FULL_DEPS=true - CLIPBOARD=xsel + - CACHE_NAME="34_slow" addons: apt: packages: @@ -158,12 +172,14 @@ matrix: - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true - BUILD_TEST=true + - CACHE_NAME="27_build_test_conda" - python: 3.5 env: - JOB_NAME: "35_numpy_dev" - JOB_TAG=_NUMPY_DEV - NOSE_ARGS="not slow and not network and not disabled" - PANDAS_TESTING_MODE="deprecate" + - CACHE_NAME="35_numpy_dev" addons: apt: packages: @@ -176,6 +192,7 @@ matrix: - LOCALE_OVERRIDE="it_IT.UTF-8" - INSTALL_TEST=true - JOB_TAG=_COMPAT + - CACHE_NAME="27_nslow_nnet_COMPAT" addons: apt: packages: @@ -186,6 +203,7 @@ matrix: - FULL_DEPS=true - DOC_BUILD=true - JOB_TAG=_DOC_BUILD + - CACHE_NAME="doc_build" before_install: - echo "before_install" diff --git a/ci/prep_ccache.sh b/ci/prep_ccache.sh index 7e586cc4d3085..da2acba8296ec 100755 --- a/ci/prep_ccache.sh +++ b/ci/prep_ccache.sh @@ -1,35 +1,19 @@ #!/bin/bash -if [ "${TRAVIS_OS_NAME}" != "linux" ]; then - echo "not using ccache on non-linux" - exit 0 -fi - -if [ "$IRON_TOKEN" ]; then - - home_dir=$(pwd) +CACHE_DIR="$HOME/.cache" - # install the compiler cache - sudo apt-get $APT_ARGS install ccache p7zip-full - # iron_cache, pending py3 fixes upstream - pip install -I --allow-external --allow-insecure git+https://github.com/iron-io/iron_cache_python.git@8a451c7d7e4d16e0c3bedffd0f280d5d9bd4fe59#egg=iron_cache +clear_cache=0 - python ci/ironcache/get.py - ccache -C +if [ -f "$CACHE_DIR/cache.tar.gz" ]; then - clear_cache=0 - if [ -f ~/ccache.7z ]; then - echo "Cache retrieved" - clear_cache=1 - cd $HOME - 7za e $HOME/ccache.7z - # ls -l $HOME - cd / - tar xvf $HOME/ccache - rm -rf $HOME/ccache.7z - rm -rf $HOME/ccache + home_dir=$(pwd) - fi + echo "Cache retrieved" + clear_cache=1 + cd $HOME + # ls -l $HOME + cd / + tar xzvf $CACHE_DIR/cache.tar.gz # did the last commit change cython files? cd $home_dir @@ -37,16 +21,19 @@ if [ "$IRON_TOKEN" ]; then retval=$(git diff HEAD~3 --numstat | grep -P "pyx|pxd"|wc -l) echo "number of cython files changed: $retval" - if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] - then - # nope, reuse cython files - echo "Will reuse cached cython file" - touch "$TRAVIS_BUILD_DIR"/pandas/*.c - touch "$TRAVIS_BUILD_DIR"/pandas/src/*.c - touch "$TRAVIS_BUILD_DIR"/pandas/*.cpp - else - echo "Rebuilding cythonized files" - fi fi +if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] +then + # nope, reuse cython files + echo "Will reuse cached cython file" + touch "$TRAVIS_BUILD_DIR"/pandas/*.c + touch "$TRAVIS_BUILD_DIR"/pandas/src/*.c + touch "$TRAVIS_BUILD_DIR"/pandas/*.cpp +else + echo "Rebuilding cythonized files" + rm -rf $CACHE_DIR/cache.tar.gz +fi + + exit 0 diff --git a/ci/submit_ccache.sh b/ci/submit_ccache.sh index 7630bb7cc2760..df70bbac8d5d9 100755 --- a/ci/submit_ccache.sh +++ b/ci/submit_ccache.sh @@ -1,39 +1,19 @@ #!/bin/bash -if [ "${TRAVIS_OS_NAME}" != "linux" ]; then - echo "not using ccache on non-linux" - exit 0 -fi - -if [ "$IRON_TOKEN" ]; then - - home_dir=$(pwd) - ccache -s +CACHE_DIR="$HOME/.cache" +home_dir=$(pwd) - MISSES=$(ccache -s | grep "cache miss" | grep -Po "\d+") - echo "MISSES: $MISSES" +if [ -f "$CACHE_DIR/cache.tar.gz" ]; then - if [ x"$MISSES" == x"0" ]; then - echo "No cache misses detected, skipping upload" - exit 0 - fi + echo "Not creating cache - file exists" - # install the compiler cache - sudo apt-get $APT_ARGS install ccache p7zip-full - # iron_cache, pending py3 fixes upstream - pip install -I --allow-external --allow-insecure git+https://github.com/iron-io/iron_cache_python.git@8a451c7d7e4d16e0c3bedffd0f280d5d9bd4fe59#egg=iron_cache +else - rm -rf $HOME/ccache.7z - - tar cf - $HOME/.ccache \ + tar cfz $CACHE_DIR/cache.tar.gz \ "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing}.c \ - "$TRAVIS_BUILD_DIR"/pandas/msgpack.cpp \ - | 7za a -si $HOME/ccache.7z - - split -b 500000 -d $HOME/ccache.7z $HOME/ccache. + "$TRAVIS_BUILD_DIR"/pandas/msgpack.cpp - python ci/ironcache/put.py fi exit 0 From 25aea25bd5fc490de5c0d9b40c2ce931a8bca083 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 09:45:22 +0100 Subject: [PATCH 16/52] Change cache name --- .travis.yml | 4 ++-- ci/{prep_ccache.sh => prep_cache.sh} | 0 ci/{submit_ccache.sh => submit_cache.sh} | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) rename ci/{prep_ccache.sh => prep_cache.sh} (100%) rename ci/{submit_ccache.sh => submit_cache.sh} (91%) diff --git a/.travis.yml b/.travis.yml index aedfac0430a0b..6fb4adccb253b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -222,9 +222,9 @@ before_install: install: - echo "install start" - - ci/prep_ccache.sh + - ci/prep_cache.sh - ci/install_travis.sh - - ci/submit_ccache.sh + - ci/submit_cache.sh - echo "install done" before_script: diff --git a/ci/prep_ccache.sh b/ci/prep_cache.sh similarity index 100% rename from ci/prep_ccache.sh rename to ci/prep_cache.sh diff --git a/ci/submit_ccache.sh b/ci/submit_cache.sh similarity index 91% rename from ci/submit_ccache.sh rename to ci/submit_cache.sh index df70bbac8d5d9..9e428edc5c353 100755 --- a/ci/submit_ccache.sh +++ b/ci/submit_cache.sh @@ -9,6 +9,7 @@ if [ -f "$CACHE_DIR/cache.tar.gz" ]; then else + ls "$TRAVIS_BUILD_DIR"/pandas/" tar cfz $CACHE_DIR/cache.tar.gz \ "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing}.c \ From 30c09f44816ff3096b120e746a22d53dd7a4465f Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 09:53:32 +0100 Subject: [PATCH 17/52] Don't recreate pandas if miniconda is from cache --- ci/install_travis.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 70fbe9a1b005a..933d39b798658 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -58,10 +58,12 @@ else # Useful for debugging any issues with conda conda info -a || exit 1 + + time conda create -n pandas python=$TRAVIS_PYTHON_VERSION nose coverage flake8 || exit 1 + fi # build deps REQ="ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.build" -time conda create -n pandas python=$TRAVIS_PYTHON_VERSION nose coverage flake8 || exit 1 # may have additional installation instructions for this build INSTALL="ci/install-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.sh" From 6bc70e0321607d96182b17dbc4099f35878cbd0d Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 10:04:20 +0100 Subject: [PATCH 18/52] Bug fix --- ci/install_travis.sh | 11 ++++++++++- ci/submit_cache.sh | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 933d39b798658..3c4395a417d4f 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -41,6 +41,15 @@ MINICONDA_DIR="$HOME/miniconda" if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ]; then echo "Miniconda install already present from cache: $MINICONDA_DIR" + + conda config --set always_yes yes --set changeps1 no || exit 1 + conda update -q conda || exit 1 + conda config --add channels http://conda.anaconda.org/pandas || exit 1 + conda config --set ssl_verify false || exit 1 + + # Useful for debugging any issues with conda + conda info -a || exit 1 + else rm -rf "$MINICONDA_DIR" # install miniconda @@ -60,7 +69,7 @@ else conda info -a || exit 1 time conda create -n pandas python=$TRAVIS_PYTHON_VERSION nose coverage flake8 || exit 1 - + fi # build deps REQ="ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.build" diff --git a/ci/submit_cache.sh b/ci/submit_cache.sh index 9e428edc5c353..61236b8dbfcea 100755 --- a/ci/submit_cache.sh +++ b/ci/submit_cache.sh @@ -13,7 +13,7 @@ else tar cfz $CACHE_DIR/cache.tar.gz \ "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing}.c \ - "$TRAVIS_BUILD_DIR"/pandas/msgpack.cpp + "$TRAVIS_BUILD_DIR"/pandas/msgpack.cpp" fi From d9f0eb39f48eea294c0132474c37b12933779f32 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 10:55:18 +0100 Subject: [PATCH 19/52] Try adding ccache to travis --- .travis.yml | 1 + ci/install_travis.sh | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6fb4adccb253b..27e09944100a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ sudo: false language: python cache: + - ccache directories: - $HOME/miniconda - $HOME/.cache diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 3c4395a417d4f..f5056cd309609 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -86,14 +86,14 @@ time conda install -n pandas --file=${REQ} || exit 1 source activate pandas # set the compiler cache to work -if [[ "$IRON_TOKEN" && "${TRAVIS_OS_NAME}" == "linux" ]]; then - export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH - gcc=$(which gcc) - echo "gcc: $gcc" - ccache=$(which ccache) - echo "ccache: $ccache" - export CC='ccache gcc' -fi +#if [[ "$IRON_TOKEN" && "${TRAVIS_OS_NAME}" == "linux" ]]; then +# export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH +# gcc=$(which gcc) +# echo "gcc: $gcc" +# ccache=$(which ccache) +# echo "ccache: $ccache" +# export CC='ccache gcc' +#fi if [ "$BUILD_TEST" ]; then From f3adf0a0f32b276a9f3aa009d337d314254ab7dd Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 11:08:37 +0100 Subject: [PATCH 20/52] Bug fix --- .travis.yml | 1 - ci/install_travis.sh | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27e09944100a9..6fb4adccb253b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ sudo: false language: python cache: - - ccache directories: - $HOME/miniconda - $HOME/.cache diff --git a/ci/install_travis.sh b/ci/install_travis.sh index f5056cd309609..2ebc5265dd811 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -85,6 +85,8 @@ time conda install -n pandas --file=${REQ} || exit 1 source activate pandas +which ccache + # set the compiler cache to work #if [[ "$IRON_TOKEN" && "${TRAVIS_OS_NAME}" == "linux" ]]; then # export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH From ecde40792b4366dd02f86e154f080107fc26af81 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 11:35:12 +0100 Subject: [PATCH 21/52] Try caching ccache as well --- .travis.yml | 1 + ci/install_travis.sh | 18 ++++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6fb4adccb253b..649c33a22041c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ cache: directories: - $HOME/miniconda - $HOME/.cache + - $HOME/.ccache env: global: diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 2ebc5265dd811..cf5b3a08e6cb1 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -85,17 +85,15 @@ time conda install -n pandas --file=${REQ} || exit 1 source activate pandas -which ccache - # set the compiler cache to work -#if [[ "$IRON_TOKEN" && "${TRAVIS_OS_NAME}" == "linux" ]]; then -# export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH -# gcc=$(which gcc) -# echo "gcc: $gcc" -# ccache=$(which ccache) -# echo "ccache: $ccache" -# export CC='ccache gcc' -#fi +if [["${TRAVIS_OS_NAME}" == "linux" ]]; then + export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH + gcc=$(which gcc) + echo "gcc: $gcc" + ccache=$(which ccache) + echo "ccache: $ccache" + export CC='ccache gcc' +fi if [ "$BUILD_TEST" ]; then From cf818c1600411c7429c125d97566e765fd804f42 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 11:54:56 +0100 Subject: [PATCH 22/52] Bug fix --- ci/install_travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index cf5b3a08e6cb1..9129b58ab0beb 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -86,7 +86,7 @@ time conda install -n pandas --file=${REQ} || exit 1 source activate pandas # set the compiler cache to work -if [["${TRAVIS_OS_NAME}" == "linux" ]]; then +if ["${TRAVIS_OS_NAME}" == "linux" ]; then export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH gcc=$(which gcc) echo "gcc: $gcc" From 5659b0e4feef75b2ad7dbd002191a8c49e0b8326 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 12:14:08 +0100 Subject: [PATCH 23/52] Bug fix --- ci/install_travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 9129b58ab0beb..29421f9eceade 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -86,7 +86,7 @@ time conda install -n pandas --file=${REQ} || exit 1 source activate pandas # set the compiler cache to work -if ["${TRAVIS_OS_NAME}" == "linux" ]; then +if [ "${TRAVIS_OS_NAME}" == "linux" ]; then export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH gcc=$(which gcc) echo "gcc: $gcc" From 669c115c0a1334824a40705dfbcaffbba8b60935 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 13:37:26 +0100 Subject: [PATCH 24/52] Test ccache --- ci/install_travis.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 29421f9eceade..427620708ef0f 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -87,6 +87,7 @@ source activate pandas # set the compiler cache to work if [ "${TRAVIS_OS_NAME}" == "linux" ]; then + export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH gcc=$(which gcc) echo "gcc: $gcc" From e813f2dd7e3c6c8b69724f022e9fac630a6d3edb Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 20:15:18 +0100 Subject: [PATCH 25/52] Code clean up --- .travis.yml | 14 +++++++------- ci/{prep_cache.sh => prep_cython_cache.sh} | 9 +++++---- ci/submit_cache.sh | 20 -------------------- ci/submit_cython_cache.sh | 11 +++++++++++ 4 files changed, 23 insertions(+), 31 deletions(-) rename ci/{prep_cache.sh => prep_cython_cache.sh} (81%) delete mode 100755 ci/submit_cache.sh create mode 100755 ci/submit_cython_cache.sh diff --git a/.travis.yml b/.travis.yml index 649c33a22041c..91e771ef83757 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,17 @@ language: python cache: directories: - - $HOME/miniconda - - $HOME/.cache - - $HOME/.ccache + - $HOME/miniconda # miniconda cache + - $HOME/.cache # cython cache + - $HOME/.ccache # compiler cache env: global: # scatterci API key #- secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ=" # ironcache API key - - secure: "e4eEFn9nDQc3Xa5BWYkzfX37jaWVq89XidVX+rcCNEr5OlOImvveeXnF1IzbRXznH4Sv0YsLwUd8RGUWOmyCvkONq/VJeqCHWtTMyfaCIdqSyhIP9Odz8r9ahch+Y0XFepBey92AJHmlnTh+2GjCDgIiqq4fzglojnp56Vg1ojA=" - - secure: "CjmYmY5qEu3KrvMtel6zWFEtMq8ORBeS1S1odJHnjQpbwT1KY2YFZRVlLphfyDQXSz6svKUdeRrCNp65baBzs3DQNA8lIuXGIBYFeJxqVGtYAZZs6+TzBPfJJK798sGOj5RshrOJkFG2rdlWNuTq/XphI0JOrN3nPUkRrdQRpAw=" + #- secure: "e4eEFn9nDQc3Xa5BWYkzfX37jaWVq89XidVX+rcCNEr5OlOImvveeXnF1IzbRXznH4Sv0YsLwUd8RGUWOmyCvkONq/VJeqCHWtTMyfaCIdqSyhIP9Odz8r9ahch+Y0XFepBey92AJHmlnTh+2GjCDgIiqq4fzglojnp56Vg1ojA=" + #- secure: "CjmYmY5qEu3KrvMtel6zWFEtMq8ORBeS1S1odJHnjQpbwT1KY2YFZRVlLphfyDQXSz6svKUdeRrCNp65baBzs3DQNA8lIuXGIBYFeJxqVGtYAZZs6+TzBPfJJK798sGOj5RshrOJkFG2rdlWNuTq/XphI0JOrN3nPUkRrdQRpAw=" # pandas-docs-bot GH - secure: "PCzUFR8CHmw9lH84p4ygnojdF7Z8U5h7YfY0RyT+5K/aiQ1ZTU3ZkDTPI0/rR5FVMxsEEKEQKMcc5fvqW0PeD7Q2wRmluloKgT9w4EVEJ1ppKf7lITPcvZR2QgVOvjv4AfDtibLHFNiaSjzoqyJVjM4igjOu8WTlF3JfZcmOQjQ=" @@ -223,9 +223,9 @@ before_install: install: - echo "install start" - - ci/prep_cache.sh + - ci/prep_cython_cache.sh - ci/install_travis.sh - - ci/submit_cache.sh + - ci/submit_cython_cache.sh - echo "install done" before_script: diff --git a/ci/prep_cache.sh b/ci/prep_cython_cache.sh similarity index 81% rename from ci/prep_cache.sh rename to ci/prep_cython_cache.sh index da2acba8296ec..d81b7d1edd615 100755 --- a/ci/prep_cache.sh +++ b/ci/prep_cython_cache.sh @@ -1,10 +1,10 @@ #!/bin/bash -CACHE_DIR="$HOME/.cache" +CACHE_File="$HOME/.cache/cython_files.tar" clear_cache=0 -if [ -f "$CACHE_DIR/cache.tar.gz" ]; then +if [ -f "$CACHE_File" ]; then home_dir=$(pwd) @@ -13,7 +13,7 @@ if [ -f "$CACHE_DIR/cache.tar.gz" ]; then cd $HOME # ls -l $HOME cd / - tar xzvf $CACHE_DIR/cache.tar.gz + tar xvf $CACHE_File # did the last commit change cython files? cd $home_dir @@ -21,6 +21,8 @@ if [ -f "$CACHE_DIR/cache.tar.gz" ]; then retval=$(git diff HEAD~3 --numstat | grep -P "pyx|pxd"|wc -l) echo "number of cython files changed: $retval" + rm -rf $CACHE_File + fi if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] @@ -32,7 +34,6 @@ then touch "$TRAVIS_BUILD_DIR"/pandas/*.cpp else echo "Rebuilding cythonized files" - rm -rf $CACHE_DIR/cache.tar.gz fi diff --git a/ci/submit_cache.sh b/ci/submit_cache.sh deleted file mode 100755 index 61236b8dbfcea..0000000000000 --- a/ci/submit_cache.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -CACHE_DIR="$HOME/.cache" -home_dir=$(pwd) - -if [ -f "$CACHE_DIR/cache.tar.gz" ]; then - - echo "Not creating cache - file exists" - -else - - ls "$TRAVIS_BUILD_DIR"/pandas/" - tar cfz $CACHE_DIR/cache.tar.gz \ - "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ - "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing}.c \ - "$TRAVIS_BUILD_DIR"/pandas/msgpack.cpp" - -fi - -exit 0 diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh new file mode 100755 index 0000000000000..cf6657de8f21f --- /dev/null +++ b/ci/submit_cython_cache.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +CACHE_File="$HOME/.cache/cython_files.tar" +home_dir=$(pwd) +ls "$TRAVIS_BUILD_DIR"/pandas/" +tar cf $CACHE_DIR/cython_files.tar \ +"$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ +"$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,}.c \ +"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker,period_helper}.cpp" + +exit 0 From db0cd90df7bf96b40cc73a0823af3b578961e2e1 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 22:19:15 +0100 Subject: [PATCH 26/52] Test commit --- ci/install_travis.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 427620708ef0f..8dc6f13b5a724 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -67,7 +67,6 @@ else # Useful for debugging any issues with conda conda info -a || exit 1 - time conda create -n pandas python=$TRAVIS_PYTHON_VERSION nose coverage flake8 || exit 1 fi From 6201825e4fc946279fa48990759f464ff8c286bf Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 22:38:01 +0100 Subject: [PATCH 27/52] bug fix --- ci/submit_cython_cache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index cf6657de8f21f..7b247f4841121 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -2,10 +2,10 @@ CACHE_File="$HOME/.cache/cython_files.tar" home_dir=$(pwd) -ls "$TRAVIS_BUILD_DIR"/pandas/" +ls "$TRAVIS_BUILD_DIR/pandas/" tar cf $CACHE_DIR/cython_files.tar \ "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,}.c \ -"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker,period_helper}.cpp" +"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker,period_helper}.cpp exit 0 From 9158c0b5304e8347c7a67462133ecb5a52e06695 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 31 May 2016 22:48:05 +0100 Subject: [PATCH 28/52] Bug fix --- ci/submit_cython_cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index 7b247f4841121..8400b374bd1fa 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -3,7 +3,7 @@ CACHE_File="$HOME/.cache/cython_files.tar" home_dir=$(pwd) ls "$TRAVIS_BUILD_DIR/pandas/" -tar cf $CACHE_DIR/cython_files.tar \ +tar cf "$CACHE_File" \ "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,}.c \ "$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker,period_helper}.cpp From eba7f08b5e1199b8aab119011fbb28b59ec23f88 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Wed, 1 Jun 2016 09:21:32 +0100 Subject: [PATCH 29/52] Bug fix --- ci/submit_cython_cache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index 8400b374bd1fa..608fb562e72ca 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -5,7 +5,7 @@ home_dir=$(pwd) ls "$TRAVIS_BUILD_DIR/pandas/" tar cf "$CACHE_File" \ "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ -"$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,}.c \ -"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker,period_helper}.cpp +"$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,period_helper}.c \ +"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker}.cpp exit 0 From e81cd73ec114c747dc6207e14495defa7c1894d1 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sat, 4 Jun 2016 20:32:53 +0100 Subject: [PATCH 30/52] Touch msgpack --- ci/prep_cython_cache.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index d81b7d1edd615..a692dd08919f5 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -32,6 +32,7 @@ then touch "$TRAVIS_BUILD_DIR"/pandas/*.c touch "$TRAVIS_BUILD_DIR"/pandas/src/*.c touch "$TRAVIS_BUILD_DIR"/pandas/*.cpp + touch "$TRAVIS_BUILD_DIR"/pandas/msgpack/*.cpp else echo "Rebuilding cythonized files" fi From 6e4d88d0aaeddfb9736255f939a22ecd1df5efd5 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sat, 4 Jun 2016 20:56:24 +0100 Subject: [PATCH 31/52] Remove sudo even though file is not being used --- ci/install-2.7_NUMPY_DEV.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/install-2.7_NUMPY_DEV.sh b/ci/install-2.7_NUMPY_DEV.sh index 00b6255daf70f..22ac8f6547879 100644 --- a/ci/install-2.7_NUMPY_DEV.sh +++ b/ci/install-2.7_NUMPY_DEV.sh @@ -12,8 +12,6 @@ pip uninstall numpy -y # these wheels don't play nice with the conda libgfortran / openblas # time conda install -n pandas libgfortran openblas || exit 1 -time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran - # install numpy wheel from master pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy From ab55c604b4467245a5a69590d6b0b545cffb7e00 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sat, 4 Jun 2016 21:00:06 +0100 Subject: [PATCH 32/52] Change way version is displayed --- .travis.yml | 2 +- ci/script.sh | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0ca6b6b44cb4..b41d14b70d46c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -226,6 +226,6 @@ after_success: after_script: - echo "after_script start" - ci/install_test.sh - - source activate pandas && ci/print_versions.py + - source activate pandas && python -c "import pandas; pandas.show_versions();" - ci/print_skipped.py /tmp/nosetests.xml - echo "after_script done" diff --git a/ci/script.sh b/ci/script.sh index b5971c2312547..e2ba883b81883 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -17,10 +17,6 @@ if [ -n "$LOCALE_OVERRIDE" ]; then python -c "$pycmd" fi -echo "Version info:" -pycmd="import pandas; pandas.show_versions();" -python -c "$pycmd" - if [ "$BUILD_TEST" ]; then echo "We are not running nosetests as this is simply a build test." elif [ "$COVERAGE" ]; then From b73e7cc2e85c101a94b35781114aebab764c7407 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sat, 4 Jun 2016 21:14:25 +0100 Subject: [PATCH 33/52] Add use cache --- .travis.yml | 17 +++++++++++++++++ ci/install_travis.sh | 10 +++++++--- ci/prep_cython_cache.sh | 7 +++++-- ci/submit_cython_cache.sh | 18 ++++++++++-------- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 91e771ef83757..e7192995af814 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,7 @@ matrix: - JOB_TAG=_OSX - TRAVIS_PYTHON_VERSION=3.5 - CACHE_NAME="35_osx" + - USE_CACHE=true - python: 2.7 env: - JOB_NAME: "27_slow_nnet_LOCALE" @@ -43,6 +44,7 @@ matrix: - FULL_DEPS=true - JOB_TAG=_LOCALE - CACHE_NAME="27_slow_nnet_LOCALE" + - USE_CACHE=true addons: apt: packages: @@ -55,6 +57,7 @@ matrix: - CLIPBOARD_GUI=gtk2 - LINT=true - CACHE_NAME="27_nslow" + - USE_CACHE=true addons: apt: packages: @@ -66,6 +69,7 @@ matrix: - FULL_DEPS=true - CLIPBOARD=xsel - CACHE_NAME="34_nslow" + - USE_CACHE=true addons: apt: packages: @@ -78,6 +82,7 @@ matrix: - CLIPBOARD=xsel - COVERAGE=true - CACHE_NAME="35_nslow" + - USE_CACHE=true addons: apt: packages: @@ -90,6 +95,7 @@ matrix: - NOSE_ARGS="slow and not network and not disabled" - FULL_DEPS=true - CACHE_NAME="27_slow" + - USE_CACHE=true # In allow_failures - python: 3.4 env: @@ -99,6 +105,7 @@ matrix: - FULL_DEPS=true - CLIPBOARD=xsel - CACHE_NAME="34_slow" + - USE_CACHE=true addons: apt: packages: @@ -112,6 +119,7 @@ matrix: - FULL_DEPS=true - BUILD_TEST=true - CACHE_NAME="27_build_test_conda" + - USE_CACHE=true # In allow_failures - python: 3.5 env: @@ -120,6 +128,7 @@ matrix: - NOSE_ARGS="not slow and not network and not disabled" - PANDAS_TESTING_MODE="deprecate" - CACHE_NAME="35_numpy_dev" + - USE_CACHE=true addons: apt: packages: @@ -134,6 +143,7 @@ matrix: - INSTALL_TEST=true - JOB_TAG=_COMPAT - CACHE_NAME="27_nslow_nnet_COMPAT" + - USE_CACHE=true addons: apt: packages: @@ -146,6 +156,7 @@ matrix: - DOC_BUILD=true # if rst files were changed, build docs in parallel with tests - JOB_TAG=_DOC_BUILD - CACHE_NAME="doc_build" + - USE_CACHE=true allow_failures: - python: 2.7 env: @@ -154,6 +165,7 @@ matrix: - NOSE_ARGS="slow and not network and not disabled" - FULL_DEPS=true - CACHE_NAME="27_slow" + - USE_CACHE=true - python: 3.4 env: - JOB_NAME: "34_slow" @@ -162,6 +174,7 @@ matrix: - FULL_DEPS=true - CLIPBOARD=xsel - CACHE_NAME="34_slow" + - USE_CACHE=true addons: apt: packages: @@ -174,6 +187,7 @@ matrix: - FULL_DEPS=true - BUILD_TEST=true - CACHE_NAME="27_build_test_conda" + - USE_CACHE=true - python: 3.5 env: - JOB_NAME: "35_numpy_dev" @@ -181,6 +195,7 @@ matrix: - NOSE_ARGS="not slow and not network and not disabled" - PANDAS_TESTING_MODE="deprecate" - CACHE_NAME="35_numpy_dev" + - USE_CACHE=true addons: apt: packages: @@ -194,6 +209,7 @@ matrix: - INSTALL_TEST=true - JOB_TAG=_COMPAT - CACHE_NAME="27_nslow_nnet_COMPAT" + - USE_CACHE=true addons: apt: packages: @@ -205,6 +221,7 @@ matrix: - DOC_BUILD=true - JOB_TAG=_DOC_BUILD - CACHE_NAME="doc_build" + - USE_CACHE=true before_install: - echo "before_install" diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 8dc6f13b5a724..70803d236d038 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -39,7 +39,7 @@ python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" MINICONDA_DIR="$HOME/miniconda" -if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ]; then +if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ] && [ "$USE_CACHE" ]; then echo "Miniconda install already present from cache: $MINICONDA_DIR" conda config --set always_yes yes --set changeps1 no || exit 1 @@ -51,6 +51,7 @@ if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ]; then conda info -a || exit 1 else + echo "Using clean Miniconda install" rm -rf "$MINICONDA_DIR" # install miniconda if [ "${TRAVIS_OS_NAME}" == "osx" ]; then @@ -85,14 +86,17 @@ time conda install -n pandas --file=${REQ} || exit 1 source activate pandas # set the compiler cache to work -if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - +if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "$USE_CACHE" ]; then + echo "Using ccache" export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH gcc=$(which gcc) echo "gcc: $gcc" ccache=$(which ccache) echo "ccache: $ccache" export CC='ccache gcc' + +else + echo "Not using ccache" fi if [ "$BUILD_TEST" ]; then diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index a692dd08919f5..a4b54a774da9b 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -1,10 +1,11 @@ #!/bin/bash +ls "$HOME/.cache/" CACHE_File="$HOME/.cache/cython_files.tar" clear_cache=0 -if [ -f "$CACHE_File" ]; then +if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then home_dir=$(pwd) @@ -25,7 +26,7 @@ if [ -f "$CACHE_File" ]; then fi -if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] +if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] && [ "$USE_CACHE" ] then # nope, reuse cython files echo "Will reuse cached cython file" @@ -35,6 +36,8 @@ then touch "$TRAVIS_BUILD_DIR"/pandas/msgpack/*.cpp else echo "Rebuilding cythonized files" + echo "Use cache = $USE_CACHE" + echo "Clear cache = $clear_cache" fi diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index 608fb562e72ca..a27a27602d93b 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -1,11 +1,13 @@ #!/bin/bash -CACHE_File="$HOME/.cache/cython_files.tar" -home_dir=$(pwd) -ls "$TRAVIS_BUILD_DIR/pandas/" -tar cf "$CACHE_File" \ -"$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ -"$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,period_helper}.c \ -"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker}.cpp - +if [ "$USE_CACHE" ] +then + CACHE_File="$HOME/.cache/cython_files.tar" + home_dir=$(pwd) + ls "$TRAVIS_BUILD_DIR/pandas/" + tar cf "$CACHE_File" \ + "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ + "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,period_helper}.c \ + "$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker}.cpp +fi exit 0 From ba16f4aab4186c4e733f8fab7ba512ec47503798 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sat, 4 Jun 2016 20:56:24 +0100 Subject: [PATCH 34/52] Remove sudo even though file is not being used --- ci/install-2.7_NUMPY_DEV.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/install-2.7_NUMPY_DEV.sh b/ci/install-2.7_NUMPY_DEV.sh index 00b6255daf70f..22ac8f6547879 100644 --- a/ci/install-2.7_NUMPY_DEV.sh +++ b/ci/install-2.7_NUMPY_DEV.sh @@ -12,8 +12,6 @@ pip uninstall numpy -y # these wheels don't play nice with the conda libgfortran / openblas # time conda install -n pandas libgfortran openblas || exit 1 -time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran - # install numpy wheel from master pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy From 4d01ef6dde4ef37280f5fd3f40f81b2ce5e0e0fe Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sat, 4 Jun 2016 21:00:06 +0100 Subject: [PATCH 35/52] Change way version is displayed --- .travis.yml | 2 +- ci/script.sh | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index e7192995af814..d80c52d67365d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -262,6 +262,6 @@ after_success: after_script: - echo "after_script start" - ci/install_test.sh - - source activate pandas && ci/print_versions.py + - source activate pandas && python -c "import pandas; pandas.show_versions();" - ci/print_skipped.py /tmp/nosetests.xml - echo "after_script done" diff --git a/ci/script.sh b/ci/script.sh index b5971c2312547..e2ba883b81883 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -17,10 +17,6 @@ if [ -n "$LOCALE_OVERRIDE" ]; then python -c "$pycmd" fi -echo "Version info:" -pycmd="import pandas; pandas.show_versions();" -python -c "$pycmd" - if [ "$BUILD_TEST" ]; then echo "We are not running nosetests as this is simply a build test." elif [ "$COVERAGE" ]; then From 346ced8170d900fb6c885baadcc1fdee64abf2b3 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 5 Jun 2016 00:13:20 +0100 Subject: [PATCH 36/52] Grep - P is not available on mac OS X --- ci/prep_cython_cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index a4b54a774da9b..ee1064ccfe40d 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -19,7 +19,7 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then # did the last commit change cython files? cd $home_dir - retval=$(git diff HEAD~3 --numstat | grep -P "pyx|pxd"|wc -l) + retval=$(git diff HEAD~3 --numstat | grep -E "pyx|pxd"| wc -l) echo "number of cython files changed: $retval" rm -rf $CACHE_File From 56cc93c782a07e5b244fe113cd09e13dbe5a0e73 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 5 Jun 2016 11:25:34 +0100 Subject: [PATCH 37/52] Add cython files to the cache even if we have not used the cache. Therefore keeping cache up to date for if it is turned on in the future. Add comments to the travis file about use_cache. --- .travis.yml | 4 ++++ ci/submit_cython_cache.sh | 18 ++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index b3fd292e744d9..0fc567a017523 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ sudo: false language: python +# To turn off cached miniconda, cython files and compiler cache comment out the +# USE_CACHE=true line for the build in the matrix below. To delete caches go to +# https://travis-ci.org/OWNER/REPOSITORY/caches or run +# travis cache --delete inside the project directory from the travis command line client cache: directories: - $HOME/miniconda # miniconda cache diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index a27a27602d93b..608fb562e72ca 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -1,13 +1,11 @@ #!/bin/bash -if [ "$USE_CACHE" ] -then - CACHE_File="$HOME/.cache/cython_files.tar" - home_dir=$(pwd) - ls "$TRAVIS_BUILD_DIR/pandas/" - tar cf "$CACHE_File" \ - "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ - "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,period_helper}.c \ - "$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker}.cpp -fi +CACHE_File="$HOME/.cache/cython_files.tar" +home_dir=$(pwd) +ls "$TRAVIS_BUILD_DIR/pandas/" +tar cf "$CACHE_File" \ +"$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ +"$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,period_helper}.c \ +"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker}.cpp + exit 0 From ccd11258d78c6251de1f8b199a47cdeeee036554 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 5 Jun 2016 11:43:35 +0100 Subject: [PATCH 38/52] Only on back 2 commits not 3 when looking for python file changes. Make sure cython tar file is removed if it's going to be remade. --- ci/prep_cython_cache.sh | 4 ++-- ci/submit_cython_cache.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index ee1064ccfe40d..d41d76f77d72d 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -18,8 +18,8 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then # did the last commit change cython files? cd $home_dir - - retval=$(git diff HEAD~3 --numstat | grep -E "pyx|pxd"| wc -l) + # go back 2 commit + retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l) echo "number of cython files changed: $retval" rm -rf $CACHE_File diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index 608fb562e72ca..b0e82a95222e3 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -1,6 +1,7 @@ #!/bin/bash CACHE_File="$HOME/.cache/cython_files.tar" +rm -rf $CACHE_File home_dir=$(pwd) ls "$TRAVIS_BUILD_DIR/pandas/" tar cf "$CACHE_File" \ From 7d17f0077a693331fa791a8bea627f0a31a47ca3 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 5 Jun 2016 12:45:55 +0100 Subject: [PATCH 39/52] Turn off cache for 35_nslow --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0fc567a017523..bb7005b2631f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,7 +86,7 @@ matrix: - CLIPBOARD=xsel - COVERAGE=true - CACHE_NAME="35_nslow" - - USE_CACHE=true +# - USE_CACHE=true # Don't use cache for 35_nslow addons: apt: packages: From 359f00b1e31448609343ad3f95b0062dde3143a2 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 19 Jun 2016 19:07:38 +0100 Subject: [PATCH 40/52] Don't hard code in python files. Find them using the pyx files. --- ci/prep_cython_cache.sh | 23 ++++++----------------- ci/submit_cython_cache.sh | 21 ++++++++++++++++----- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index d41d76f77d72d..e798d82ec274b 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -4,36 +4,25 @@ ls "$HOME/.cache/" CACHE_File="$HOME/.cache/cython_files.tar" clear_cache=0 +home_dir=$(pwd) if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then - home_dir=$(pwd) - - echo "Cache retrieved" + echo "Cache available" clear_cache=1 - cd $HOME - # ls -l $HOME - cd / - tar xvf $CACHE_File - # did the last commit change cython files? - cd $home_dir - # go back 2 commit + # go back 2 commits retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l) echo "number of cython files changed: $retval" - - rm -rf $CACHE_File - fi if [ $clear_cache -eq 1 ] && [ $retval -eq 0 ] && [ "$USE_CACHE" ] then # nope, reuse cython files echo "Will reuse cached cython file" - touch "$TRAVIS_BUILD_DIR"/pandas/*.c - touch "$TRAVIS_BUILD_DIR"/pandas/src/*.c - touch "$TRAVIS_BUILD_DIR"/pandas/*.cpp - touch "$TRAVIS_BUILD_DIR"/pandas/msgpack/*.cpp + cd / + tar xvmf $CACHE_File + cd $home_dir else echo "Rebuilding cythonized files" echo "Use cache = $USE_CACHE" diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index b0e82a95222e3..b5ff93b2a9db9 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -2,11 +2,22 @@ CACHE_File="$HOME/.cache/cython_files.tar" rm -rf $CACHE_File + home_dir=$(pwd) -ls "$TRAVIS_BUILD_DIR/pandas/" -tar cf "$CACHE_File" \ -"$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \ -"$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing,period,period_helper}.c \ -"$TRAVIS_BUILD_DIR"/pandas/msgpack/{_packer,_unpacker}.cpp + +pyx_files=`find ${TRAVIS_BUILD_DIR} -name "*.pyx"` +echo "pyx files:" +echo $pyx_files + +tar cf ${CACHE_File} --files-from /dev/null + +for i in ${pyx_files} +do + f=${i%.pyx} + ls $f.c* | tar rf ${CACHE_File} -T - +done + +echo "Cython files in cache tar:" +tar tvf ${CACHE_File} exit 0 From bcad1887e168971bf650f326d66f7b61f18eeb57 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 19 Jun 2016 21:30:17 +0100 Subject: [PATCH 41/52] Add check cache to delete all cache if ci changes --- .travis.yml | 1 + ci/check_cache.sh | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 ci/check_cache.sh diff --git a/.travis.yml b/.travis.yml index bb7005b2631f3..bcadf2cc21882 100644 --- a/.travis.yml +++ b/.travis.yml @@ -244,6 +244,7 @@ before_install: install: - echo "install start" + - ci/check_cache.sh - ci/prep_cython_cache.sh - ci/install_travis.sh - ci/submit_cython_cache.sh diff --git a/ci/check_cache.sh b/ci/check_cache.sh new file mode 100644 index 0000000000000..81458b370cffa --- /dev/null +++ b/ci/check_cache.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +ci_changes=$(git diff HEAD~1 --numstat | grep -E "ci/"| wc -l) + +MINICONDA_DIR="$HOME/miniconda" +CACHE_DIR="$HOME/.cache" +CCACHE_DIR="$HOME/.ccache" + +if [ $ci_changes -ne 0 ] +then + echo "CI has been changed in the last commit deleting all caches" + rm -rf "$MINICONDA_DIR" + rm -rf "$CACHE_DIR" + rm -rf "$CCACHE_DIR" + if [ "$USE_CACHE" ] + then + unset USE_CACHE + fi +fi \ No newline at end of file From f607a6b9f93dc92b008b8f6cf491697d086e298b Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 19 Jun 2016 21:33:34 +0100 Subject: [PATCH 42/52] Bug fix --- ci/check_cache.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 ci/check_cache.sh diff --git a/ci/check_cache.sh b/ci/check_cache.sh old mode 100644 new mode 100755 index 81458b370cffa..3d87ee5d53c9b --- a/ci/check_cache.sh +++ b/ci/check_cache.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env bash +#!/bin/bash -ci_changes=$(git diff HEAD~1 --numstat | grep -E "ci/"| wc -l) +ci_changes=$(git diff HEAD~2 --numstat | grep -E "ci/"| wc -l) MINICONDA_DIR="$HOME/miniconda" CACHE_DIR="$HOME/.cache" @@ -8,7 +8,7 @@ CCACHE_DIR="$HOME/.ccache" if [ $ci_changes -ne 0 ] then - echo "CI has been changed in the last commit deleting all caches" + echo "CI has been changed in the last two commits deleting all caches" rm -rf "$MINICONDA_DIR" rm -rf "$CACHE_DIR" rm -rf "$CCACHE_DIR" From f0cba140920337bcf5e09062a87cc2db06b08937 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 19 Jun 2016 21:49:15 +0100 Subject: [PATCH 43/52] Don't use ccache if not using conda cache --- ci/check_cache.sh | 4 ---- ci/install_travis.sh | 26 ++++++++++++-------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/ci/check_cache.sh b/ci/check_cache.sh index 3d87ee5d53c9b..4209b21a61b80 100755 --- a/ci/check_cache.sh +++ b/ci/check_cache.sh @@ -12,8 +12,4 @@ then rm -rf "$MINICONDA_DIR" rm -rf "$CACHE_DIR" rm -rf "$CCACHE_DIR" - if [ "$USE_CACHE" ] - then - unset USE_CACHE - fi fi \ No newline at end of file diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 01cd4ccbfef4d..3d9651d4f579b 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -46,8 +46,20 @@ if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ] && [ "$USE_CACHE # Useful for debugging any issues with conda conda info -a || exit 1 + # set the compiler cache to work + if [ "${TRAVIS_OS_NAME}" == "linux" ]; then + echo "Using ccache" + export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH + gcc=$(which gcc) + echo "gcc: $gcc" + ccache=$(which ccache) + echo "ccache: $ccache" + export CC='ccache gcc' + fi + else echo "Using clean Miniconda install" + echo "Not using ccache" rm -rf "$MINICONDA_DIR" # install miniconda if [ "${TRAVIS_OS_NAME}" == "osx" ]; then @@ -90,20 +102,6 @@ time conda install -n pandas --file=${REQ} || exit 1 source activate pandas -# set the compiler cache to work -if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "$USE_CACHE" ]; then - echo "Using ccache" - export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH - gcc=$(which gcc) - echo "gcc: $gcc" - ccache=$(which ccache) - echo "ccache: $ccache" - export CC='ccache gcc' - -else - echo "Not using ccache" -fi - if [ "$BUILD_TEST" ]; then # build testing From 446489c6eb76d81dab58644bd96d573a5da08850 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Sun, 19 Jun 2016 22:16:32 +0100 Subject: [PATCH 44/52] Bug fix --- .travis.yml | 1 + ci/check_cache.sh | 6 +++--- ci/submit_cython_cache.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcadf2cc21882..46327ba8f91d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ language: python # USE_CACHE=true line for the build in the matrix below. To delete caches go to # https://travis-ci.org/OWNER/REPOSITORY/caches or run # travis cache --delete inside the project directory from the travis command line client +# The cash directories will be deleted if anything in ci/ changes in a commit cache: directories: - $HOME/miniconda # miniconda cache diff --git a/ci/check_cache.sh b/ci/check_cache.sh index 4209b21a61b80..9dd0d98a29fcd 100755 --- a/ci/check_cache.sh +++ b/ci/check_cache.sh @@ -2,9 +2,9 @@ ci_changes=$(git diff HEAD~2 --numstat | grep -E "ci/"| wc -l) -MINICONDA_DIR="$HOME/miniconda" -CACHE_DIR="$HOME/.cache" -CCACHE_DIR="$HOME/.ccache" +MINICONDA_DIR="$HOME/miniconda/" +CACHE_DIR="$HOME/.cache/" +CCACHE_DIR="$HOME/.ccache/" if [ $ci_changes -ne 0 ] then diff --git a/ci/submit_cython_cache.sh b/ci/submit_cython_cache.sh index b5ff93b2a9db9..3d41d652960c9 100755 --- a/ci/submit_cython_cache.sh +++ b/ci/submit_cython_cache.sh @@ -14,7 +14,7 @@ tar cf ${CACHE_File} --files-from /dev/null for i in ${pyx_files} do f=${i%.pyx} - ls $f.c* | tar rf ${CACHE_File} -T - + ls $f.{c,cpp} | tar rf ${CACHE_File} -T - done echo "Cython files in cache tar:" From 62320f6ab5dac3fd2b9394a161be5096721a12e1 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Nimmi Date: Fri, 17 Jun 2016 18:00:14 -0400 Subject: [PATCH 45/52] BUG: fix to_datetime to handle int16 and int8 closes #13451 Author: Ravi Kumar Nimmi Closes #13464 from ravinimmi/bugfix and squashes the following commits: dc4944d [Ravi Kumar Nimmi] BUG: fix to_datetime to handle int16 and int8 --- doc/source/whatsnew/v0.18.2.txt | 1 + pandas/tseries/tests/test_timeseries.py | 27 +++++++++++++++++++++++++ pandas/tseries/tools.py | 7 ++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.18.2.txt b/doc/source/whatsnew/v0.18.2.txt index c0251f7170534..8a14765aa6df2 100644 --- a/doc/source/whatsnew/v0.18.2.txt +++ b/doc/source/whatsnew/v0.18.2.txt @@ -504,6 +504,7 @@ Bug Fixes - Bug in ``pd.to_datetime()`` when passing invalid datatypes (e.g. bool); will now respect the ``errors`` keyword (:issue:`13176`) +- Bug in ``pd.to_datetime()`` which overflowed on ``int8``, `int16`` dtypes (:issue:`13451`) - Bug in extension dtype creation where the created types were not is/identical (:issue:`13285`) - Bug in ``NaT`` - ``Period`` raises ``AttributeError`` (:issue:`13071`) diff --git a/pandas/tseries/tests/test_timeseries.py b/pandas/tseries/tests/test_timeseries.py index fcc544ec7f239..b0caa1f6a77cb 100644 --- a/pandas/tseries/tests/test_timeseries.py +++ b/pandas/tseries/tests/test_timeseries.py @@ -2563,6 +2563,33 @@ def test_dataframe(self): with self.assertRaises(ValueError): to_datetime(df2) + def test_dataframe_dtypes(self): + # #13451 + df = DataFrame({'year': [2015, 2016], + 'month': [2, 3], + 'day': [4, 5]}) + + # int16 + result = to_datetime(df.astype('int16')) + expected = Series([Timestamp('20150204 00:00:00'), + Timestamp('20160305 00:00:00')]) + assert_series_equal(result, expected) + + # mixed dtypes + df['month'] = df['month'].astype('int8') + df['day'] = df['day'].astype('int8') + result = to_datetime(df) + expected = Series([Timestamp('20150204 00:00:00'), + Timestamp('20160305 00:00:00')]) + assert_series_equal(result, expected) + + # float + df = DataFrame({'year': [2000, 2001], + 'month': [1.5, 1], + 'day': [1, 1]}) + with self.assertRaises(ValueError): + to_datetime(df) + class TestDatetimeIndex(tm.TestCase): _multiprocess_can_split_ = True diff --git a/pandas/tseries/tools.py b/pandas/tseries/tools.py index d5e87d1df2462..efb8590dfccf4 100644 --- a/pandas/tseries/tools.py +++ b/pandas/tseries/tools.py @@ -508,7 +508,12 @@ def f(value): def coerce(values): # we allow coercion to if errors allows - return to_numeric(values, errors=errors) + values = to_numeric(values, errors=errors) + + # prevent overflow in case of int8 or int16 + if com.is_integer_dtype(values): + values = values.astype('int64', copy=False) + return values values = (coerce(arg[unit_rev['year']]) * 10000 + coerce(arg[unit_rev['month']]) * 100 + From 5d0e7065b3768b69ef9d690f44396adc886d5f6e Mon Sep 17 00:00:00 2001 From: cmazzullo Date: Sat, 18 Jun 2016 11:22:02 -0400 Subject: [PATCH 46/52] BUG: df.pivot_table: margins_name ignored when aggfunc is a list closes #13354 Author: cmazzullo Closes #13435 from cmazzullo/pivot_table and squashes the following commits: 6017e25 [cmazzullo] Fixed up whitespace for pep8 b4b09bf [cmazzullo] Compared whole dataframes in test_pivot_table_margins_name_with_aggfunc_list 98ab8be [cmazzullo] Added doublequotes to variable names 669931f [cmazzullo] Added unit test for pivot_table margins_name when aggfunc is a list 0dcb78f [cmazzullo] Switched documentation to v0.18.2 d6a6036 [cmazzullo] BUG: df.pivot_table: margins_name is ignored when there aggfunc is list #13354 --- doc/source/whatsnew/v0.18.2.txt | 1 + pandas/tools/pivot.py | 2 +- pandas/tools/tests/test_pivot.py | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.18.2.txt b/doc/source/whatsnew/v0.18.2.txt index 8a14765aa6df2..575a1f3ddcfc2 100644 --- a/doc/source/whatsnew/v0.18.2.txt +++ b/doc/source/whatsnew/v0.18.2.txt @@ -497,6 +497,7 @@ Bug Fixes - Bug in ``pd.read_csv()`` that prevents ``usecols`` from being an empty set (:issue:`13402`) - Bug in ``pd.read_csv()`` with ``engine=='c'`` in which null ``quotechar`` was not accepted even though ``quoting`` was specified as ``None`` (:issue:`13411`) - Bug in ``pd.read_csv()`` with ``engine=='c'`` in which fields were not properly cast to float when quoting was specified as non-numeric (:issue:`13411`) +- Bug in ``pd.pivot_table()`` where ``margins_name`` is ignored when ``aggfunc`` is a list (:issue:`13354`) diff --git a/pandas/tools/pivot.py b/pandas/tools/pivot.py index a4e6cc404a457..e1405bc9e6add 100644 --- a/pandas/tools/pivot.py +++ b/pandas/tools/pivot.py @@ -86,7 +86,7 @@ def pivot_table(data, values=None, index=None, columns=None, aggfunc='mean', table = pivot_table(data, values=values, index=index, columns=columns, fill_value=fill_value, aggfunc=func, - margins=margins) + margins=margins, margins_name=margins_name) pieces.append(table) keys.append(func.__name__) return concat(pieces, keys=keys, axis=1) diff --git a/pandas/tools/tests/test_pivot.py b/pandas/tools/tests/test_pivot.py index 82feaae13f771..7ec4018d301af 100644 --- a/pandas/tools/tests/test_pivot.py +++ b/pandas/tools/tests/test_pivot.py @@ -779,6 +779,28 @@ def test_pivot_table_with_iterator_values(self): ) tm.assert_frame_equal(pivot_values_gen, pivot_values_list) + def test_pivot_table_margins_name_with_aggfunc_list(self): + # GH 13354 + margins_name = 'Weekly' + costs = pd.DataFrame( + {'item': ['bacon', 'cheese', 'bacon', 'cheese'], + 'cost': [2.5, 4.5, 3.2, 3.3], + 'day': ['M', 'M', 'T', 'T']} + ) + table = costs.pivot_table( + index="item", columns="day", margins=True, + margins_name=margins_name, aggfunc=[np.mean, max] + ) + ix = pd.Index( + ['bacon', 'cheese', margins_name], dtype='object', name='item' + ) + tups = [('mean', 'cost', 'M'), ('mean', 'cost', 'T'), + ('mean', 'cost', margins_name), ('max', 'cost', 'M'), + ('max', 'cost', 'T'), ('max', 'cost', margins_name)] + cols = pd.MultiIndex.from_tuples(tups, names=[None, None, 'day']) + expected = pd.DataFrame(table.values, index=ix, columns=cols) + tm.assert_frame_equal(table, expected) + class TestCrosstab(tm.TestCase): From b1127f18f4c4c00516c55ca8e1b560ccb0312925 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Mon, 20 Jun 2016 13:31:48 +0100 Subject: [PATCH 47/52] Check whole PR for python files change if we are in a PR --- ci/prep_cython_cache.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index e798d82ec274b..afd19ada65d15 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -12,7 +12,14 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then clear_cache=1 # did the last commit change cython files? # go back 2 commits - retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l) + if [ "$TRAVIS_PULL_REQUEST" == "false" ] + then + echo "Not a PR: checking for cython files changes from last 2 commits" + retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l) + else + echo "PR: checking for any cython file changes from whole PR" + retval=$(git diff --name-only master | grep -E "pyx|pxd"| wc -l) + fi echo "number of cython files changed: $retval" fi From bac40457b48b47bb81cb83190b5a64a63d832751 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 16 Jun 2016 21:13:11 -0400 Subject: [PATCH 48/52] BLD: update appveyor build to use numpy 1.11 for 3.5 build BLD: correctly install tools.tests.data closes #13472 --- appveyor.yml | 3 ++- setup.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0af538aed40dc..c424420dda666 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,7 +32,8 @@ environment: PYTHON_VERSION: "3.5" PYTHON_ARCH: "64" CONDA_PY: "35" - CONDA_NPY: "110" + CONDA_NPY: "111" + # We always use a 64-bit machine, but can build x86 distributions # with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). diff --git a/setup.py b/setup.py index adea92896d382..8f8865ecc3b7a 100755 --- a/setup.py +++ b/setup.py @@ -591,7 +591,6 @@ def pxd(name): 'tests/data/*.xlsx', 'tests/data/*.xlsm', 'tests/data/*.table', - 'tests/tools/data/*.csv', 'tests/parser/data/*.csv', 'tests/parser/data/*.gz', 'tests/parser/data/*.bz2', @@ -602,7 +601,7 @@ def pxd(name): 'tests/data/*.html', 'tests/data/html_encoding/*.html', 'tests/json/data/*.json'], - 'pandas.tools': ['tests/*.csv'], + 'pandas.tools': ['tests/data/*.csv'], 'pandas.tests': ['data/*.pickle', 'data/*.csv'], 'pandas.tests.formats': ['data/*.csv'], From 38cee16873ab77a2a3385bfb527a363f058e40cb Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 20 Jun 2016 09:34:09 -0500 Subject: [PATCH 49/52] DOC: specify correct kernelspec for converting notebooks (#13491) --- doc/make.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/make.py b/doc/make.py index 05bf618ee677e..8e7b1d95dbafb 100755 --- a/doc/make.py +++ b/doc/make.py @@ -116,6 +116,11 @@ def cleanup_nb(nb): pass +def get_kernel(): + """Find the kernel name for your python version""" + return 'python%s' % sys.version_info.major + + def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''): """ Execute notebook in `src` and write the output to `dst` @@ -184,10 +189,12 @@ def html(): with cleanup_nb(nb): try: print("Converting %s" % nb) - executed = execute_nb(nb, nb + '.executed', allow_errors=True) + kernel_name = get_kernel() + executed = execute_nb(nb, nb + '.executed', allow_errors=True, + kernel_name=kernel_name) convert_nb(executed, nb.rstrip('.ipynb') + '.html') - except ImportError: - pass + except (ImportError, IndexError): + print("Failed to convert %s" % nb) if os.system('sphinx-build -P -b html -d build/doctrees ' 'source build/html'): @@ -199,6 +206,7 @@ def html(): except: pass + def zip_html(): try: print("\nZipping up HTML docs...") From abc26c93f123d284b0820a3364e85d72fbf86a46 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Mon, 20 Jun 2016 22:14:01 +0100 Subject: [PATCH 50/52] Display output of git diff HEAD~2 --numstat | grep -E "ci/" and remove travis.yml comment --- .travis.yml | 2 +- ci/check_cache.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 46327ba8f91d3..b909a1f980d6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -158,7 +158,7 @@ matrix: env: - JOB_NAME: "doc_build" - FULL_DEPS=true - - DOC_BUILD=true # if rst files were changed, build docs in parallel with tests + - DOC_BUILD=true - JOB_TAG=_DOC_BUILD - CACHE_NAME="doc_build" - USE_CACHE=true diff --git a/ci/check_cache.sh b/ci/check_cache.sh index 9dd0d98a29fcd..700d0b22459f5 100755 --- a/ci/check_cache.sh +++ b/ci/check_cache.sh @@ -1,5 +1,6 @@ #!/bin/bash +git diff HEAD~2 --numstat | grep -E "ci/" ci_changes=$(git diff HEAD~2 --numstat | grep -E "ci/"| wc -l) MINICONDA_DIR="$HOME/miniconda/" From 58b55642ec4504f87dfdd596ec1f75d47923c144 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Mon, 20 Jun 2016 23:12:41 +0100 Subject: [PATCH 51/52] Try fetch head in pull requests --- ci/check_cache.sh | 14 +++++++++++--- ci/prep_cython_cache.sh | 6 ++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ci/check_cache.sh b/ci/check_cache.sh index 700d0b22459f5..c2b4ee494218a 100755 --- a/ci/check_cache.sh +++ b/ci/check_cache.sh @@ -1,7 +1,15 @@ #!/bin/bash -git diff HEAD~2 --numstat | grep -E "ci/" -ci_changes=$(git diff HEAD~2 --numstat | grep -E "ci/"| wc -l) +if [ "$TRAVIS_PULL_REQUEST" == "false" ] +then + echo "Not a PR: checking for changes in ci/ from last 2 commits" + git diff HEAD~2 --numstat | grep -E "ci/" + ci_changes=$(git diff HEAD~2 --numstat | grep -E "ci/"| wc -l) +else + echo "PR: checking for changes in ci/ from last 2 commits" + git diff FETCH_HEAD~2 --numstat | grep -E "ci/" + ci_changes=$(git diff FETCH_HEAD~2 --numstat | grep -E "ci/"| wc -l) +fi MINICONDA_DIR="$HOME/miniconda/" CACHE_DIR="$HOME/.cache/" @@ -9,7 +17,7 @@ CCACHE_DIR="$HOME/.ccache/" if [ $ci_changes -ne 0 ] then - echo "CI has been changed in the last two commits deleting all caches" + echo "Files have changed in ci/ deleting all caches" rm -rf "$MINICONDA_DIR" rm -rf "$CACHE_DIR" rm -rf "$CCACHE_DIR" diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index afd19ada65d15..37cdfbe8db31f 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -15,10 +15,12 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then if [ "$TRAVIS_PULL_REQUEST" == "false" ] then echo "Not a PR: checking for cython files changes from last 2 commits" + git diff HEAD~2 --numstat | grep -E "pyx|pxd" retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l) else - echo "PR: checking for any cython file changes from whole PR" - retval=$(git diff --name-only master | grep -E "pyx|pxd"| wc -l) + echo "PR: checking for any cython file changes from last 5 commits" + git diff FETCH_HEAD~5 --numstat | grep -E "pyx|pxd" + retval=$(git diff FETCH_HEAD~5 --numstat | grep -E "pyx|pxd"| wc -l) fi echo "number of cython files changed: $retval" fi From 8b72f55ce593e4c8a24ddaacb96f4211d95c5326 Mon Sep 17 00:00:00 2001 From: Neil Parley Date: Tue, 21 Jun 2016 09:24:38 +0100 Subject: [PATCH 52/52] Fetch pull request head so we can look back out the last commits to the full request --- ci/check_cache.sh | 5 +++-- ci/prep_cython_cache.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/check_cache.sh b/ci/check_cache.sh index c2b4ee494218a..cd7a6e8f6b6f9 100755 --- a/ci/check_cache.sh +++ b/ci/check_cache.sh @@ -7,8 +7,9 @@ then ci_changes=$(git diff HEAD~2 --numstat | grep -E "ci/"| wc -l) else echo "PR: checking for changes in ci/ from last 2 commits" - git diff FETCH_HEAD~2 --numstat | grep -E "ci/" - ci_changes=$(git diff FETCH_HEAD~2 --numstat | grep -E "ci/"| wc -l) + git fetch origin pull/${TRAVIS_PULL_REQUEST}/head:PR_HEAD + git diff PR_HEAD~2 --numstat | grep -E "ci/" + ci_changes=$(git diff PR_HEAD~2 --numstat | grep -E "ci/"| wc -l) fi MINICONDA_DIR="$HOME/miniconda/" diff --git a/ci/prep_cython_cache.sh b/ci/prep_cython_cache.sh index 37cdfbe8db31f..162f7a1034be6 100755 --- a/ci/prep_cython_cache.sh +++ b/ci/prep_cython_cache.sh @@ -19,8 +19,8 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ]; then retval=$(git diff HEAD~2 --numstat | grep -E "pyx|pxd"| wc -l) else echo "PR: checking for any cython file changes from last 5 commits" - git diff FETCH_HEAD~5 --numstat | grep -E "pyx|pxd" - retval=$(git diff FETCH_HEAD~5 --numstat | grep -E "pyx|pxd"| wc -l) + git diff PR_HEAD~5 --numstat | grep -E "pyx|pxd" + retval=$(git diff PR_HEAD~5 --numstat | grep -E "pyx|pxd"| wc -l) fi echo "number of cython files changed: $retval" fi