Merge pull request #107 from mattip/azure-pipelines

add azure CI, temporarily disable others

Author: mattip

.travis.yml renamed to _travis.yml.backup

@@ -4,10 +4,8 @@ env:
         # Also see DAILY_COMMIT below
         - BUILD_COMMIT=master
         - PLAT=x86_64
-        - NP_BUILD_DEP="numpy==1.16.5"
         - CYTHON_BUILD_DEP="Cython==0.29.18"
         - PYBIND11_BUILD_DEP="pybind11==2.4.3"
-        - NP_TEST_DEP="numpy==1.16.5"
         - UNICODE_WIDTH=32
         - MANYLINUX_URL="https://5cf40426d9f06eb7461d-6fe47d9331aba7cd62fc36c7196769e4.ssl.cf2.rackcdn.com"
         - WHEELHOUSE_UPLOADER_USERNAME=travis-worker
@@ -38,7 +36,6 @@ jobs:
       env:
         - MB_PYTHON_VERSION=3.7
         - PLAT=aarch64
-        - NP_BUILD_DEP=numpy==1.19.1
         - CYTHON_BUILD_DEP="Cython"
         - DOCKER_IMAGE=quay.io/pypa/manylinux2014_${PLAT}
       script:
@@ -59,7 +56,6 @@ jobs:
       env:
         - MB_PYTHON_VERSION=3.8
         - PLAT=aarch64
-        - NP_BUILD_DEP=numpy==1.19.1
         - CYTHON_BUILD_DEP="Cython"
         - DOCKER_IMAGE=quay.io/pypa/manylinux2014_${PLAT}
       script:
@@ -80,7 +76,6 @@ jobs:
       env:
         - MB_PYTHON_VERSION=3.9
         - PLAT=aarch64
-        - NP_BUILD_DEP=numpy==1.19.1
         - CYTHON_BUILD_DEP="Cython"
         - DOCKER_IMAGE=quay.io/pypa/manylinux2014_${PLAT}
       script:
@@ -92,77 +87,6 @@ jobs:
             - wheelhouse
       after_success:
         - echo "This stage will not upload aarch64 wheel"
-    - os: linux
-      name: amd64-Linux-Py39
-      env:
-        - MB_PYTHON_VERSION=3.9
-        - NP_BUILD_DEP=numpy==1.19.1
-        - NP_TEST_DEP=numpy==1.19.1
-    - os: linux
-      name: 32bit-amd64-Linux-Py39
-      env:
-        - MB_PYTHON_VERSION=3.9
-        - PLAT=i686
-        - NP_BUILD_DEP=numpy==1.19.3
-        - NP_TEST_DEP=numpy==1.19.3
-    - os: linux
-      name: amd64-Linux-Py37
-      env:
-        - MB_PYTHON_VERSION=3.7
-        - NP_BUILD_DEP=numpy==1.16.5
-        - NP_TEST_DEP=numpy==1.16.5
-        - CYTHON_BUILD_DEP="Cython"
-    - os: linux
-      name: amd64-Linux-Py38
-      env:
-        - MB_PYTHON_VERSION=3.8
-        - NP_BUILD_DEP=numpy==1.17.3
-        - NP_TEST_DEP=numpy==1.17.3
-        - CYTHON_BUILD_DEP="Cython"
-    - os: linux
-      name: 32bit-amd64-Linux-Py38
-      env:
-        - MB_PYTHON_VERSION=3.8
-        - PLAT=i686
-        - NP_BUILD_DEP=numpy==1.17.3
-        - NP_TEST_DEP=numpy==1.17.3
-        - CYTHON_BUILD_DEP="Cython"
-    - os: linux
-      name: 32bit-amd64-Linux-Py37
-      env:
-        - MB_PYTHON_VERSION=3.7
-        - PLAT=i686
-        - NP_BUILD_DEP=numpy==1.16.5
-        - NP_TEST_DEP=numpy==1.16.5
-        - CYTHON_BUILD_DEP="Cython"
-    - os: osx
-      name: osx-Py37
-      language: generic
-      env:
-        - MB_PYTHON_VERSION=3.7
-        - NP_BUILD_DEP=numpy==1.16.5
-        - NP_TEST_DEP=numpy==1.16.5
-        - CYTHON_BUILD_DEP="Cython"
-    - os: osx
-      name: osx-Py38
-      language: generic
-      osx_image: xcode10.1
-      env:
-        - MB_PYTHON_VERSION=3.8
-        - NP_BUILD_DEP=numpy==1.17.3
-        - NP_TEST_DEP=numpy==1.17.3
-        - CYTHON_BUILD_DEP="Cython"
-        - MB_PYTHON_OSX_VER=10.9
-    - os: osx
-      name: osx-Py39
-      language: generic
-      osx_image: xcode10.1
-      env:
-        - MB_PYTHON_VERSION=3.9
-        - NP_BUILD_DEP=numpy==1.19.1
-        - NP_TEST_DEP=numpy==1.19.1
-        - CYTHON_BUILD_DEP="Cython"
-        - MB_PYTHON_OSX_VER=10.9
 
     - stage: Test wheel
       name: ARM64-Linux-Py37
@@ -173,7 +97,6 @@ jobs:
       env:
         - MB_PYTHON_VERSION=3.7
         - PLAT=aarch64
-        - NP_TEST_DEP=numpy==1.19.1
         - CYTHON_BUILD_DEP="Cython"
         - DOCKER_TEST_IMAGE=multibuild/xenial_arm64v8
       workspaces:
@@ -188,7 +111,6 @@ jobs:
       env:
         - MB_PYTHON_VERSION=3.9
         - PLAT=aarch64
-        - NP_TEST_DEP=numpy==1.19.1
         - CYTHON_BUILD_DEP="Cython"
         - DOCKER_TEST_IMAGE=multibuild/xenial_arm64v8
       workspaces:
@@ -203,7 +125,6 @@ jobs:
       env:
         - MB_PYTHON_VERSION=3.8
         - PLAT=aarch64
-        - NP_TEST_DEP=numpy==1.19.1
         - CYTHON_BUILD_DEP="Cython"
         - DOCKER_TEST_IMAGE=multibuild/xenial_arm64v8
       workspaces:
@@ -219,8 +140,8 @@ before_install:
           CONTAINER=wheels;
           UPLOAD_ARGS="--no-update-index";
       fi
-    - BUILD_DEPENDS="$NP_BUILD_DEP $CYTHON_BUILD_DEP $PYBIND11_BUILD_DEP"
-    - TEST_DEPENDS="$NP_TEST_DEP pytest pytest-xdist pytest-faulthandler pytest-env"
+    - BUILD_DEPENDS="wheel oldest-supported-numpy $CYTHON_BUILD_DEP $PYBIND11_BUILD_DEP"
+    - TEST_DEPENDS="oldest-supported-numpy pytest pytest-xdist pytest-faulthandler pytest-env"
     - source multibuild/common_utils.sh
     - source multibuild/travis_steps.sh
     - before_install

azure-pipelines.yml

@@ -1,19 +1,50 @@
-# Starter pipeline
-# Start with a minimal pipeline that you can customize to build and deploy your code.
-# Add steps that build, run tests, deploy, and more:
-# https://aka.ms/yaml
+schedules:
+- cron: "27 3 * * 0"
+  # 3:27am UTC every Sunday
+  displayName: Weekly build
+  branches:
+    include:
+    - master
+  always: true
 
-trigger:
-- main
+pr:
+- master
 
-pool:
-  vmImage: 'ubuntu-latest'
+variables:
+  BUILD_COMMIT: "master"
 
-steps:
-- script: echo Hello, world!
-  displayName: 'Run a one-line script'
+jobs:
+  - template: azure-posix.yml
+    parameters:
+      name: linux
+      vmImage: ubuntu-18.04
+      matrix:
+        amd64-linux-py39:
+          MB_PYTHON_VERSION: "3.9"
+        32bit-amd64-linux-py39:
+          MB_PYTHON_VERSION: "3.9"
+          PLAT: "i686"
+        amd64-linux-py38:
+          MB_PYTHON_VERSION: "3.8"
+        32bit-amd64-linux-py38:
+          MB_PYTHON_VERSION: "3.8"
+          PLAT: "i686"
+        amd64-linux-py37:
+          MB_PYTHON_VERSION: "3.7"
+        32bit-amd64-linux-py37:
+          MB_PYTHON_VERSION: "3.7"
+          PLAT: "i686"
 
-- script: |
-    echo Add other tasks to build, test, and deploy your project.
-    echo See https://aka.ms/yaml
-  displayName: 'Run a multi-line script'
+  - template: azure-posix.yml
+    parameters:
+      name: macOS
+      vmImage: macOS-10.14
+      matrix:
+        osx-Py37:
+          MB_PYTHON_VERSION: "3.7"
+        osx-Py38:
+          MB_PYTHON_VERSION: "3.8"
+          MB_PYTHON_OSX_VER: "10.9"
+        osx-Py39:
+          MB_PYTHON_VERSION: "3.9"
+          MB_PYTHON_OSX_VER: "10.9"

azure-posix.yml

@@ -0,0 +1,124 @@
+parameters:
+  name: ""
+  vmImage: ""
+  matrix: []
+
+jobs:
+  - job: ${{ parameters.name }}
+    pool:
+      vmImage: ${{ parameters.vmImage }}
+    variables:
+      REPO_DIR: "scipy"
+      PLAT: "x86_64"
+      NIGHTLY_BUILD_COMMIT: "master"
+      DAILY_COMMIT: "master"
+    strategy:
+      matrix:
+        ${{ insert }}: ${{ parameters.matrix }}
+
+    steps:
+      - checkout: self
+        submodules: true
+
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $(AZURE_PYTHON_VERSION)
+        displayName: Set python version from AZURE
+        condition: ne( coalesce (variables['AZURE_PYTHON_VERSION'], 'NONE'), 'NONE')
+
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $(MB_PYTHON_VERSION)
+        displayName: Set python version from MB
+        condition: eq( coalesce (variables['AZURE_PYTHON_VERSION'], 'NONE'), 'NONE')
+
+      - bash: |
+          set -e
+
+          if [ "$BUILD_REASON" == "Schedule" ]; then
+            BUILD_COMMIT=$NIGHTLY_BUILD_COMMIT
+          fi
+          echo "Building scipy@$BUILD_COMMIT"
+          echo "##vso[task.setvariable variable=BUILD_COMMIT]$BUILD_COMMIT"
+
+          # Platform variables used in multibuild scripts
+          if [ `uname` == 'Darwin' ]; then
+            echo "##vso[task.setvariable variable=TRAVIS_OS_NAME]osx"
+            echo "##vso[task.setvariable variable=MACOSX_DEPLOYMENT_TARGET]10.9"
+          else
+            echo "##vso[task.setvariable variable=TRAVIS_OS_NAME]linux"
+          fi
+
+          # Store original Python path to be able to create test_venv pointing
+          # to same Python version.
+          PYTHON_EXE=`which python`
+          echo "##vso[task.setvariable variable=PYTHON_EXE]$PYTHON_EXE"
+        displayName: Define build env variables
+
+      - bash: |
+          set -e
+          echo building for commit "$BUILD_COMMIT"
+          pip install --upgrade virtualenv wheel setuptools
+          BUILD_DEPENDS="wheel oldest-supported-numpy Cython>=0.29.18 pybind11>=2.4.3"
+
+          source multibuild/common_utils.sh
+          source multibuild/travis_steps.sh
+
+          before_install
+
+          clean_code $REPO_DIR $BUILD_COMMIT
+          ./patch_code.sh $REPO_DIR
+          build_wheel $REPO_DIR $PLAT
+        displayName: Build wheel
+
+      - bash: |
+          set -xe
+          TEST_DEPENDS="oldest-supported-numpy pytest pytest-xdist pytest-faulthandler pytest-env"
+          source multibuild/common_utils.sh
+          source multibuild/travis_steps.sh
+          source extra_functions.sh
+          install_run $PLAT
+        displayName: Install wheel and test
+
+      - bash: |
+          echo "##vso[task.prependpath]$CONDA/bin"
+          sudo chown -R $USER $CONDA
+        displayName: Add conda to PATH
+
+      - bash: conda install -q -y anaconda-client
+        displayName: Install anaconda-client
+
+      - bash: |
+          set -e
+          if [ "$BUILD_COMMIT" == "master" ]; then
+            ANACONDA_ORG="scipy-wheels-nightly"
+            TOKEN="$MAPPED_SCIPY_NIGHTLY_UPLOAD_TOKEN"
+          else
+            ANACONDA_ORG="multibuild-wheels-staging"
+            TOKEN="$MAPPED_SCIPY_STAGING_UPLOAD_TOKEN"
+          fi
+          if [ "$TOKEN" == "" -o "${TOKEN:0:7}" == "\$(NUMPY" ]; then
+            echo "##[warning] Could not find anaconda.org upload token in secret variables"
+            TOKEN=""
+          fi
+          echo "##vso[task.setvariable variable=TOKEN]$TOKEN"
+          echo "##vso[task.setvariable variable=ANACONDA_ORG]$ANACONDA_ORG"
+        displayName: Retrieve secret upload token
+        env:
+          # Secret variables need to mapped to env variables explicitly:
+          MAPPED_SCIPY_NIGHTLY_UPLOAD_TOKEN: $(SCIPY_NIGHTLY_UPLOAD_TOKEN)
+          MAPPED_SCIPY_STAGING_UPLOAD_TOKEN: $(SCIPY_STAGING_UPLOAD_TOKEN)
+
+      - bash: |
+          set -e
+          if [ $ANACONDA_ORG == "scipy-wheels-nightly" ]; then
+            source extra_functions.sh
+            for f in wheelhouse/*.whl; do rename_wheel $f; done
+          fi
+
+          echo uploading wheelhouse/*.whl
+
+          anaconda -t $TOKEN upload -u $ANACONDA_ORG wheelhouse/*.whl
+          echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple"
+        displayName: Upload to anaconda.org (only if secret token is retrieved)
+        condition: ne(variables['TOKEN'], '')

config.sh

@@ -69,21 +69,22 @@ function run_tests {
         local testmode="fast"
     fi
     # Check bundled license file
-    python ../check_installed_package.py
+    $PYTHON_EXE ../check_installed_package.py
     # Run tests
     if [[ -z "$IS_OSX" && `uname -m` != 'aarch64' ]]; then
-        python ../run_scipy_tests.py $testmode -- -n2 -rfEX
+        $PYTHON_EXE ../run_scipy_tests.py $testmode -- -n2 -rfEX
     else
-        python ../run_scipy_tests.py $testmode -- -n8 -rfEX
+        $PYTHON_EXE ../run_scipy_tests.py $testmode -- -n8 -rfEX
     fi
     # Show BLAS / LAPACK used
-    python -c 'import scipy; scipy.show_config()'
+    $PYTHON_EXE -c 'import scipy; scipy.show_config()'
 }
 
 function install_run {
     # Override multibuild test running command, to preinstall packages
     # that have to be installed before TEST_DEPENDS.
-    pip install $(pip_opts) setuptools_scm
+    PIP_CMD="$PYTHON_EXE -m pip"
+    $PYTHON_EXE -m pip install $(pip_opts) setuptools_scm
 
     # Copypaste from multibuild/common_utils.sh:install_run
     install_wheel