Conda envs and CI conf update

Author: Alexsandruss

azure-pipelines.yml

@@ -31,13 +31,27 @@ jobs:
         pip install isort black
         isort --check . && black --check .
       displayName: 'Linting'
-  - job: Linux_Sklearn
+  - job: Linux
+    dependsOn: Lint
+    strategy:
+      matrix:
+        Pip:
+          PKG_MANAGER: "pip"
+        Conda:
+          PKG_MANAGER: "conda"
     pool:
       vmImage: "ubuntu-latest"
     steps:
-    - template: test-configuration.yml
-  - job: Windows_Sklearn
+    - template: test-configuration-linux.yml
+  - job: Windows
+    dependsOn: Lint
+    strategy:
+      matrix:
+        Pip:
+          PKG_MANAGER: "pip"
+        Conda:
+          PKG_MANAGER: "conda"
     pool:
       vmImage: "windows-latest"
     steps:
-    - template: test-configuration.yml
+    - template: test-configuration-win.yml

envs/conda-env-rapids.yml

@@ -2,6 +2,7 @@ channels:
   - rapidsai
   - conda-forge
   - nvidia
+  - nodefaults
 dependencies:
   - python=3.10
   - rapids

envs/conda-env-sklearn.yml

@@ -9,10 +9,13 @@ dependencies:
   - catboost
   - lightgbm
   - faiss-cpu
-  - scikit-learn-intelex
   - modin-all
+  - intel::scikit-learn-intelex
+  - intel::daal4py
+  - intel::dpctl
+  - intel::dpnp
   # sklbench dependencies
-  - scikit-learn
+  - scikit-learn=1.4.*
   - pandas
   - tabulate
   - fastparquet

test-configuration-linux.yml

@@ -0,0 +1,47 @@
+#===============================================================================
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+steps:
+  - task: UsePythonVersion@0
+    displayName: "Use Python $(python.version)"
+    inputs:
+      versionSpec: "$(python.version)"
+  - script: |
+      conda create -y -n bench-env -c conda-forge -c nodefaults python=$(python.version)
+    displayName: Environment update
+  - script: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate bench-env
+      pip install -r envs/requirements-sklearn.txt
+      pip list
+    displayName: Install requirements via pip
+    condition: eq(variables['PKG_MANAGER'], 'pip')
+  - script: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate bench-env
+      conda env update -f envs/conda-env-sklearn.yml -n bench-env
+      conda list -n bench-env
+    displayName: Install requirements via conda
+    condition: eq(variables['PKG_MANAGER'], 'conda')
+  - script: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate bench-env
+      python -m sklbench -l DEBUG -p algorithm:library=sklearnex algorithm:estimator=KMeans data:dataset=a9a
+    displayName: CLI arguments run
+  - script: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate bench-env
+      python -m sklbench -l INFO -c configs/testing/azure-pipelines-ci.json --prefetch-datasets --report --diff-cols library --compatibility-mode
+    displayName: CI config run

test-configuration.yml renamed to test-configuration-win.yml

@@ -18,13 +18,28 @@ steps:
     displayName: "Use Python $(python.version)"
     inputs:
       versionSpec: "$(python.version)"
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to Windows PATH
   - script: |
+      conda create -y -n bench-env -c conda-forge -c nodefaults python=$(python.version)
+    displayName: Environment update
+  - script: |
+      call activate bench-env
       pip install -r envs/requirements-sklearn.txt
       pip list
-    displayName: Install requirements
+    displayName: Install requirements via pip
+    condition: eq(variables['PKG_MANAGER'], 'pip')
+  - script: |
+      call activate bench-env
+      conda env update -f envs/conda-env-sklearn.yml -n bench-env
+      conda list -n bench-env
+    displayName: Install requirements via conda
+    condition: eq(variables['PKG_MANAGER'], 'conda')
   - script: |
+      call activate bench-env
       python -m sklbench -l DEBUG -p algorithm:library=sklearnex algorithm:estimator=KMeans data:dataset=a9a
     displayName: CLI arguments run
   - script: |
+      call activate bench-env
       python -m sklbench -l INFO -c configs/testing/azure-pipelines-ci.json --prefetch-datasets --report --diff-cols library --compatibility-mode
     displayName: CI config run