Split tests into groups for parallel execution in CI/CD runners (#417)

Split tests into groups for parallel execution in CI/CD runners

[ committed by @billschereriii ]
[ reviewed by @al-rigazzi @ankona ]

Author: billschereriii

.github/workflows/run_tests.yml

@@ -44,11 +44,12 @@ env:
 
 jobs:
   run_tests:
-    name: Run tests with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
+    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        subset: [backends, slow_tests, group_a, group_b]
         os: [macos-12, ubuntu-20.04] # Operating systems
         compiler: [8] # GNU compiler version
         rai: [1.2.7] # Redis AI versions
@@ -121,10 +122,19 @@ jobs:
       - name: Run Pylint
         run: make check-lint
 
+      # Run pytest (backends subdirectory)
       - name: Run Pytest
+        if: (matrix.subset == 'backends')
         run: |
           echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
-          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/
+          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/backends
+
+      # Run pytest (test subsets)
+      - name: Run Pytest
+        if: "!contains(matrix.subset, 'backends')" # if not running backend tests
+        run: |
+          echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
+          py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests
 
       # Upload artifacts on failure, ignoring binary files
       - name: Upload Artifact
@@ -143,5 +153,5 @@ jobs:
       - name: Upload Pytest coverage to Codecov
         uses: codecov/codecov-action@v2
         with:
-          fail_ci_if_error: true
+          fail_ci_if_error: false
           files: ./coverage.xml

doc/changelog.rst

@@ -19,6 +19,7 @@ To be released at some future point in time
 
 Description
 
+- Split tests into groups for parallel execution in CI/CD pipeline
 - Change signature of `Experiment.summary()`
 - Expose first_device parameter for scripts, functions, models
 - Added support for MINBATCHTIMEOUT in model execution
@@ -27,6 +28,7 @@ Description
 
 Detailed Notes
 
+- Split tests into `group_a`, `group_b`, `slow_tests` for parallel execution in CI/CD pipeline (PR417_)
 - Change `format` argument to `style` in `Experiment.summary()`, this is
   an API break (PR391_)
 - Added support for first_device parameter for scripts, functions,
@@ -43,6 +45,7 @@ Detailed Notes
 - Add support for creation of multiple databases with unique identifiers. (PR342_)
 
 
+  .. _PR417: https://github.com/CrayLabs/SmartSim/pull/417
   .. _PR391: https://github.com/CrayLabs/SmartSim/pull/391
   .. _PR342: https://github.com/CrayLabs/SmartSim/pull/342
   .. _PR394: https://github.com/CrayLabs/SmartSim/pull/394

tests/test_alps_settings.py

@@ -30,6 +30,9 @@
 from smartsim.error import SSUnsupportedError
 from smartsim.settings import AprunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_aprun_settings():
     settings = AprunSettings("python")

tests/test_batch_settings.py

@@ -30,6 +30,9 @@
 from smartsim.settings import BsubBatchSettings, QsubBatchSettings, SbatchSettings
 from smartsim.settings.settings import create_batch_settings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_create_pbs_batch():
     pbs_batch = create_batch_settings(

tests/test_buildenv.py

@@ -30,6 +30,9 @@
 from smartsim._core._install.buildenv import Version_
 from pkg_resources import packaging  # type: ignore
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_version_hash_eq():
     """Ensure hashes are equal if data is equal"""

tests/test_cli.py

@@ -45,6 +45,10 @@
     configure_parser as validate_parser,
 )
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
+
 def mock_execute_custom(msg: str = None, good: bool = True) -> int:
     retval = 0 if good else 1
     print(msg)

tests/test_cobalt_parser.py

@@ -24,9 +24,13 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import pytest
 
 from smartsim._core.launcher.cobalt import cobaltParser
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_parse_step_id():
     output = "JobName      JobId \n" "=====================\n" "smartsim     507975 \n"

tests/test_colo_model_local.py

@@ -32,6 +32,10 @@
 from smartsim.error import SSUnsupportedError
 from smartsim.entity import Model
 
+# The tests in this file belong to the slow_tests group
+pytestmark = pytest.mark.slow_tests
+
+
 if sys.platform == "darwin":
     supported_dbs = ["tcp", "deprecated"]
 else:

tests/test_colo_model_lsf.py

@@ -33,6 +33,9 @@
 import smartsim.settings.base
 from smartsim.settings.lsfSettings import JsrunSettings
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 is_mac = sys.platform == "darwin"
 

tests/test_config.py

@@ -28,14 +28,15 @@
 from pathlib import Path
 
 import os
-from re import A, L
 import pytest
-import shutil
 import typing as t
 
 from smartsim._core.config.config import Config
 from smartsim.error import SSConfigError
 
+# The tests in this file belong to the group_a group
+pytestmark = pytest.mark.group_a
+
 
 def test_all_config_defaults():
     config = Config()