tests(dist): attempt to fix dist-test build

Author: csegarragonz

.github/workflows/tests.yml

@@ -121,6 +121,7 @@ jobs:
         run: ./bin/inv_wrapper.sh dev.cc faabric_tests
       - name: "Run tests"
         run: ./bin/inv_wrapper.sh tests
+        timeout-minutes: 15
 
   dist-tests:
     if: github.event.pull_request.draft == false
@@ -138,6 +139,7 @@ jobs:
         run: ./dist-test/build.sh
       - name: "Run the distributed tests"
         run: ./dist-test/run.sh
+        timeout-minutes: 15
       - name: "Print planner logs"
         if: always()
         run: docker compose logs planner

dist-test/build.sh

@@ -6,9 +6,9 @@ export PROJ_ROOT=$(dirname $(dirname $(readlink -f $0)))
 pushd ${PROJ_ROOT} >> /dev/null
 
 # Run the build
+export FAABRIC_DEPLOYMENT_TYPE=gha-ci
 docker compose \
     run \
-    -e FAABRIC_DEPLOYMENT_TYPE=gha-ci \
     --rm \
     cli \
     /code/faabric/dist-test/build_internal.sh

dist-test/build_internal.sh

@@ -9,7 +9,7 @@ pushd ${PROJ_ROOT} >> /dev/null
 source ./bin/workon.sh
 
 # Run the debug build
-inv dev.cmake --build=Debug
+inv dev.cmake --build=Debug --clean
 inv dev.cc faabric_dist_tests
 inv dev.cc faabric_dist_test_server
 inv dev.cc planner_server

dist-test/run.sh

@@ -16,6 +16,7 @@ docker compose \
 # Run the tests directly
 docker compose \
     run \
+    -e LOG_LEVEL=trace \
     --rm \
     cli \
     /build/faabric/static/bin/faabric_dist_tests

tests/dist/dist_test_fixtures.h

@@ -167,7 +167,8 @@ class MpiDistTestsFixture : public DistTestsFixture
         while (batchResults->messageresults_size() != worldSize) {
             if (numRetries >= maxRetries) {
                 SPDLOG_ERROR(
-                  "Timed-out waiting for MPI messages results ({}/{})",
+                  "Timed-out waiting for MPI messages results (app: {}, {}/{})",
+                  req->appid(),
                   batchResults->messageresults_size(),
                   worldSize);
                 throw std::runtime_error("Timed-out waiting for MPI messges");