ci: move toolstates.json to /tmp/toolstate/ and docker mount it

Before this commit toolstates.json was stored in /tmp and it wasn't
mounted outside the build container. That caused uploading the file in
the upload-artifacts task to fail, as the file was missing on the host.

Mounting /tmp/toolstates.json alone is not the best approach: if the
file is missing when the container is started the Docker engine will
create a *directory* named /tmp/toolstates.json.

The Docker issue could be solved by pre-creating an empty file named
/tmp/toolstates.json, but doing that could cause problems if bootstrap
fails to generate the file and the toolstate scripts receive an empty
JSON.

The approach I took in this commit is to instead mount a /tmp/toolstate
directory inside Docker, and create the toolstates.json file in it. That
also required a small bootstrap change to ensure the directory is
created if it's missing.

Author: pietroalbini

src/bootstrap/lib.rs

@@ -1080,6 +1080,10 @@ impl Build {
     /// done. The file is updated immediately after this function completes.
     pub fn save_toolstate(&self, tool: &str, state: ToolState) {
         if let Some(ref path) = self.config.save_toolstates {
+            if let Some(parent) = path.parent() {
+                // Ensure the parent directory always exists
+                t!(std::fs::create_dir_all(parent));
+            }
             let mut file = t!(fs::OpenOptions::new()
                 .create(true)
                 .read(true)

src/ci/azure-pipelines/auto.yml

@@ -263,8 +263,8 @@ jobs:
       # MSVC tools tests
       x86_64-msvc-tools:
         MSYS_BITS: 64
-        SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json
+        SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
         DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
 
       # 32/64-bit MinGW builds.

src/ci/docker/run.sh

@@ -106,6 +106,7 @@ fi
 mkdir -p $HOME/.cargo
 mkdir -p $objdir/tmp
 mkdir -p $objdir/cores
+mkdir -p /tmp/toolstate
 
 args=
 if [ "$SCCACHE_BUCKET" != "" ]; then
@@ -156,6 +157,7 @@ else
   args="$args --volume $objdir:/checkout/obj"
   args="$args --volume $HOME/.cargo:/cargo"
   args="$args --volume $HOME/rustsrc:$HOME/rustsrc"
+  args="$args --volume /tmp/toolstate:/tmp/toolstate"
   args="$args --env LOCAL_USER_ID=`id -u`"
 fi
 

src/ci/docker/x86_64-gnu-tools/Dockerfile

@@ -26,5 +26,5 @@ ENV CHECK_LINKS 1
 
 ENV RUST_CONFIGURE_ARGS \
   --build=x86_64-unknown-linux-gnu \
-  --save-toolstates=/tmp/toolstates.json
-ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json linux
+  --save-toolstates=/tmp/toolstate/toolstates.json
+ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstate/toolstates.json linux

src/ci/docker/x86_64-gnu-tools/checktools.sh

@@ -3,7 +3,7 @@
 set -eu
 
 X_PY="$1"
-TOOLSTATE_FILE="$(realpath $2)"
+TOOLSTATE_FILE="$(realpath -m $2)"
 OS="$3"
 COMMIT="$(git rev-parse HEAD)"
 CHANGED_FILES="$(git diff --name-status HEAD HEAD^)"
@@ -13,6 +13,7 @@ SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))"
 #   The Wednesday after this has value 0.
 #   We track this value to prevent regressing tools in the last week of the 6-week cycle.
 
+mkdir -p "$(dirname $TOOLSTATE_FILE)"
 touch "$TOOLSTATE_FILE"
 
 # Try to test all the tools and store the build/test success in the TOOLSTATE_FILE

src/ci/scripts/upload-artifacts.sh

@@ -25,7 +25,7 @@ cp cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv"
 
 # Toolstate data.
 if [[ -n "${DEPLOY_TOOLSTATES_JSON+x}" ]]; then
-    cp /tmp/toolstates.json "${upload_dir}/${DEPLOY_TOOLSTATES_JSON}"
+    cp /tmp/toolstate/toolstates.json "${upload_dir}/${DEPLOY_TOOLSTATES_JSON}"
 fi
 
 echo "Files that will be uploaded:"