diff --git a/ci/build-run-docker.sh b/ci/build-run-docker.sh
index 02e30665c7..bbc8490779 100644
--- a/ci/build-run-docker.sh
+++ b/ci/build-run-docker.sh
@@ -8,6 +8,7 @@ DOCKER="$1"
 TARGET="$2"
 SKIP_TESTS="$3"
 
+bash ci/fetch-rust-docker.sh "$TARGET"
 if [ -f "ci/docker/$DOCKER/Dockerfile" ]; then
   docker build -t "$DOCKER" "ci/docker/$DOCKER/"
 fi
diff --git a/ci/docker/mips-unknown-linux-gnu/Dockerfile b/ci/docker/mips-unknown-linux-gnu/Dockerfile
index 10d1d4241a..d64552a1f9 100644
--- a/ci/docker/mips-unknown-linux-gnu/Dockerfile
+++ b/ci/docker/mips-unknown-linux-gnu/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get install -y \
-  gcc-mips-linux-gnu \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
+FROM rust-mips-unknown-linux-gnu
 
 ENV CC_mips_unknown_linux_gnu=mips-linux-gnu-gcc
 ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER=mips-linux-gnu-gcc
diff --git a/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
index ae2a1f8132..6076122385 100644
--- a/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
+++ b/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get install -y \
-  gcc-mips64-linux-gnuabi64 \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
+FROM rust-mips64-unknown-linux-gnuabi64
 
 ENV CC_mips64_unknown_linux_gnuabi64=mips64-linux-gnuabi64-gcc
 ENV CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_LINKER=mips64-linux-gnuabi64-gcc
diff --git a/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile b/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
index 8e6911ad5c..0d0138220a 100644
--- a/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
+++ b/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get install -y \
-  gcc-mips64el-linux-gnuabi64 \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
+FROM rust-mips64el-unknown-linux-gnuabi64
 
 ENV CC_mips64el_unknown_linux_gnuabi64=mips64el-linux-gnuabi64-gcc
 ENV CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_LINKER=mips64el-linux-gnuabi64-gcc
diff --git a/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
index 5af20a21cf..25857c4d84 100644
--- a/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
+++ b/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get install -y \
-  gcc-mipsel-linux-gnu \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
+FROM rust-mipsel-unknown-linux-gnu
 
 ENV CC_mipsel_unknown_linux_gnu=mipsel-linux-gnu-gcc
 ENV CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_LINKER=mipsel-linux-gnu-gcc
diff --git a/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
index 8cfbbf54d8..c96f43e36a 100644
--- a/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
+++ b/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
+FROM rust-powerpc-unknown-linux-gnu
 
-RUN apt-get update && apt-get install -y \
-  gcc-powerpc-linux-gnu \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
-
-ENV CC_powerpc_unknown_linux_gnu=powerpc-linux-gnu-gcc
-ENV CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_LINKER=powerpc-linux-gnu-gcc
+ENV CC_powerpc_unknown_linux_gnu=powerpc-unknown-linux-gnu-gcc
+ENV CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_LINKER=powerpc-unknown-linux-gnu-gcc
diff --git a/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
index fb5572a81a..9ba83377ea 100644
--- a/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
+++ b/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
+FROM rust-powerpc64-unknown-linux-gnu
 
-RUN apt-get update && apt-get install -y \
-  gcc-powerpc64-linux-gnu \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
-
-ENV CC_powerpc64_unknown_linux_gnu=powerpc64-linux-gnu-gcc
-ENV CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_LINKER=powerpc64-linux-gnu-gcc
+ENV CC_powerpc64_unknown_linux_gnu=powerpc64-unknown-linux-gnu-gcc
+ENV CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_LINKER=powerpc64-unknown-linux-gnu-gcc
diff --git a/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
index 642e99c792..0cc0432700 100644
--- a/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
+++ b/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get install -y \
-  gcc-powerpc64le-linux-gnu \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
+FROM rust-powerpc64le-unknown-linux-gnu
 
 ENV CC_powerpc64le_unknown_linux_gnu=powerpc64le-linux-gnu-gcc
 ENV CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_LINKER=powerpc64le-linux-gnu-gcc
diff --git a/ci/docker/s390x-unknown-linux-gnu/Dockerfile b/ci/docker/s390x-unknown-linux-gnu/Dockerfile
index 6068783084..653cb5c763 100644
--- a/ci/docker/s390x-unknown-linux-gnu/Dockerfile
+++ b/ci/docker/s390x-unknown-linux-gnu/Dockerfile
@@ -1,12 +1,4 @@
-FROM ubuntu:18.04
+FROM rust-s390x-unknown-linux-gnu
 
-RUN apt-get update && apt-get install -y \
-  gcc-s390x-linux-gnu \
-  curl \
-  ca-certificates \
-  perl \
-  make \
-  gcc
-
-ENV CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc
-ENV CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=s390x-linux-gnu-gcc
+ENV CC_s390x_unknown_linux_gnu=s390x-ibm-linux-gnu-gcc
+ENV CARGO_TARGET_S390X_UNKNOWN_LINUX_GNU_LINKER=s390x-ibm-linux-gnu-gcc
diff --git a/ci/fetch-rust-docker.sh b/ci/fetch-rust-docker.sh
new file mode 100644
index 0000000000..d1d6f85db9
--- /dev/null
+++ b/ci/fetch-rust-docker.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -ex
+
+TARGET="$1"
+
+RUST_REPO="https://github.com/rust-lang/rust"
+S3_BASE_URL="https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rustc-builds"
+
+# See http://unix.stackexchange.com/questions/82598
+# Duplicated from rust-lang/rust/src/ci/shared.sh
+function retry {
+  echo "Attempting with retry:" "$@"
+  local n=1
+  local max=5
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        sleep $n  # don't retry immediately
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+      else
+        echo "The command has failed after $n attempts."
+        return 1
+      fi
+    }
+  done
+}
+
+# Use images from rustc master
+case "$TARGET" in
+  mips-unknown-linux-gnu)          image=dist-mips-linux ;;
+  mips64-unknown-linux-gnuabi64)   image=dist-mips64-linux ;;
+  mips64el-unknown-linux-gnuabi64) image=dist-mips64el-linux ;;
+  mipsel-unknown-linux-gnu)        image=dist-mipsel-linux ;;
+  powerpc-unknown-linux-gnu)       image=dist-powerpc-linux ;;
+  powerpc64-unknown-linux-gnu)     image=dist-powerpc64-linux ;;
+  powerpc64le-unknown-linux-gnu)   image=dist-powerpc64le-linux ;;
+  s390x-unknown-linux-gnu)         image=dist-s390x-linux ;;
+  *) exit ;;
+esac
+
+master=$(git ls-remote "$RUST_REPO" refs/heads/master | cut -f1)
+image_url="$S3_BASE_URL/$master/image-$image.txt"
+info="/tmp/image-$image.txt"
+
+rm -f "$info"
+curl -o "$info" "$image_url"
+digest=$(grep -m1 ^sha "$info")
+
+if ! docker tag "$digest" "rust-$TARGET"; then
+  url=$(grep -m1 ^https "$info")
+  cache=/tmp/rustci_docker_cache
+  echo "Attempting to download $url"
+  rm -f "$cache"
+  set +e
+  retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o "$cache" "$url"
+  docker load -i "$cache"
+  set -e
+  docker tag "$digest" "rust-$TARGET"
+fi