Rollup merge of rust-lang#41958 - malbarbo:docker-share, r=alexcrichton

ci: allows files to be shared by docker images

Change `src/ci/docker/run.sh` to allow files to be shared when building docker containers. For while, only android related shell scripts are shared. Others containers can be updated at any time.

sccache is installed last, as it is frequently updated this avoids the need to rebuilt the whole container.

Author: Mark-Simulacrum

src/ci/docker/README.md

@@ -16,6 +16,12 @@ for example:
 
 Images will output artifacts in an `obj` dir at the root of a repository.
 
+## Filesystem layout
+
+- Each directory, excluding `scripts` and `disabled`, corresponds to a docker image
+- `scripts` contains files shared by docker images
+- `disabled` contains images that are not build travis
+
 ## Cross toolchains
 
 A number of these images take quite a long time to compile as they're building

src/ci/docker/arm-android/Dockerfile

@@ -2,52 +2,44 @@ FROM ubuntu:16.04
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+  ca-certificates \
+  cmake \
+  curl \
+  file \
   g++ \
+  git \
+  libssl-dev \
   make \
-  file \
-  curl \
-  ca-certificates \
+  pkg-config \
   python2.7 \
-  git \
-  cmake \
-  unzip \
   sudo \
-  xz-utils \
-  libssl-dev \
-  pkg-config
-
-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+  unzip \
+  xz-utils
 
-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-    chmod +x /usr/local/bin/sccache
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh
 
-# Install NDK
-COPY install-ndk.sh /tmp
-RUN . /tmp/install-ndk.sh && \
-    download_ndk android-ndk-r13b-linux-x86_64.zip && \
-    make_standalone_toolchain arm 9 && \
-    remove_ndk
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
+    download_and_make_toolchain android-ndk-r13b-linux-x86_64.zip arm 9
 
-# Install SDK
+# sdk
 RUN dpkg --add-architecture i386 && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-  openjdk-9-jre-headless \
-  tzdata \
-  libstdc++6:i386 \
   libgl1-mesa-glx \
-  libpulse0
+  libpulse0 \
+  libstdc++6:i386 \
+  openjdk-9-jre-headless \
+  tzdata
 
-COPY install-sdk.sh /tmp
-RUN . /tmp/install-sdk.sh && \
-    download_sdk tools_r25.2.5-linux.zip && \
-    download_sysimage armeabi-v7a 18 && \
-    create_avd armeabi-v7a 18
+COPY scripts/android-sdk.sh /scripts/
+RUN . /scripts/android-sdk.sh && \
+    download_and_create_avd tools_r25.2.5-linux.zip armeabi-v7a 18
 
-# Setup env
+# env
 ENV PATH=$PATH:/android/sdk/tools
 ENV PATH=$PATH:/android/sdk/platform-tools
 
@@ -57,8 +49,12 @@ ENV RUST_CONFIGURE_ARGS \
       --target=$TARGETS \
       --arm-linux-androideabi-ndk=/android/ndk/arm-9
 
-ENV SCRIPT python2.7 ../x.py test --target $TARGETS --verbose
+ENV SCRIPT python2.7 ../x.py test --target $TARGETS
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
 
-# Entrypoint
-COPY start-emulator.sh /android/
-ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
+# init
+COPY scripts/android-start-emulator.sh /scripts/
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/scripts/android-start-emulator.sh"]

src/ci/docker/arm-android/install-ndk.sh

@@ -31,7 +31,7 @@ WORKDIR /build
 # The `vexpress_config` config file was a previously generated config file for
 # the kernel. This file was generated by running `make vexpress_defconfig`
 # followed by `make menuconfig` and then enabling the IPv6 protocol page.
-COPY vexpress_config /build/.config
+COPY armhf-gnu/vexpress_config /build/.config
 RUN curl https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.4.42.tar.xz | \
       tar xJf - && \
       cd /build/linux-4.4.42 && \
@@ -63,11 +63,11 @@ RUN curl http://cdimage.ubuntu.com/ubuntu-base/releases/16.04/release/ubuntu-bas
 
 # Copy over our init script, which starts up our test server and also a few
 # other misc tasks.
-COPY rcS rootfs/etc/init.d/rcS
+COPY armhf-gnu/rcS rootfs/etc/init.d/rcS
 RUN chmod +x rootfs/etc/init.d/rcS
 
 # Helper to quickly fill the entropy pool in the kernel.
-COPY addentropy.c /tmp/
+COPY armhf-gnu/addentropy.c /tmp/
 RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static
 
 # TODO: What is this?!

src/ci/docker/armhf-gnu/Dockerfile

@@ -32,10 +32,10 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 WORKDIR /tmp
 
-COPY build-rumprun.sh /tmp/
+COPY cross/build-rumprun.sh /tmp/
 RUN ./build-rumprun.sh
 
-COPY build-arm-musl.sh /tmp/
+COPY cross/build-arm-musl.sh /tmp/
 RUN ./build-arm-musl.sh
 
 # originally from

src/ci/docker/cross/Dockerfile

@@ -2,36 +2,30 @@ FROM ubuntu:16.04
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+  ca-certificates \
+  cmake \
+  curl \
+  file \
   g++ \
+  git \
+  libssl-dev \
   make \
-  file \
-  curl \
-  ca-certificates \
+  pkg-config \
   python2.7 \
-  git \
-  cmake \
-  unzip \
   sudo \
-  xz-utils \
-  libssl-dev \
-  pkg-config
-
-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
-
-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
+  unzip \
+  xz-utils
 
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh
 
-COPY android-ndk.sh /
-RUN . /android-ndk.sh && \
-    download_ndk android-ndk-r13b-linux-x86_64.zip && \
-    make_standalone_toolchain arm64 21 && \
-    remove_ndk
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
+    download_and_make_toolchain android-ndk-r13b-linux-x86_64.zip arm64 21
 
+# env
 ENV PATH=$PATH:/android/ndk/arm64-21/bin
 
 ENV DEP_Z_ROOT=/android/ndk/arm64-21/sysroot/usr/
@@ -47,3 +41,10 @@ ENV RUST_CONFIGURE_ARGS \
       --enable-cargo-openssl-static
 
 ENV SCRIPT python2.7 ../x.py dist --target $HOSTS --host $HOSTS
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]

src/ci/docker/disabled/dist-aarch64-android/Dockerfile

@@ -2,37 +2,36 @@ FROM ubuntu:16.04
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+  ca-certificates \
+  cmake \
+  curl \
+  file \
   g++ \
+  git \
+  libssl-dev \
   make \
-  file \
-  curl \
-  ca-certificates \
+  pkg-config \
   python2.7 \
-  git \
-  cmake \
-  unzip \
   sudo \
-  xz-utils \
-  libssl-dev \
-  pkg-config
-
-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
-
-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
+  unzip \
+  xz-utils
 
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh
 
-COPY android-ndk.sh /
-RUN . /android-ndk.sh && \
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
     download_ndk android-ndk-r13b-linux-x86_64.zip && \
     make_standalone_toolchain arm 9 && \
     make_standalone_toolchain arm 21 && \
     remove_ndk
 
+RUN chmod 777 /android/ndk && \
+    ln -s /android/ndk/arm-21 /android/ndk/arm
+
+# env
 ENV PATH=$PATH:/android/ndk/arm-9/bin
 
 ENV DEP_Z_ROOT=/android/ndk/arm-9/sysroot/usr/
@@ -54,12 +53,16 @@ ENV RUST_CONFIGURE_ARGS \
 # level 9), the default linker behavior is to generate an error, to allow the
 # build to finish we use --warn-unresolved-symbols. Note that the missing
 # symbols does not affect std, only the compiler (llvm) and cargo (openssl).
-RUN chmod 777 /android/ndk && \
-    ln -s /android/ndk/arm-21 /android/ndk/arm
-
 ENV SCRIPT \
   python2.7 ../x.py build src/llvm --host $HOSTS --target $HOSTS && \
   (export RUSTFLAGS="\"-C link-arg=-Wl,--warn-unresolved-symbols\""; \
     rm /android/ndk/arm && \
     ln -s /android/ndk/arm-9 /android/ndk/arm && \
     python2.7 ../x.py dist --host $HOSTS --target $HOSTS)
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]

src/ci/docker/disabled/dist-armv7-android/Dockerfile

@@ -2,37 +2,36 @@ FROM ubuntu:16.04
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+  ca-certificates \
+  cmake \
+  curl \
+  file \
   g++ \
+  git \
+  libssl-dev \
   make \
-  file \
-  curl \
-  ca-certificates \
+  pkg-config \
   python2.7 \
-  git \
-  cmake \
-  unzip \
   sudo \
-  xz-utils \
-  libssl-dev \
-  pkg-config
-
-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
-
-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
+  unzip \
+  xz-utils
 
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh
 
-COPY android-ndk.sh /
-RUN . /android-ndk.sh && \
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
     download_ndk android-ndk-r13b-linux-x86_64.zip && \
     make_standalone_toolchain x86 9 && \
     make_standalone_toolchain x86 21 && \
     remove_ndk
 
+RUN chmod 777 /android/ndk && \
+    ln -s /android/ndk/x86-21 /android/ndk/x86
+
+# env
 ENV PATH=$PATH:/android/ndk/x86-9/bin
 
 ENV DEP_Z_ROOT=/android/ndk/x86-9/sysroot/usr/
@@ -54,12 +53,16 @@ ENV RUST_CONFIGURE_ARGS \
 # level 9), the default linker behavior is to generate an error, to allow the
 # build to finish we use --warn-unresolved-symbols. Note that the missing
 # symbols does not affect std, only the compiler (llvm) and cargo (openssl).
-RUN chmod 777 /android/ndk && \
-    ln -s /android/ndk/x86-21 /android/ndk/x86
-
 ENV SCRIPT \
   python2.7 ../x.py build src/llvm --host $HOSTS --target $HOSTS && \
   (export RUSTFLAGS="\"-C link-arg=-Wl,--warn-unresolved-symbols\""; \
     rm /android/ndk/x86 && \
     ln -s /android/ndk/x86-9 /android/ndk/x86 && \
     python2.7 ../x.py dist --host $HOSTS --target $HOSTS)
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# init
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]