Use internal mirror repositories for Ubuntu docker containers

Depending on different external URLs makes our provisioning scripts flaky, because they can fail if any one of the external servers goes down. In this patch * the docker images qt_ubuntu_16.04 and qt_ubuntu_18.04 are being built first, they apt-get from the internal ubuntu mirror * all the existing docker images have been rebased to these two images, so they inherit this behavior * images that fetch files from the internet have been modified to not do the fetching themselves; insted URLs are fetched in advance using "DownloadURL" which accepts internal mirrored URLs, and the downloaded files are ADDed to the images with a local copy * previously existing git clone commands have been modified to fetch internally mirrored tarballs Task-number: QTQAINFRA-3343 Change-Id: Ic4640a4f7776e38623edba90f0eba7cb68631223 Reviewed-by: Sona Kurazyan <sona.kurazyan@qt.io> Reviewed-by: Tony Sarajärvi <tony.sarajarvi@qt.io>
2026-02-24 16:05:36 +08:00 · 2019-11-05 18:21:19 +01:00
parent 302b228c7a
commit fe4a724a3e
23 changed files with 142 additions and 72 deletions
--- a/coin/provisioning/common/linux/docker.sh
+++ b/coin/provisioning/common/linux/docker.sh
@@ -36,7 +36,9 @@
 set -e


-. $(dirname "$0")/../../common/unix/DownloadURL.sh
+PROVISIONING_DIR="$(dirname "$0")/../../"
+. "$PROVISIONING_DIR"/common/unix/common.sourced.sh
+. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh


 localRepo=http://ci-files01-hki.intra.qt.io/input/docker
@@ -61,7 +63,7 @@ sudo docker --version
 # Download and install the docker-compose extension from https://github.com/docker/compose/releases
 f=docker-compose-$(uname -s)-$(uname -m)
 DownloadURL  \
-    $localRepo/$f  \
+    $localRepo/$f-1.24.1  \
    https://github.com/docker/compose/releases/download/1.24.1/$f \
    cfb3439956216b1248308141f7193776fcf4b9c9b49cbbe2fb07885678e2bb8a
 sudo install -m 755 ./docker-compose* /usr/local/bin/docker-compose
--- a/coin/provisioning/common/shared/testserver/apache2/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/apache2/Dockerfile
@@ -1,7 +1,8 @@
-FROM ubuntu:16.04
-ARG packages="apache2 libcgi-session-perl wget avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon apache2 libcgi-session-perl"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 80 443

 # install configurations and test data
-RUN wget https://tools.ietf.org/rfc/rfc3252.txt
+
+COPY rfc3252.txt .
--- a/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="apache2 libcgi-session-perl avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon apache2 libcgi-session-perl"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 80 443
--- a/coin/provisioning/common/shared/testserver/californium/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/californium/Dockerfile
@@ -1,15 +1,13 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get -y install git maven default-jdk avahi-daemon
-
-RUN mkdir -p /root/src/
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon maven default-jdk"
+RUN apt-get update && apt-get -y install $packages

 # Get californium-based CoAP test server
 WORKDIR /root/src
-RUN git clone https://github.com/selart/californium.git
+ADD californium-*.tar.gz .
+RUN mv californium-* californium
 WORKDIR /root/src/californium
 RUN mvn clean install -q -DskipTests
-
 WORKDIR /

 EXPOSE 5683/udp 5684/udp
--- a/coin/provisioning/common/shared/testserver/cyrus/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/cyrus/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="cyrus-imapd avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon cyrus-imapd"
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages
 EXPOSE 143 993
--- a/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="cyrus-imapd avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon cyrus-imapd"
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages
 EXPOSE 143 993
--- a/coin/provisioning/common/shared/testserver/danted/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/danted/Dockerfile
@@ -1,8 +1,9 @@
-FROM ubuntu:16.04
-ARG packages="gdebi-core wget avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
-RUN wget http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb
-RUN gdebi -n dante-server_1.4.1-1_amd64.deb
+COPY dante-server_1.4.1-1_amd64.deb .
+RUN  apt -y install ./dante-server_1.4.1-1_amd64.deb  \
+  && rm -f          ./dante-server_1.4.1-1_amd64.deb
 EXPOSE 1080-1081

 # install configurations and test data
--- a/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="dante-server avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon dante-server"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 1080-1081
--- a/coin/provisioning/common/shared/testserver/docker_images.sh
+++ b/coin/provisioning/common/shared/testserver/docker_images.sh
@@ -33,13 +33,13 @@
 ##
 #############################################################################

-set -ex
+set -e
+
+
+PROVISIONING_DIR="$(dirname "$0")/../../../"
+. "$PROVISIONING_DIR"/common/unix/common.sourced.sh
+. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh

-[ -x "$(command -v realpath)" ] && FILE=$(realpath ${BASH_SOURCE[0]}) || FILE=${BASH_SOURCE[0]}
-case $FILE in
-    */*) SERVER_PATH="${FILE%/*}" ;;
-    *) SERVER_PATH="." ;;
-esac

 # Sort files by their SHA-1, and then return the accumulated result
 sha1tree () {
@@ -51,20 +51,76 @@ sha1tree () {
        sort | ${SHASUM-sha1sum} | cut -d ' ' -f 1
 }

-# A tag labels a specific image version. In the docker compose file
-# (docker-compose.yml) that launches the containers, the tag used is
-# "latest". Here the images are also tagged with the SHA1 of each server
-# context, so if needed we can modify docker-compose.yml to launch a very
-# specific image, thus providing a way to stage backwards-incompatible changes
-# across repositories.

-source "$SERVER_PATH/settings.sh"
+SERVER_PATH="$PROVISIONING_DIR/common/shared/testserver"
+
+. "$SERVER_PATH/settings.sh"
+
+
+# Download all necessary dependencies outside of the dockerfiles, so that we
+# can use provisioning functionality for cached and verified downloads. In the
+# dockerfiles we just do COPY to put them where needed.
+
+echo 'Downloading support files for the docker images'
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/rfc3252.txt  \
+    https://tools.ietf.org/rfc/rfc3252.txt  \
+    50c323dedce95e4fdc2db35cd1b8ebf9d74711bf5296ef438b88d186d7dd082d
+cp rfc3252.txt "$SERVER_PATH/vsftpd/"
+cp rfc3252.txt "$SERVER_PATH/apache2/"
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/dante-server_1.4.1-1_amd64.deb  \
+    http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb  \
+    674a06f356cebd92c64920cec38a6687650a6f880198fbbad05aaaccca5c0a21
+mv dante-server_1.4.1-1_amd64.deb "$SERVER_PATH/danted/"
+
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/FreeCoAP-0.7.tar.gz  \
+    https://github.com/keith-cullen/FreeCoAP/archive/v0.7.tar.gz  \
+    fa6602e27dc8eaee6e34ff53400c0519da0c5c7cd47bf6f13acb564f52a693ee  \
+    FreeCoAP-0.7.tar.gz
+mv FreeCoAP-0.7.tar.gz "$SERVER_PATH/freecoap/"
+
+# Custom fork of Eclipse Californium with changes not upstream
+DownloadURL  \
+    http://ci-files01-hki.intra.qt.io/input/docker/californium-secure-test-server.tar.gz  \
+    https://github.com/sonakur/californium/archive/secure-test-server.tar.gz  \
+    0ee7f5d4366b9e31f6d2d42e389cb7a66d2db54987b700a38a3a31e8f38a7a19  \
+    californium-secure-test-server.tar.gz
+mv californium-secure-test-server.tar.gz "$SERVER_PATH/californium/"
+
+
+echo 'Building the docker images...'
+
+# Build the 2 base layers: qt_ubuntu_1604, qt_ubuntu_1804.
+# These are the base for all other docker images.
+for image in  qt_ubuntu_16.04 qt_ubuntu_18.04
+do
+    docker build -t $image  \
+        --build-arg  COIN_RUNS_IN_QT_COMPANY="$COIN_RUNS_IN_QT_COMPANY"  \
+        "$SERVER_PATH/$image"
+done
+

 for server in $testserver
 do
+
+    # We label each docker image with `-t name:tag`.
+    # A tag labels a specific image version. In the docker compose file
+    # (docker-compose.yml) that launches the containers, the tag used is
+    # "latest". Here the images are additionally tagged with the SHA1 of each
+    # image directory (context), so that if needed we can modify
+    # docker-compose.yml and modify "latest" to a SHA in order to launch a
+    # very specific image, thus providing a way to stage
+    # backwards-incompatible changes across repositories.
+
    context="$SERVER_PATH/$server"
    tag=$(sha1tree $context)
-    docker build -t qt-test-server-$server:$tag -t qt-test-server-$server:latest  $context
+    docker build -t qt-test-server-$server:latest  \
+                 -t qt-test-server-$server:$tag    \
+        $context
 done

 docker images
--- a/coin/provisioning/common/shared/testserver/echo/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/echo/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="xinetd avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon xinetd"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 7 7/UDP 13
--- a/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="xinetd avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon xinetd"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 7 7/UDP 13
--- a/coin/provisioning/common/shared/testserver/freecoap/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/freecoap/Dockerfile
@@ -1,20 +1,14 @@
-FROM ubuntu:18.04
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon autoconf automake libtool make libgnutls28-dev"
+RUN apt-get update && apt-get -y install $packages

-RUN apt-get update && apt-get -y install git avahi-daemon autoconf automake libtool make libgnutls28-dev
-
-RUN mkdir -p /root/src/
-
-# Get FreeCoAP test server
 WORKDIR /root/src
-RUN git clone https://github.com/keith-cullen/FreeCoAP.git
+ADD FreeCoAP-*.tar.gz .
+RUN mv FreeCoAP-* FreeCoAP
 WORKDIR /root/src/FreeCoAP
-RUN autoreconf --install
-RUN ./configure
+RUN autoreconf --install && ./configure && make && make install
+WORKDIR sample/time_server
 RUN make
-RUN make install
-WORKDIR /root/src/FreeCoAP/sample/time_server
-RUN make
-
 WORKDIR /

 EXPOSE 5685/udp
--- a/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="ftp-proxy avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon ftp-proxy"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 2121
--- a/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="ftp-proxy avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon ftp-proxy"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 2121
--- a/coin/provisioning/common/shared/testserver/iptables/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/iptables/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="iptables avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon iptables"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 1357
--- a/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="iptables avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon iptables"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 1357
--- a/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile
@@ -0,0 +1,5 @@
+FROM ubuntu:16.04
+ARG COIN_RUNS_IN_QT_COMPANY
+RUN  test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue  \
+  && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \
+  || echo "Internal package repository not found. Using public repositories."
--- a/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile
@@ -0,0 +1,5 @@
+FROM ubuntu:18.04
+ARG COIN_RUNS_IN_QT_COMPANY
+RUN  test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue  \
+  && sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \
+  || echo "Internal package repository not found. Using public repositories."
--- a/coin/provisioning/common/shared/testserver/squid/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/squid/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
-ARG packages="squid avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon squid"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 3128-3130
--- a/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="squid avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon squid"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 3128-3130
--- a/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/vsftpd/Dockerfile
@@ -1,7 +1,7 @@
-FROM ubuntu:16.04
-ARG packages="vsftpd ftp wget avahi-daemon"
+FROM qt_ubuntu_16.04
+ARG packages="avahi-daemon vsftpd ftp"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 20-21

 # install configurations and test data
-RUN wget https://tools.ietf.org/rfc/rfc3252.txt
+COPY rfc3252.txt .
--- a/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile
+++ b/coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
-ARG packages="vsftpd avahi-daemon"
+FROM qt_ubuntu_18.04
+ARG packages="avahi-daemon vsftpd"
 RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
 EXPOSE 20-21
--- a/coin/provisioning/common/unix/common.sourced.sh
+++ b/coin/provisioning/common/unix/common.sourced.sh
@@ -43,6 +43,7 @@ export PROVISIONING_ARCH
 export CMD_PKG_INSTALL
 export CMD_PKG_LOCALINSTALL
 export CMD_INSTALL
+export COIN_RUNS_IN_QT_COMPANY



@@ -143,6 +144,13 @@ set_common_environment () {
    esac

    CMD_INSTALL="sudo install"
+
+    COIN_RUNS_IN_QT_COMPANY=false
+    if  ping -c1 repo-clones.ci.qt.io  >/dev/null 2>&1
+    then
+        COIN_RUNS_IN_QT_COMPANY=true
+    fi
+
 }

 set_common_environment