lili/qt5
1
0
Fork 0
mirror of git://code.qt.io/qt/qt5.git synced 2026-04-21 20:36:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use internal mirror repositories for Ubuntu docker containers

Browse Source 
Depending on different external URLs makes our provisioning scripts flaky,
because they can fail if any one of the external servers goes down.
In this patch

* the docker images qt_ubuntu_16.04 and qt_ubuntu_18.04 are being built first,
  they apt-get from the internal ubuntu mirror

* all the existing docker images have been rebased to these two images, so
  they inherit this behavior

* images that fetch files from the internet have been modified to not do the
  fetching themselves; insted URLs are fetched in advance using "DownloadURL"
  which accepts internal mirrored URLs, and the downloaded files are ADDed to
  the images with a local copy

* previously existing git clone commands have been modified to fetch
  internally mirrored tarballs

Task-number: QTQAINFRA-3343
Change-Id: Ic4640a4f7776e38623edba90f0eba7cb68631223
Reviewed-by: Sona Kurazyan <sona.kurazyan@qt.io>
Reviewed-by: Tony Sarajärvi <tony.sarajarvi@qt.io>
This commit is contained in:
Dimitrios Apostolou
2019-11-05 18:21:19 +01:00
parent 302b228c7a
commit fe4a724a3e
23 changed files with 142 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
coin/provisioning/common/linux/docker.sh
View File

@@ -36,7 +36,9 @@
set -e set -e
. $(dirname "$0")/../../common/unix/DownloadURL.sh PROVISIONING_DIR="$(dirname "$0")/../../"
. "$PROVISIONING_DIR"/common/unix/common.sourced.sh
. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh
localRepo=http://ci-files01-hki.intra.qt.io/input/docker localRepo=http://ci-files01-hki.intra.qt.io/input/docker
@@ -61,7 +63,7 @@ sudo docker --version
# Download and install the docker-compose extension from https://github.com/docker/compose/releases # Download and install the docker-compose extension from https://github.com/docker/compose/releases
f=docker-compose-$(uname -s)-$(uname -m) f=docker-compose-$(uname -s)-$(uname -m)
DownloadURL \ DownloadURL \
$localRepo/$f \ $localRepo/$f-1.24.1 \
https://github.com/docker/compose/releases/download/1.24.1/$f \ https://github.com/docker/compose/releases/download/1.24.1/$f \
cfb3439956216b1248308141f7193776fcf4b9c9b49cbbe2fb07885678e2bb8a cfb3439956216b1248308141f7193776fcf4b9c9b49cbbe2fb07885678e2bb8a
sudo install -m 755 ./docker-compose* /usr/local/bin/docker-compose sudo install -m 755 ./docker-compose* /usr/local/bin/docker-compose

7
coin/provisioning/common/shared/testserver/apache2/Dockerfile
View File

@@ -1,7 +1,8 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="apache2 libcgi-session-perl wget avahi-daemon" ARG packages="avahi-daemon apache2 libcgi-session-perl"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 80 443 EXPOSE 80 443
# install configurations and test data # install configurations and test data
RUN wget https://tools.ietf.org/rfc/rfc3252.txt
COPY rfc3252.txt .

4
coin/provisioning/common/shared/testserver/apache2_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="apache2 libcgi-session-perl avahi-daemon" ARG packages="avahi-daemon apache2 libcgi-session-perl"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 80 443 EXPOSE 80 443

12
coin/provisioning/common/shared/testserver/californium/Dockerfile
View File

@@ -1,15 +1,13 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="avahi-daemon maven default-jdk"
RUN apt-get update && apt-get -y install git maven default-jdk avahi-daemon RUN apt-get update && apt-get -y install $packages
RUN mkdir -p /root/src/
# Get californium-based CoAP test server # Get californium-based CoAP test server
WORKDIR /root/src WORKDIR /root/src
RUN git clone https://github.com/selart/californium.git ADD californium-*.tar.gz .
RUN mv californium-* californium
WORKDIR /root/src/californium WORKDIR /root/src/californium
RUN mvn clean install -q -DskipTests RUN mvn clean install -q -DskipTests
WORKDIR / WORKDIR /
EXPOSE 5683/udp 5684/udp EXPOSE 5683/udp 5684/udp

4
coin/provisioning/common/shared/testserver/cyrus/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="cyrus-imapd avahi-daemon" ARG packages="avahi-daemon cyrus-imapd"
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages
EXPOSE 143 993 EXPOSE 143 993

4
coin/provisioning/common/shared/testserver/cyrus_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="cyrus-imapd avahi-daemon" ARG packages="avahi-daemon cyrus-imapd"
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y $packages && dpkg -l $packages
EXPOSE 143 993 EXPOSE 143 993

9
coin/provisioning/common/shared/testserver/danted/Dockerfile
View File

@@ -1,8 +1,9 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="gdebi-core wget avahi-daemon" ARG packages="avahi-daemon"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
RUN wget http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb COPY dante-server_1.4.1-1_amd64.deb .
RUN gdebi -n dante-server_1.4.1-1_amd64.deb RUN apt -y install ./dante-server_1.4.1-1_amd64.deb \
&& rm -f ./dante-server_1.4.1-1_amd64.deb
EXPOSE 1080-1081 EXPOSE 1080-1081
# install configurations and test data # install configurations and test data

4
coin/provisioning/common/shared/testserver/danted_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="dante-server avahi-daemon" ARG packages="avahi-daemon dante-server"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 1080-1081 EXPOSE 1080-1081

84
coin/provisioning/common/shared/testserver/docker_images.sh
View File

@@ -33,13 +33,13 @@
## ##
############################################################################# #############################################################################
set -ex set -e
PROVISIONING_DIR="$(dirname "$0")/../../../"
. "$PROVISIONING_DIR"/common/unix/common.sourced.sh
. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh
[ -x "$(command -v realpath)" ] && FILE=$(realpath ${BASH_SOURCE[0]}) || FILE=${BASH_SOURCE[0]}
case $FILE in
*/*) SERVER_PATH="${FILE%/*}" ;;
*) SERVER_PATH="." ;;
esac
# Sort files by their SHA-1, and then return the accumulated result # Sort files by their SHA-1, and then return the accumulated result
sha1tree () { sha1tree () {
@@ -51,20 +51,76 @@ sha1tree () {
sort | ${SHASUM-sha1sum} | cut -d ' ' -f 1 sort | ${SHASUM-sha1sum} | cut -d ' ' -f 1
} }
# A tag labels a specific image version. In the docker compose file
# (docker-compose.yml) that launches the containers, the tag used is
# "latest". Here the images are also tagged with the SHA1 of each server
# context, so if needed we can modify docker-compose.yml to launch a very
# specific image, thus providing a way to stage backwards-incompatible changes
# across repositories.
source "$SERVER_PATH/settings.sh" SERVER_PATH="$PROVISIONING_DIR/common/shared/testserver"
. "$SERVER_PATH/settings.sh"
# Download all necessary dependencies outside of the dockerfiles, so that we
# can use provisioning functionality for cached and verified downloads. In the
# dockerfiles we just do COPY to put them where needed.
echo 'Downloading support files for the docker images'
DownloadURL \
http://ci-files01-hki.intra.qt.io/input/docker/rfc3252.txt \
https://tools.ietf.org/rfc/rfc3252.txt \
50c323dedce95e4fdc2db35cd1b8ebf9d74711bf5296ef438b88d186d7dd082d
cp rfc3252.txt "$SERVER_PATH/vsftpd/"
cp rfc3252.txt "$SERVER_PATH/apache2/"
DownloadURL \
http://ci-files01-hki.intra.qt.io/input/docker/dante-server_1.4.1-1_amd64.deb \
http://ppa.launchpad.net/dajhorn/dante/ubuntu/pool/main/d/dante/dante-server_1.4.1-1_amd64.deb \
674a06f356cebd92c64920cec38a6687650a6f880198fbbad05aaaccca5c0a21
mv dante-server_1.4.1-1_amd64.deb "$SERVER_PATH/danted/"
DownloadURL \
http://ci-files01-hki.intra.qt.io/input/docker/FreeCoAP-0.7.tar.gz \
https://github.com/keith-cullen/FreeCoAP/archive/v0.7.tar.gz \
fa6602e27dc8eaee6e34ff53400c0519da0c5c7cd47bf6f13acb564f52a693ee \
FreeCoAP-0.7.tar.gz
mv FreeCoAP-0.7.tar.gz "$SERVER_PATH/freecoap/"
# Custom fork of Eclipse Californium with changes not upstream
DownloadURL \
http://ci-files01-hki.intra.qt.io/input/docker/californium-secure-test-server.tar.gz \
https://github.com/sonakur/californium/archive/secure-test-server.tar.gz \
0ee7f5d4366b9e31f6d2d42e389cb7a66d2db54987b700a38a3a31e8f38a7a19 \
californium-secure-test-server.tar.gz
mv californium-secure-test-server.tar.gz "$SERVER_PATH/californium/"
echo 'Building the docker images...'
# Build the 2 base layers: qt_ubuntu_1604, qt_ubuntu_1804.
# These are the base for all other docker images.
for image in qt_ubuntu_16.04 qt_ubuntu_18.04
do
docker build -t $image \
--build-arg COIN_RUNS_IN_QT_COMPANY="$COIN_RUNS_IN_QT_COMPANY" \
"$SERVER_PATH/$image"
done
for server in $testserver for server in $testserver
do do
# We label each docker image with `-t name:tag`.
# A tag labels a specific image version. In the docker compose file
# (docker-compose.yml) that launches the containers, the tag used is
# "latest". Here the images are additionally tagged with the SHA1 of each
# image directory (context), so that if needed we can modify
# docker-compose.yml and modify "latest" to a SHA in order to launch a
# very specific image, thus providing a way to stage
# backwards-incompatible changes across repositories.
context="$SERVER_PATH/$server" context="$SERVER_PATH/$server"
tag=$(sha1tree $context) tag=$(sha1tree $context)
docker build -t qt-test-server-$server:$tag -t qt-test-server-$server:latest $context docker build -t qt-test-server-$server:latest \
-t qt-test-server-$server:$tag \
$context
done done
docker images docker images

4
coin/provisioning/common/shared/testserver/echo/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="xinetd avahi-daemon" ARG packages="avahi-daemon xinetd"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 7 7/UDP 13 EXPOSE 7 7/UDP 13

4
coin/provisioning/common/shared/testserver/echo_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="xinetd avahi-daemon" ARG packages="avahi-daemon xinetd"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 7 7/UDP 13 EXPOSE 7 7/UDP 13

20
coin/provisioning/common/shared/testserver/freecoap/Dockerfile
View File

@@ -1,20 +1,14 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="avahi-daemon autoconf automake libtool make libgnutls28-dev"
RUN apt-get update && apt-get -y install $packages
RUN apt-get update && apt-get -y install git avahi-daemon autoconf automake libtool make libgnutls28-dev
RUN mkdir -p /root/src/
# Get FreeCoAP test server
WORKDIR /root/src WORKDIR /root/src
RUN git clone https://github.com/keith-cullen/FreeCoAP.git ADD FreeCoAP-*.tar.gz .
RUN mv FreeCoAP-* FreeCoAP
WORKDIR /root/src/FreeCoAP WORKDIR /root/src/FreeCoAP
RUN autoreconf --install RUN autoreconf --install && ./configure && make && make install
RUN ./configure WORKDIR sample/time_server
RUN make RUN make
RUN make install
WORKDIR /root/src/FreeCoAP/sample/time_server
RUN make
WORKDIR / WORKDIR /
EXPOSE 5685/udp EXPOSE 5685/udp

4
coin/provisioning/common/shared/testserver/ftp-proxy/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="ftp-proxy avahi-daemon" ARG packages="avahi-daemon ftp-proxy"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 2121 EXPOSE 2121

4
coin/provisioning/common/shared/testserver/ftp-proxy_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="ftp-proxy avahi-daemon" ARG packages="avahi-daemon ftp-proxy"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 2121 EXPOSE 2121

4
coin/provisioning/common/shared/testserver/iptables/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="iptables avahi-daemon" ARG packages="avahi-daemon iptables"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 1357 EXPOSE 1357

4
coin/provisioning/common/shared/testserver/iptables_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="iptables avahi-daemon" ARG packages="avahi-daemon iptables"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 1357 EXPOSE 1357

5
coin/provisioning/common/shared/testserver/qt_ubuntu_16.04/Dockerfile Normal file
View File

@@ -0,0 +1,5 @@
FROM ubuntu:16.04
ARG COIN_RUNS_IN_QT_COMPANY
RUN test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue \
&& sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \
|| echo "Internal package repository not found. Using public repositories."

5
coin/provisioning/common/shared/testserver/qt_ubuntu_18.04/Dockerfile Normal file
View File

@@ -0,0 +1,5 @@
FROM ubuntu:18.04
ARG COIN_RUNS_IN_QT_COMPANY
RUN test x"$COIN_RUNS_IN_QT_COMPANY" = xtrue \
&& sed -i 's;\(archive\|security\)\.ubuntu\.com;repo-clones.ci.qt.io/apt-mirror/mirror;' /etc/apt/sources.list \
|| echo "Internal package repository not found. Using public repositories."

4
coin/provisioning/common/shared/testserver/squid/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="squid avahi-daemon" ARG packages="avahi-daemon squid"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 3128-3130 EXPOSE 3128-3130

4
coin/provisioning/common/shared/testserver/squid_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="squid avahi-daemon" ARG packages="avahi-daemon squid"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 3128-3130 EXPOSE 3128-3130

6
coin/provisioning/common/shared/testserver/vsftpd/Dockerfile
View File

@@ -1,7 +1,7 @@
FROM ubuntu:16.04 FROM qt_ubuntu_16.04
ARG packages="vsftpd ftp wget avahi-daemon" ARG packages="avahi-daemon vsftpd ftp"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 20-21 EXPOSE 20-21
# install configurations and test data # install configurations and test data
RUN wget https://tools.ietf.org/rfc/rfc3252.txt COPY rfc3252.txt .

4
coin/provisioning/common/shared/testserver/vsftpd_18.04/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:18.04 FROM qt_ubuntu_18.04
ARG packages="vsftpd avahi-daemon" ARG packages="avahi-daemon vsftpd"
RUN apt-get update && apt-get install -y $packages && dpkg -l $packages RUN apt-get update && apt-get install -y $packages && dpkg -l $packages
EXPOSE 20-21 EXPOSE 20-21

8
coin/provisioning/common/unix/common.sourced.sh
View File

@@ -43,6 +43,7 @@ export PROVISIONING_ARCH
export CMD_PKG_INSTALL export CMD_PKG_INSTALL
export CMD_PKG_LOCALINSTALL export CMD_PKG_LOCALINSTALL
export CMD_INSTALL export CMD_INSTALL
export COIN_RUNS_IN_QT_COMPANY
@@ -143,6 +144,13 @@ set_common_environment () {
esac esac
CMD_INSTALL="sudo install" CMD_INSTALL="sudo install"
COIN_RUNS_IN_QT_COMPANY=false
if ping -c1 repo-clones.ci.qt.io >/dev/null 2>&1
then
COIN_RUNS_IN_QT_COMPANY=true
fi
} }
set_common_environment set_common_environment