Docker installation: do not depend on external repositories

Also updated the docker-compose binary to the latest one.

Task-number: QTQAINFRA-3279
Change-Id: Ie09a772ca8247aa5f5bd6d36774b68b6bb846386
Reviewed-by: Tony Sarajärvi <tony.sarajarvi@qt.io>
(cherry picked from commit a270bcdb6b)
Reviewed-by: Heikki Halmet <heikki.halmet@qt.io>

coin/provisioning/common/linux/docker.sh

@@ -35,18 +35,37 @@
 
 set -ex
 
-# Download and install the docker engine.
+
-sudo apt-get install curl -y
+. $(dirname "$0")/../../common/unix/DownloadURL.sh
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+
-sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+
-sudo apt-get update
+localRepo=http://ci-files01-hki.intra.qt.io/input/docker
-sudo apt-get install docker-ce -y
+upstreamRepo=https://download.docker.com/linux/ubuntu/dists/bionic/pool/stable/amd64
+
+echo '
+    f4c941807310e3fa470dddfb068d599174a3daec containerd.io_1.2.10-3_amd64.deb
+    ee640d9258fd4d3f4c7017ab2a71da63cbbead55 docker-ce_19.03.4~3-0~ubuntu-bionic_amd64.deb
+    09402bf5dac40f0c50f1071b17f38f6584a42ad1 docker-ce-cli_19.03.4~3-0~ubuntu-bionic_amd64.deb
+' \
+    | xargs -n2 | while read  sha f
+do
+    DownloadURL  $localRepo/$f  $upstreamRepo/$f  $sha
+done
+
+sudo apt-get -y install  ./containerd.io*.deb ./docker-ce*.deb ./docker-ce-cli*.deb
+rm -f                    ./containerd.io*.deb ./docker-ce*.deb ./docker-ce-cli*.deb
+
 sudo usermod -a -G docker $USER
 sudo docker info
 
-# Download and install the docker-compose extension.
+# Download and install the docker-compose extension from https://github.com/docker/compose/releases
-sudo curl -L https://github.com/docker/compose/releases/download/1.21.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
+f=docker-compose-$(uname -s)-$(uname -m)
-sudo chmod +x /usr/local/bin/docker-compose
+DownloadURL  \
+    $localRepo/$f  \
+    https://github.com/docker/compose/releases/download/1.24.1/$f \
+    cfb3439956216b1248308141f7193776fcf4b9c9b49cbbe2fb07885678e2bb8a
+sudo install -m 755 ./docker-compose* /usr/local/bin/docker-compose
+rm ./docker-compose*
 
 # Start testserver provisioning
 source "${BASH_SOURCE%/*}/testserver/docker_testserver.sh"

coin/provisioning/common/linux/remove-update_notifier.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-sudo apt -q -y remove update-notifier update-manager
+sudo apt -q -y remove update-notifier update-manager python3-distupgrade python3-update-manager ubuntu-release-upgrader-core update-manager-core

coin/provisioning/common/unix/DownloadURL.sh

@@ -1,8 +1,8 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2019 The Qt Company Ltd.
 ## Contact: http://www.qt.io/licensing/
 ##
 ## This file is part of the provisioning scripts of the Qt Toolkit.
@@ -33,31 +33,85 @@
 ##
 #############################################################################
 
+
 # A helper script used for downloading a file from a URL or an alternative
-# URL. Also the SHA1 is checked for the file. Target filename should also
+# URL. Also the SHA is checked for the file (SHA algorithm is autodetected
-# be given.
+# based on the SHA length). Target filename should also be given.
-#
-# If called directly from another script, it will exit the parent script
-# as well, if not called in its own subshell with parentheses.
 
-set -ex
+############################ BOILERPLATE ###############################
+command -v sha1sum >/dev/null   ||  alias   sha1sum='shasum -a 1'
+command -v sha256sum >/dev/null ||  alias sha256sum='shasum -a 256'
+command -v sha384sum >/dev/null ||  alias sha384sum='shasum -a 384'
+command -v sha512sum >/dev/null ||  alias sha512sum='shasum -a 512'
+########################################################################
 
-function DownloadURL {
-    url=$1
-    url_alt=$2
-    expectedSha1=$3
-    targetFile=$4
 
-    echo "Downloading from primary URL '$url'"
+Download () {
-    curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" ||  {
+    url="$1"
-        echo "Failed to download '$url' multiple times"
+    targetFile="$2"
-        echo "Downloading from alternative URL '$url_alt'"
-        curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url_alt" || { echo 'Failed to download even from alternative url'; exit 1; }
-    }
 
-    echo "Checking SHA1 on PKG '$targetFile'"
+    command -v curl >/dev/null  \
-    echo "$expectedSha1 *$targetFile" > "$targetFile.sha1"
+        && curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url"  \
-    sha1sum --check "$targetFile.sha1"
+        || wget --tries 5 -O "$targetFile" "$url"
-    rm -f "$targetFile.sha1"
 }
 
+VerifyHash () {
+    file=$1
+    expectedHash=$2
+
+    if [ ! -f "$file" ]
+    then  return 2              # file does not exist
+    fi
+
+
+    hashLength="$(echo "$expectedHash" | wc -c | sed 's/ *//g')"
+    # Use backticks because of bug with bash-3 (default on macOS),
+    # caused when there are unbalanced parentheses inside $()
+    # shellcheck disable=SC2006
+    hash=`case "$hashLength" in
+            41)  sha1sum    "$file"  ;;
+            65)  sha256sum  "$file"  ;;
+            97)  sha384sum  "$file"  ;;
+            129) sha512sum  "$file"  ;;
+            *) echo "FATAL! Unknown hash length:  $hashLength" 1>&2  &&  exit 1  ;;
+        esac | cut -d ' ' -f 1`
+
+    if [ ! "$expectedHash" = "$hash" ]
+    then
+        echo "FAIL! wrong file hash:  $file  $hash"  1>&2
+        return 1
+    fi
+    echo "OK verified integrity of:  $file"
+}
+
+# Check if file already exists and is good, otherwise download it
+DownloadURL () {
+    url=$1
+    url2=$2
+    expectedHash=$3
+    # Optional argument $4: destination filename
+    if [ x"$4" = x ]
+    then
+        # defaults to the last component of $url
+        targetFile=$(echo $url | sed 's|^.*/||')
+    else
+        targetFile=$4
+    fi
+
+    if VerifyHash "$targetFile" "$expectedHash"
+    then
+        echo "Skipping download, found and validated existing file:  $targetFile"
+    else
+        echo "Downloading from primary URL:  $url"
+        if  ! Download "$url" "$targetFile"
+        then
+            echo "FAIL! to download, trying alternative URL:  $url2"  1>&2
+            if  ! Download "$url2" "$targetFile"
+            then
+                echo 'FAIL! to download even from alternative URL'  1>&2
+                return 1
+            fi
+        fi
+        VerifyHash "$targetFile" "$expectedHash"
+    fi
+}

coin/provisioning/qtci-linux-Ubuntu-16.04-x86/02-apt.sh

@@ -44,6 +44,9 @@ for service in apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-dai
     sudo systemctl disable $service
 done
 
+# Stop fetching the dep-11 metadata, since our mirrors do not handle them well
+sudo mv /etc/apt/apt.conf.d/50appstream{,.disabled}
+
 installPackages+=(git)
 installPackages+=(p7zip-full)
 installPackages+=(expect)

coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-apt.sh

@@ -44,6 +44,9 @@ for service in apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-dai
     sudo systemctl disable $service
 done
 
+# Stop fetching the dep-11 metadata, since our mirrors do not handle them well
+sudo mv /etc/apt/apt.conf.d/50appstream{,.disabled}
+
 # aptdaemon is used by update notifiers and similar and there is no point in having those (the symptom is aptd holding a lock)
 for i in $(seq 10); do
     echo attempting to remove aptdaemon
@@ -162,14 +165,14 @@ installPackages+=(dkms)
 installPackages+=(python-pypdf2)
 
 sudo tee "/etc/apt/sources.list" > /dev/null <<-EOC
-deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial main restricted universe multiverse
+deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial main restricted universe multiverse
-deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial-updates main restricted universe multiverse
+deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial-updates main restricted universe multiverse
-deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial-backports main restricted universe
+deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial-backports main restricted universe
-deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial-security main restricted universe multiverse
+deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial-security main restricted universe multiverse
-deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial main restricted universe multiverse
+deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial main restricted universe multiverse
-deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial-updates main restricted universe multiverse
+deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial-updates main restricted universe multiverse
-deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial-backports main restricted universe
+deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial-backports main restricted universe
-deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ xenial-security main restricted universe multiverse
+deb [arch=i386] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ xenial-security main restricted universe multiverse
 EOC
 
 echo "Running update for apt"

coin/provisioning/qtci-linux-Ubuntu-16.04-x86_64/02-remove-update_notifier.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# shellcheck source=../common/linux/remove-update_notifier.sh
+source "${BASH_SOURCE%/*}/../common/linux/remove-update_notifier.sh"

coin/provisioning/qtci-linux-Ubuntu-18.04-x86_64/02-apt.sh

@@ -46,12 +46,15 @@ for service in apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-dai
 done
 
 function set_internal_repo {
+
+    # Stop fetching the dep-11 metadata, since our mirrors do not handle them well
+    sudo mv /etc/apt/apt.conf.d/50appstream{,.disabled}
+
     sudo tee "/etc/apt/sources.list" > /dev/null <<-EOC
-    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ bionic main restricted universe multiverse
+    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ bionic main restricted universe multiverse
-    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ bionic main restricted universe multiverse
+    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ bionic-updates main restricted universe multiverse
-    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ bionic-updates main restricted universe multiverse
+    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ bionic-backports main restricted universe
-    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ bionic-backports main restricted universe
+    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu/ bionic-security main restricted universe multiverse
-    deb [arch=amd64] http://repo-clones.ci.qt.io/apt-mirror/mirror/ubuntu.trumpetti.atm.tut.fi/ubuntu/ bionic-security main restricted universe multiverse
 EOC
 }