Docker Provisioning: Install Docker-based test servers on macOS

Build and cache docker images based on the configurations in testserver.
1. Move the shared Docker files to common/shared/testserver
2. Deploy boot2docker image into VirtualBox via docker-machine commands.
3. Build and tag the docker images by the SHA-1 of server context.

Change-Id: Ic5a588e5cc5e753ec4c4e2e1bc79917bbd03f95d
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

coin/provisioning/common/linux/docker.sh

@@ -2,7 +2,7 @@
 
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: http://www.qt.io/licensing/
 ##
 ## This file is part of the provisioning scripts of the Qt Toolkit.
@@ -49,4 +49,4 @@ sudo curl -L https://github.com/docker/compose/releases/download/1.21.0/docker-c
 sudo chmod +x /usr/local/bin/docker-compose
 
 # Start testserver provisioning
-source "${BASH_SOURCE%/*}/testserver/docker_testserver.sh"
+source "${BASH_SOURCE%/*}/docker_testserver.sh"

coin/provisioning/common/linux/docker_testserver.sh

@@ -2,7 +2,7 @@
 
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: http://www.qt.io/licensing/
 ##
 ## This file is part of the provisioning scripts of the Qt Toolkit.
@@ -35,24 +35,33 @@
 
 set -ex
 
+case ${BASH_SOURCE[0]} in
+    */linux/*) SERVER_PATH="${BASH_SOURCE[0]%/linux/*}/shared/testserver" ;;
+    */*) SERVER_PATH="${BASH_SOURCE[0]%/*}/../shared/testserver" ;;
+    *) SERVER_PATH="../shared/testserver" ;;
+esac
+
+# testserver shared scripts
+source "$SERVER_PATH/testserver_util.sh"
+
 # Using SHA-1 of each server context as the tag of docker images. A tag labels a
 # specific image version. It is used by docker compose file (docker-compose.yml)
 # to launch the corresponding docker containers. If one of the server contexts
 # (./apache2, ./danted, ...) gets changes, all the related compose files in
 # qtbase should be updated as well.
-#
-# For example, here's how to get the SHA-1 of apache test server.
-# find ./apache2 -type f -print0 | sort -z | xargs -r0 sha1sum | \
-# awk '{ print $1 }' | sha1sum | awk '{ print $1 }'
 
-testserver='apache2 squid vsftpd ftp-proxy danted'
+source "$SERVER_PATH/settings.sh"
 
 for server in $testserver
 do
-    context="${BASH_SOURCE%/*}/$server"
+    context="$SERVER_PATH/$server"
+    # obsolete - This one is still needed for CI. It will be removed later.
     sha1=$(find $context -type f -print0 | sort -z | xargs -r0 sha1sum | awk '{ print $1 }' | \
            sha1sum | awk '{ print $1 }')
     sudo docker build -t qt-test-server-$server:$sha1 $context
+
+    # Sort files by their SHA-1 and use the accumulated result as the TAG
+    sudo docker build -t qt-test-server-$server:$(sha1tree $context) $context
 done
 
 sudo docker images

coin/provisioning/common/macos/docker.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: http://www.qt.io/licensing/
+##
+## This file is part of the provisioning scripts of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:LGPL21$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see http://www.qt.io/terms-conditions. For further
+## information use the contact form at http://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 2.1 or version 3 as published by the Free
+## Software Foundation and appearing in the file LICENSE.LGPLv21 and
+## LICENSE.LGPLv3 included in the packaging of this file. Please review the
+## following information to ensure the GNU Lesser General Public License
+## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+##
+## As a special exception, The Qt Company gives you certain additional
+## rights. These rights are described in The Qt Company LGPL Exception
+## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+set -ex
+
+# Download and install the Docker Toolbox for macOS (Docker Compose and Docker Machine).
+url="https://download.docker.com/mac/stable/DockerToolbox.pkg"
+target_file="DockerToolbox.pkg"
+
+if [ -x "$(command -v sha1sum)" ]
+then
+    # This part shall be used in CI environment only. The DownloadURL script needs sha1sum
+    # which is not included in the default macOS system. In addition, the cached pkg can't
+    # be downloaded out of the Qt internal network.
+    case ${BASH_SOURCE[0]} in
+        */macos/*) UNIX_PATH="${BASH_SOURCE[0]%/macos/*}/unix" ;;
+        */*) UNIX_PATH="${BASH_SOURCE[0]%/*}/../unix" ;;
+        *) UNIX_PATH="../unix" ;;
+    esac
+
+    source "$UNIX_PATH/DownloadURL.sh"
+    url_cached="http://ci-files01-hki.ci.local/input/windows/DockerToolbox.pkg"
+    sha1="7196d2d30648d486978d29adb5837ff7876517c1"
+    DownloadURL $url_cached $url $sha1 $target_file
+else
+    curl $url -o $target_file
+fi
+sudo installer -pkg $target_file -target /
+
+# Start testserver provisioning
+source "${BASH_SOURCE%/*}/docker_testserver.sh"

coin/provisioning/common/macos/docker_testserver.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: http://www.qt.io/licensing/
+##
+## This file is part of the provisioning scripts of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:LGPL21$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see http://www.qt.io/terms-conditions. For further
+## information use the contact form at http://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 2.1 or version 3 as published by the Free
+## Software Foundation and appearing in the file LICENSE.LGPLv21 and
+## LICENSE.LGPLv3 included in the packaging of this file. Please review the
+## following information to ensure the GNU Lesser General Public License
+## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+##
+## As a special exception, The Qt Company gives you certain additional
+## rights. These rights are described in The Qt Company LGPL Exception
+## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+set -ex
+
+case ${BASH_SOURCE[0]} in
+    */macos/*) SERVER_PATH="${BASH_SOURCE[0]%/macos/*}/shared/testserver" ;;
+    */*) SERVER_PATH="${BASH_SOURCE[0]%/*}/../shared/testserver" ;;
+    *) SERVER_PATH="../shared/testserver" ;;
+esac
+
+# testserver shared scripts
+source "$SERVER_PATH/testserver_util.sh"
+
+# Create docker virtual machine (Boot2docker)
+source "$SERVER_PATH/docker_machine.sh"
+
+# Using SHA-1 of each server context as the tag of docker images. A tag labels a
+# specific image version. It is used by docker compose file (docker-compose.yml)
+# to launch the corresponding docker containers. If one of the server contexts
+# (./apache2, ./danted, ...) gets changes, all the related compose files in
+# qtbase should be updated as well.
+
+source "$SERVER_PATH/settings.sh"
+
+for server in $testserver
+do
+    context="$SERVER_PATH/$server"
+    docker build -t qt-test-server-$server:$(sha1tree $context) $context
+done
+
+docker images

coin/provisioning/common/shared/testserver/docker_machine.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: http://www.qt.io/licensing/
+##
+## This file is part of the provisioning scripts of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:LGPL21$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see http://www.qt.io/terms-conditions. For further
+## information use the contact form at http://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 2.1 or version 3 as published by the Free
+## Software Foundation and appearing in the file LICENSE.LGPLv21 and
+## LICENSE.LGPLv3 included in the packaging of this file. Please review the
+## following information to ensure the GNU Lesser General Public License
+## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+##
+## As a special exception, The Qt Company gives you certain additional
+## rights. These rights are described in The Qt Company LGPL Exception
+## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+set -ex
+
+TestMachine='qt-test-server'
+
+# Deploy docker virtual machine (Boot2Docker) into VirtualBox only if it doesn't exist
+if [ -z $(docker-machine ls -q --filter "name=$TestMachine") ]
+then
+    docker-machine create -d virtualbox $TestMachine
+    docker-machine ip $TestMachine
+fi
+
+# Switch the docker engine to $TestMachine
+eval $(docker-machine env $TestMachine)
+
+docker-machine ls

coin/provisioning/common/shared/testserver/settings.sh

@@ -0,0 +1,37 @@
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: http://www.qt.io/licensing/
+##
+## This file is part of the provisioning scripts of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:LGPL21$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see http://www.qt.io/terms-conditions. For further
+## information use the contact form at http://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 2.1 or version 3 as published by the Free
+## Software Foundation and appearing in the file LICENSE.LGPLv21 and
+## LICENSE.LGPLv3 included in the packaging of this file. Please review the
+## following information to ensure the GNU Lesser General Public License
+## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+##
+## As a special exception, The Qt Company gives you certain additional
+## rights. These rights are described in The Qt Company LGPL Exception
+## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+set -ex
+
+# A list of test servers to be provisioned
+testserver='apache2 squid vsftpd ftp-proxy danted'

coin/provisioning/common/shared/testserver/testserver_util.sh

@@ -0,0 +1,41 @@
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: http://www.qt.io/licensing/
+##
+## This file is part of the provisioning scripts of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:LGPL21$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see http://www.qt.io/terms-conditions. For further
+## information use the contact form at http://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 2.1 or version 3 as published by the Free
+## Software Foundation and appearing in the file LICENSE.LGPLv21 and
+## LICENSE.LGPLv3 included in the packaging of this file. Please review the
+## following information to ensure the GNU Lesser General Public License
+## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+##
+## As a special exception, The Qt Company gives you certain additional
+## rights. These rights are described in The Qt Company LGPL Exception
+## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+set -ex
+
+# Sort files by their SHA-1, and then return the accumulated result
+sha1tree () {
+    find "$@" -type f -print0 | \
+        xargs -0 shasum | cut -d ' ' -f 1 | \
+        sort | shasum | cut -d ' ' -f 1
+}

coin/provisioning/qtci-macos-10.13-x86_64/80-docker.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+BASEDIR=$(dirname "$0")
+$BASEDIR/../common/macos/docker.sh