From 7b4b2f86a03a176fa5fe0ca1059dc4935c1e665f Mon Sep 17 00:00:00 2001 From: Dominik Holland Date: Mon, 10 Feb 2020 09:15:15 +0100 Subject: [PATCH 01/18] Provisioning: Update jinja2 to 2.10.3 This makes sure we don't run into CVE-2019-10906 when using jinja2 in QtIvis Autogenerator. Change-Id: If51bf54bd26858c6b5faa3dd6a6627f2979b862d Reviewed-by: Robert Griebl Reviewed-by: Jani Heikkinen --- coin/provisioning/common/shared/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coin/provisioning/common/shared/requirements.txt b/coin/provisioning/common/shared/requirements.txt index 60239142..7285caca 100644 --- a/coin/provisioning/common/shared/requirements.txt +++ b/coin/provisioning/common/shared/requirements.txt @@ -9,7 +9,7 @@ argh==0.26.2 click==6.7 coloredlogs==10.0 humanfriendly==4.15.1 -Jinja2==2.10 +Jinja2==2.10.3 MarkupSafe==1.0 path.py==11.0.1 pathtools==0.1.2 From c0033df4b98eee349fcd7b92f35cb6ee4ca18927 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Thu, 6 Feb 2020 19:45:36 +0100 Subject: [PATCH 02/18] Execute a specific version of homebrew install script Also first try to download it from local mirror. This install script appears to be doing a lot of git-cloning and downloading itself though, so it is still far from the ideal solution: to only download from locally cached endpoints. Task-number: QTQAINFRA-3134 Change-Id: I32131045d6e62f302ec666b4ac055ca5ee9662d5 Reviewed-by: Heikki Halmet --- coin/provisioning/common/macos/homebrew.sh | 17 +++++++++++++---- .../qtci-macos-10.13-x86_64/25-homebrew.sh | 2 +- .../qtci-macos-10.14-x86_64/25-homebrew.sh | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/coin/provisioning/common/macos/homebrew.sh b/coin/provisioning/common/macos/homebrew.sh index 715477c0..66463c44 100755 --- a/coin/provisioning/common/macos/homebrew.sh +++ b/coin/provisioning/common/macos/homebrew.sh @@ -1,12 +1,21 @@ -#!/usr/bin/env bash +#!/bin/sh # Will install homebrew package manager for macOS. # WARNING: Requires commandlinetools -# TODO audit and cache this file locally, see QTQAINFRA-3134 -curl -L -o /tmp/homebrew_install https://raw.githubusercontent.com/Homebrew/install/master/install +set -e + +. "$(dirname "$0")"/../../common/unix/DownloadURL.sh + + +DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/mac/homebrew-install.c744a716f9845988d01e6e238eee7117b8c366c9.rb \ + https://raw.githubusercontent.com/Homebrew/install/c744a716f9845988d01e6e238eee7117b8c366c9/install \ + b9782cc0b550229de77b429b56ffce04157e60486ab9df00461ccf3dad565b0a \ + /tmp/homebrew_install /usr/bin/ruby /tmp/homebrew_install Date: Mon, 10 Feb 2020 17:21:32 +0100 Subject: [PATCH 03/18] Alias expansion is not working reliably in non-interactive shells It happened that old bash version on macOS the alias was not being expanded for non-interactive shells running under "bash" but was being expanded fine when under "sh". Using functions is the recommended way. Here we define a function conditionally. Change-Id: I607b265cb9e88b2b529ed170121e5a0833003acd Reviewed-by: Heikki Halmet --- coin/provisioning/common/unix/DownloadURL.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/coin/provisioning/common/unix/DownloadURL.sh b/coin/provisioning/common/unix/DownloadURL.sh index 5bafc33d..5a3130af 100755 --- a/coin/provisioning/common/unix/DownloadURL.sh +++ b/coin/provisioning/common/unix/DownloadURL.sh @@ -39,10 +39,12 @@ # based on the SHA length). Target filename should also be given. ############################ BOILERPLATE ############################### -command -v sha1sum >/dev/null || alias sha1sum='shasum -a 1' -command -v sha256sum >/dev/null || alias sha256sum='shasum -a 256' -command -v sha384sum >/dev/null || alias sha384sum='shasum -a 384' -command -v sha512sum >/dev/null || alias sha512sum='shasum -a 512' + +command -v sha1sum >/dev/null || sha1sum () { shasum -a 1 "$@" ; } +command -v sha256sum >/dev/null || sha256sum () { shasum -a 256 "$@" ; } +command -v sha384sum >/dev/null || sha384sum () { shasum -a 384 "$@" ; } +command -v sha512sum >/dev/null || sha512sum () { shasum -a 512 "$@" ; } + ######################################################################## From b018b841c1ce49b412fa981a65b90a4778e933de Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Fri, 7 Feb 2020 16:00:40 +0100 Subject: [PATCH 04/18] Do not complain about existing empty files Sometimes we pre-create the target filename using mktemp. In that case of course the hash will not match, so avoid comparing. Change-Id: Id0feb0178d659e03f5ceb000f738167cdc28a3ec Reviewed-by: Heikki Halmet --- coin/provisioning/common/unix/DownloadURL.sh | 31 ++++++++++++-------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/coin/provisioning/common/unix/DownloadURL.sh b/coin/provisioning/common/unix/DownloadURL.sh index 5a3130af..1d48a565 100755 --- a/coin/provisioning/common/unix/DownloadURL.sh +++ b/coin/provisioning/common/unix/DownloadURL.sh @@ -100,20 +100,27 @@ DownloadURL () { targetFile=$4 fi - if VerifyHash "$targetFile" "$expectedHash" + # If a non-empty file already exists + if [ -s "$targetFile" ] then - echo "Skipping download, found and validated existing file: $targetFile" - else - echo "Downloading from primary URL: $url" - if ! Download "$url" "$targetFile" + if VerifyHash "$targetFile" "$expectedHash" then - echo "FAIL! to download, trying alternative URL: $url2" 1>&2 - if ! Download "$url2" "$targetFile" - then - echo 'FAIL! to download even from alternative URL' 1>&2 - return 1 - fi + echo "Skipping download, found and validated existing file: $targetFile" + return + else + echo "WARNING: Non-empty but different file found at destination; will re-download and overwrite file: $targetFile" fi - VerifyHash "$targetFile" "$expectedHash" fi + + echo "Downloading from primary URL: $url" + if ! Download "$url" "$targetFile" + then + echo "FAIL! to download, trying alternative URL: $url2" 1>&2 + if ! Download "$url2" "$targetFile" + then + echo 'FAIL! to download even from alternative URL' 1>&2 + return 1 + fi + fi + VerifyHash "$targetFile" "$expectedHash" } From fc2d0b5150045d525c5ff629564baaa4f9ecb535 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Fri, 7 Feb 2020 15:59:00 +0100 Subject: [PATCH 05/18] Create more meaningful temporary file names Change-Id: I4bbc94993e51502ea08109a4c99f3e212349ab24 Reviewed-by: Heikki Halmet --- .../provisioning/common/macos/InstallPKGFromURL.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/coin/provisioning/common/macos/InstallPKGFromURL.sh b/coin/provisioning/common/macos/InstallPKGFromURL.sh index 77ef128a..d9b55d3b 100755 --- a/coin/provisioning/common/macos/InstallPKGFromURL.sh +++ b/coin/provisioning/common/macos/InstallPKGFromURL.sh @@ -42,7 +42,9 @@ function InstallPKGFromURL { targetDirectory=$4 echo "Creating temporary file" - targetFile=$(mktemp "$TMPDIR$(uuidgen).pkg") + package_basename=$(echo "$url" | sed 's|^.*/||') + tmpdir=$(mktemp -d) + targetFile="$tmpdir/$package_basename" echo "Downloading PKG from primary URL '$url'" curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url" || ( echo "Failed to download '$url' multiple times" @@ -50,10 +52,12 @@ function InstallPKGFromURL { curl --fail -L --retry 5 --retry-delay 5 -o "$targetFile" "$url_alt" ) echo "Checking SHA1 on PKG '$targetFile'" - echo "$expectedSha1 *$targetFile" > "$targetFile.sha1" - /usr/bin/shasum --check "$targetFile.sha1" + echo "$expectedSha1 *$targetFile" > "$targetFile".sha1 + /usr/bin/shasum --check "$targetFile".sha1 echo "Run installer on PKG" sudo installer -package "$targetFile" -target "$targetDirectory" - echo "Removing file '$targetFile'" - rm "$targetFile" + + rm -f "$targetFile".sha1 + rm -f "$targetFile" + rmdir "$tmpdir" } From 2693e482b625af0ea73fdf5dc9ab9d2bdabdd08a Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 5 Feb 2020 11:27:30 +0100 Subject: [PATCH 06/18] Extract archive directly to destination filesystem Save some seconds or even minutes during provisioning, by avoiding copying many files across filesystems, from /tmp to /usr/local. We now install directly to /usr/local. Change-Id: I3a8f545f1400995095193c4eb337f9dd8534ef72 Reviewed-by: Heikki Halmet --- coin/provisioning/common/unix/libclang.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coin/provisioning/common/unix/libclang.sh b/coin/provisioning/common/unix/libclang.sh index a9d1cc49..6fa6c040 100755 --- a/coin/provisioning/common/unix/libclang.sh +++ b/coin/provisioning/common/unix/libclang.sh @@ -65,10 +65,10 @@ zip="/tmp/libclang.7z" destination="/usr/local/libclang-$version" DownloadURL $url_cached $url $sha1 $zip -7z x $zip -o/tmp/ +sudo 7z x $zip -o/usr/local/ +sudo mv /usr/local/libclang "$destination" rm -rf $zip -sudo mv /tmp/libclang "$destination" echo "export LLVM_INSTALL_DIR=$destination" >> ~/.bash_profile echo "libClang = $version" >> ~/versions.txt From db23c64876028d8c4584b58c049f0211764c775a Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Tue, 4 Feb 2020 18:30:04 +0100 Subject: [PATCH 07/18] 08-pythondev build should not execute external script The important bits of the script were copied and modified. NOTE that we no longer build openssl from source, the distro (SLES15) provides a version that works well. Task-number: QTQAINFRA-3528 Change-Id: Iad63d75055a7efc3e104789e4a515b8511cf0880 Reviewed-by: Heikki Halmet --- .../qtci-linux-SLES-15-x86_64/08-pythondev.sh | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh index cda48de7..ab35a009 100755 --- a/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh +++ b/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh @@ -41,11 +41,45 @@ set -ex sudo zypper -nq install python-devel -wget -q https://raw.githubusercontent.com/linux-on-ibm-z/scripts/master/Python3/build_python3.sh -bash build_python3.sh +# Selected installation instructions coming from: +# https://raw.githubusercontent.com/linux-on-ibm-z/scripts/master/Python3/build_python3.sh +export PACKAGE_NAME="python" +export PACKAGE_VERSION="3.7.2" +( -export -python3 --version + sudo zypper install -y gcc gcc-c++ make ncurses patch wget tar zlib-devel zlib libffi-devel libopenssl-devel + + printf -- 'Configuration and Installation started \n' + + #Downloading Source code + wget "https://www.python.org/ftp/${PACKAGE_NAME}/${PACKAGE_VERSION}/Python-${PACKAGE_VERSION}.tar.xz" + tar -xvf "Python-${PACKAGE_VERSION}.tar.xz" + + #Configure and Build + cd "Python-${PACKAGE_VERSION}" + ./configure --prefix=/usr/local --exec-prefix=/usr/local + make + sudo make install + + export PATH="/usr/local/bin:${PATH}" + printf -- '\nInstalled python successfully \n' + + #Cleanup + cd - + rm "Python-${PACKAGE_VERSION}.tar.xz" + + #Verify python installation + if command -V "$PACKAGE_NAME"${PACKAGE_VERSION:0:1} >/dev/null + then + printf -- "%s installation completed. Please check the Usage to start the service.\n" "$PACKAGE_NAME" + else + printf -- "Error while installing %s, exiting with 127 \n" "$PACKAGE_NAME" + exit 127 + fi +) + + +python3 --version | fgrep "$PACKAGE_VERSION" pip3 install --user wheel pip3 install --user virtualenv From 5dda5f374fa6c6044dc4bc76446ff59924110b23 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Tue, 4 Feb 2020 13:43:40 +0100 Subject: [PATCH 08/18] Mask PackageKit on SUSE Fixes a flaky conflict between zypper and packagekitd. packagekit is a *static* unit, that means it is missing an [Install] section, so it can't be enabled or disabled. It is started as a dependency by other units, or by dbus activation. We *mask* the unit to make it impossible to start in any way. For that we had replace a few occurrences of "pkcon" with the native package manager. Finally the package "libpython3_4m1_0" is not being installed any longer because it never was: this package does not exist. It's just that pkcon does not fail on invalid package names. Fixes: QTQAINFRA-3523 Change-Id: I6ff580afb05aa508042c444b2ce777c3a9443dfe Reviewed-by: Heikki Halmet --- .../provisioning/qtci-linux-SLES-15-x86_64/01-systemsetup.sh | 2 ++ .../qtci-linux-openSUSE-15.0-x86_64/01-systemsetup.sh | 3 +++ .../qtci-linux-openSUSE-15.0-x86_64/08-pythondev.sh | 5 ++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/coin/provisioning/qtci-linux-SLES-15-x86_64/01-systemsetup.sh b/coin/provisioning/qtci-linux-SLES-15-x86_64/01-systemsetup.sh index ee0b6894..28f25728 100755 --- a/coin/provisioning/qtci-linux-SLES-15-x86_64/01-systemsetup.sh +++ b/coin/provisioning/qtci-linux-SLES-15-x86_64/01-systemsetup.sh @@ -59,10 +59,12 @@ echo 'export DISPLAY=":0"' >> ~/.bashrc sudo systemctl stop packagekit sudo systemctl disable packagekit +sudo systemctl mask packagekit while sudo fuser /usr/lib/packagekitd >/dev/null 2>&1 ; do echo "Waiting for PackageKit to finish..." sleep 5 done +sudo zypper -nq remove gnome-software # shellcheck disable=SC2031 if [ "$http_proxy" != "" ]; then diff --git a/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/01-systemsetup.sh b/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/01-systemsetup.sh index 7e6993a5..d3724ceb 100755 --- a/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/01-systemsetup.sh +++ b/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/01-systemsetup.sh @@ -23,6 +23,9 @@ echo 'export DISPLAY=":0"' >> ~/.bashrc echo "Disabling file indexing." sudo balooctl disable +sudo systemctl stop packagekit +sudo systemctl disable packagekit +sudo systemctl mask packagekit while sudo fuser /usr/lib/packagekitd >/dev/null 2>&1 ; do echo "Waiting for PackageKit to finish..." sleep 0.5 diff --git a/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/08-pythondev.sh index 6c5d2a0f..12dabace 100755 --- a/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/08-pythondev.sh +++ b/coin/provisioning/qtci-linux-openSUSE-15.0-x86_64/08-pythondev.sh @@ -5,11 +5,10 @@ set -ex -sudo pkcon -y refresh -sudo pkcon -y install python-devel python-virtualenv +sudo zypper -nq install python-devel python-virtualenv # install python3 -sudo pkcon -y install libpython3_4m1_0 python3-base python3 python3-pip python3-devel python3-virtualenv python3-wheel +sudo zypper -nq install python3-base python3 python3-pip python3-devel python3-virtualenv python3-wheel # Install all needed packages in a special wheel cache directory pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt" From de2b280ad85c827d70ac6f7e4d2fb399780468f9 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Fri, 31 Jan 2020 19:46:11 +0100 Subject: [PATCH 09/18] Windows now report versions.txt and df info at the end of provisioning Change-Id: Ic2b299c04c91eb498822aca110cea774fdb7cf17 Reviewed-by: Heikki Halmet --- coin/provisioning/common/windows/version.ps1 | 6 ++++++ coin/provisioning/qtci-windows-10-x86/99-version.ps1 | 1 + coin/provisioning/qtci-windows-10-x86_64/99-version.ps1 | 1 + coin/provisioning/qtci-windows-7-x86/99-version.ps1 | 1 + coin/provisioning/qtci-windows-7-x86_64/99-version.ps1 | 1 + coin/provisioning/qtci-windows-8.1-x86/99-version.ps1 | 1 + coin/provisioning/qtci-windows-8.1-x86_64/99-version.ps1 | 1 + 7 files changed, 12 insertions(+) create mode 100644 coin/provisioning/common/windows/version.ps1 create mode 100644 coin/provisioning/qtci-windows-10-x86/99-version.ps1 create mode 100644 coin/provisioning/qtci-windows-10-x86_64/99-version.ps1 create mode 100644 coin/provisioning/qtci-windows-7-x86/99-version.ps1 create mode 100644 coin/provisioning/qtci-windows-7-x86_64/99-version.ps1 create mode 100644 coin/provisioning/qtci-windows-8.1-x86/99-version.ps1 create mode 100644 coin/provisioning/qtci-windows-8.1-x86_64/99-version.ps1 diff --git a/coin/provisioning/common/windows/version.ps1 b/coin/provisioning/common/windows/version.ps1 new file mode 100644 index 00000000..7f701f63 --- /dev/null +++ b/coin/provisioning/common/windows/version.ps1 @@ -0,0 +1,6 @@ +Write-Host '******************** SW VERSIONS ********************' +type ~/versions.txt +Write-Host '*****************************************************' +Write-Host '******************** Get-PSDrive ********************' +Get-PSDrive +Write-Host '*****************************************************' diff --git a/coin/provisioning/qtci-windows-10-x86/99-version.ps1 b/coin/provisioning/qtci-windows-10-x86/99-version.ps1 new file mode 100644 index 00000000..36e498bc --- /dev/null +++ b/coin/provisioning/qtci-windows-10-x86/99-version.ps1 @@ -0,0 +1 @@ +. "$PSScriptRoot\..\common\windows\version.ps1" diff --git a/coin/provisioning/qtci-windows-10-x86_64/99-version.ps1 b/coin/provisioning/qtci-windows-10-x86_64/99-version.ps1 new file mode 100644 index 00000000..36e498bc --- /dev/null +++ b/coin/provisioning/qtci-windows-10-x86_64/99-version.ps1 @@ -0,0 +1 @@ +. "$PSScriptRoot\..\common\windows\version.ps1" diff --git a/coin/provisioning/qtci-windows-7-x86/99-version.ps1 b/coin/provisioning/qtci-windows-7-x86/99-version.ps1 new file mode 100644 index 00000000..36e498bc --- /dev/null +++ b/coin/provisioning/qtci-windows-7-x86/99-version.ps1 @@ -0,0 +1 @@ +. "$PSScriptRoot\..\common\windows\version.ps1" diff --git a/coin/provisioning/qtci-windows-7-x86_64/99-version.ps1 b/coin/provisioning/qtci-windows-7-x86_64/99-version.ps1 new file mode 100644 index 00000000..36e498bc --- /dev/null +++ b/coin/provisioning/qtci-windows-7-x86_64/99-version.ps1 @@ -0,0 +1 @@ +. "$PSScriptRoot\..\common\windows\version.ps1" diff --git a/coin/provisioning/qtci-windows-8.1-x86/99-version.ps1 b/coin/provisioning/qtci-windows-8.1-x86/99-version.ps1 new file mode 100644 index 00000000..36e498bc --- /dev/null +++ b/coin/provisioning/qtci-windows-8.1-x86/99-version.ps1 @@ -0,0 +1 @@ +. "$PSScriptRoot\..\common\windows\version.ps1" diff --git a/coin/provisioning/qtci-windows-8.1-x86_64/99-version.ps1 b/coin/provisioning/qtci-windows-8.1-x86_64/99-version.ps1 new file mode 100644 index 00000000..36e498bc --- /dev/null +++ b/coin/provisioning/qtci-windows-8.1-x86_64/99-version.ps1 @@ -0,0 +1 @@ +. "$PSScriptRoot\..\common\windows\version.ps1" From 47b74fb0ab4775b83c197729fe188e5c4c0d169e Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Fri, 31 Jan 2020 18:49:08 +0100 Subject: [PATCH 10/18] Print filesystem information at the end of each provisioning Change-Id: I549a917126954fae62dcdebb5c45a4f6aeb07551 Reviewed-by: Heikki Halmet --- coin/provisioning/common/linux/version.sh | 6 ++++++ coin/provisioning/common/macos/version.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/coin/provisioning/common/linux/version.sh b/coin/provisioning/common/linux/version.sh index 3e2a7f19..9e23a77e 100755 --- a/coin/provisioning/common/linux/version.sh +++ b/coin/provisioning/common/linux/version.sh @@ -41,3 +41,9 @@ echo "*********************************************" echo "***** SW VERSIONS *****" cat ~/versions.txt echo "*********************************************" +echo "*************** mount ***********************" +mount +echo "*********************************************" +echo "*************** df **************************" +df -hT +echo "*********************************************" diff --git a/coin/provisioning/common/macos/version.sh b/coin/provisioning/common/macos/version.sh index 3e2a7f19..e57fb60b 100755 --- a/coin/provisioning/common/macos/version.sh +++ b/coin/provisioning/common/macos/version.sh @@ -41,3 +41,9 @@ echo "*********************************************" echo "***** SW VERSIONS *****" cat ~/versions.txt echo "*********************************************" +echo "*************** mount ***********************" +mount +echo "*********************************************" +echo "*************** df **************************" +df -h +echo "*********************************************" From 27e42b7a3c0ff050dfa6a16922d7ee64b7a58b30 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 29 Jan 2020 15:54:08 +0100 Subject: [PATCH 11/18] Silence sdkmanager on Windows sdkmanager is already silenced on the other OSes, as it prints thousands of lines of progress bars in the logfiles. Change-Id: I4a2aa422a83cfe5fc6a3a7ba7be69e1e9c01acc0 Reviewed-by: Heikki Halmet --- coin/provisioning/common/windows/android.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coin/provisioning/common/windows/android.ps1 b/coin/provisioning/common/windows/android.ps1 index 537e14a2..0ac07f8a 100644 --- a/coin/provisioning/common/windows/android.ps1 +++ b/coin/provisioning/common/windows/android.ps1 @@ -97,7 +97,7 @@ Out-File -FilePath C:\Utils\Android\licenses\android-sdk-license -Encoding utf8 cd $toolsFolder\bin\ $sdkmanager_args += "platforms;$sdkApiLevel", "platform-tools", "build-tools;$sdkBuildToolsVersion" -$command = 'for($i=0;$i -lt 6;$i++) { $response += "y`n"}; $response | .\sdkmanager.bat @sdkmanager_args' +$command = 'for($i=0;$i -lt 6;$i++) { $response += "y`n"}; $response | .\sdkmanager.bat @sdkmanager_args | Out-Null' Invoke-Expression $command $command = 'for($i=0;$i -lt 6;$i++) { $response += "y`n"}; $response | .\sdkmanager.bat --licenses' iex $command From d84b366d555e42797cf242c3534d29eb1f8eb45e Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 29 Jan 2020 15:48:14 +0100 Subject: [PATCH 12/18] Silence the sdkmanager progress bar on macos The exact same method is already in use on all Linux provisions, it was just copied to macos. Change-Id: I873f6efd99b147c5364e0dda6c26c7dad7dee480 Reviewed-by: Heikki Halmet --- coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh | 7 ++++++- coin/provisioning/qtci-macos-10.14-x86_64/30-android.sh | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh b/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh index 1886cce3..aa0935f0 100755 --- a/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh +++ b/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh @@ -68,8 +68,13 @@ sudo unzip -q "$toolsSourceFile" -d "$sdkTargetFolder" echo "Changing ownership of Android files." sudo chown -R qt:wheel "$targetFolder" +# Run the following command under `eval` or `sh -c` so that the shell properly splits it +sdkmanager_no_progress_bar_cmd="tr '\r' '\n' | grep -v '^\[[ =]*\]'" + echo "Running SDK manager for platforms;$sdkApiLevel, platform-tools and build-tools;$sdkBuildToolsVersion." -(echo "y"; echo "y") |"$sdkTargetFolder/tools/bin/sdkmanager" "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" +(echo "y"; echo "y") | "$sdkTargetFolder/tools/bin/sdkmanager" \ + "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" \ + | eval $sdkmanager_no_progress_bar_cmd echo "Checking the contents of Android SDK..." ls -l "$sdkTargetFolder" diff --git a/coin/provisioning/qtci-macos-10.14-x86_64/30-android.sh b/coin/provisioning/qtci-macos-10.14-x86_64/30-android.sh index 1886cce3..aa0935f0 100755 --- a/coin/provisioning/qtci-macos-10.14-x86_64/30-android.sh +++ b/coin/provisioning/qtci-macos-10.14-x86_64/30-android.sh @@ -68,8 +68,13 @@ sudo unzip -q "$toolsSourceFile" -d "$sdkTargetFolder" echo "Changing ownership of Android files." sudo chown -R qt:wheel "$targetFolder" +# Run the following command under `eval` or `sh -c` so that the shell properly splits it +sdkmanager_no_progress_bar_cmd="tr '\r' '\n' | grep -v '^\[[ =]*\]'" + echo "Running SDK manager for platforms;$sdkApiLevel, platform-tools and build-tools;$sdkBuildToolsVersion." -(echo "y"; echo "y") |"$sdkTargetFolder/tools/bin/sdkmanager" "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" +(echo "y"; echo "y") | "$sdkTargetFolder/tools/bin/sdkmanager" \ + "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion" \ + | eval $sdkmanager_no_progress_bar_cmd echo "Checking the contents of Android SDK..." ls -l "$sdkTargetFolder" From 150f6277e073f4f13194247eb272583328a54913 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 29 Jan 2020 15:10:30 +0100 Subject: [PATCH 13/18] Update telegraf version Change-Id: I1a023e4f1de860a03a135f3b10f2c99d5309dd8b Reviewed-by: Heikki Halmet --- .../shared/telegraf/telegraf_packages.sha256.txt | 10 ++++------ coin/provisioning/common/unix/telegraf_install.sh | 2 +- coin/provisioning/common/windows/install_telegraf.ps1 | 6 +++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt b/coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt index 7d1cdada..28304114 100644 --- a/coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt +++ b/coin/provisioning/common/shared/telegraf/telegraf_packages.sha256.txt @@ -1,6 +1,4 @@ -https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_windows_i386.zip d2dea67a28fd7a9e21395d14f0b7ce1b6413e85cc9ffd89cc7ff8f35e8af5837 -https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_windows_amd64.zip 88d8c8146b561f43726e61688b42d0b5ab88fc22607b94df5a2c8245bbb27e29 -https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_darwin_amd64.tar.gz 58a2b294c1b6408c396b5526719cf7cad557f7075fed10ff0b2a74141ea2d629 -https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_linux_amd64.tar.gz 070a76800d1a5f33413b8948286f75caa55ebd924f6fb6996a753b9979799ef5 -https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1_linux_i386.tar.gz 964077b6846708f579a100d7ea6090e2d8350ace75ec735d539bc452bc551038 -https://dl.influxdata.com/telegraf/releases/telegraf-1.12.1-static_linux_amd64.tar.gz bb03814c99c037ce0758e3884660910f2f6fd78f3d49a8e1ef531693b3cfe25d +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.6_darwin_amd64.tar.gz 2bc056e1ad38af2ba95d2cb1d43b513cc611fdd3b851b2ff5e09991d52b01253 +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.6_linux_amd64.tar.gz b33cf0f5935a5c225ab0de69db1e6b249d581f3213313de62a8de928aedadbec +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.6_windows_amd64.zip 5d025d85070e8c180c443580afa8a27421a7bbcf14b5044894e9f3298d0ce97a +https://dl.influxdata.com/telegraf/releases/telegraf-1.12.6_windows_i386.zip 5178a0bdaab448c2ef965b0e36f835849cea731ccd87c4a4527f0f05fbbdf271 diff --git a/coin/provisioning/common/unix/telegraf_install.sh b/coin/provisioning/common/unix/telegraf_install.sh index 1412650e..b2dd244e 100755 --- a/coin/provisioning/common/unix/telegraf_install.sh +++ b/coin/provisioning/common/unix/telegraf_install.sh @@ -73,7 +73,7 @@ $CMD_INSTALL -m 755 "$PROVISIONING_DIR"/common/shared/telegraf/telegraf-ioping. [ x"$PROVISIONING_OS" = xmacos ] && os=darwin || os=linux [ x"$PROVISIONING_ARCH" = xx86 ] && arch=i386 || arch=amd64 -package_filename=telegraf-1.12.1_${os}_${arch}.tar.gz +package_filename=telegraf-1.12.6_${os}_${arch}.tar.gz package_sha256_list="$PROVISIONING_DIR"/common/shared/telegraf/telegraf_packages.sha256.txt package_sha256=$(sed -n "s/.*$package_filename *//p" "$package_sha256_list") diff --git a/coin/provisioning/common/windows/install_telegraf.ps1 b/coin/provisioning/common/windows/install_telegraf.ps1 index fee550b1..0a64cfd4 100644 --- a/coin/provisioning/common/windows/install_telegraf.ps1 +++ b/coin/provisioning/common/windows/install_telegraf.ps1 @@ -37,14 +37,14 @@ # Install a specific version of Telegraf # https://github.com/influxdata/telegraf/releases -$version = "1.12.1" +$version = "1.12.6" if (Is64BitWinHost) { $arch = "amd64" - $sha256 = "88d8c8146b561f43726e61688b42d0b5ab88fc22607b94df5a2c8245bbb27e29" + $sha256 = "5d025d85070e8c180c443580afa8a27421a7bbcf14b5044894e9f3298d0ce97a" } else { $arch = "i386" - $sha256 = "d2dea67a28fd7a9e21395d14f0b7ce1b6413e85cc9ffd89cc7ff8f35e8af5837" + $sha256 = "5178a0bdaab448c2ef965b0e36f835849cea731ccd87c4a4527f0f05fbbdf271" } $filename = "telegraf-" + $version + "_windows_" + $arch + ".zip" From b2d3df3553edec087cbe063813b591b8fed0061a Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 29 Jan 2020 14:48:07 +0100 Subject: [PATCH 14/18] Exclude monitoring of some filesystems on macos For a short time after boot, macos sometimes mounts a filesystem of type "msdos" at the unique path /Volumes/firmwaresyncd.XXXXXX. Telegraf [inputs.disk] plug-in stores records to InfluxDB, with the unique mount path under "path" tag. The uniqueness of the path blows up the cardinality of the database. So exclude "msdos" filesystems from being monitored by telegraf. Additionally exclude "cd9660" filesystems that appear always as 100% full. Change-Id: I4cdd3fca57b3db6157df168cc8a5314e64528b2d Reviewed-by: Heikki Halmet --- coin/provisioning/common/macos/telegraf-coin.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/coin/provisioning/common/macos/telegraf-coin.conf b/coin/provisioning/common/macos/telegraf-coin.conf index 24440a05..006de88c 100644 --- a/coin/provisioning/common/macos/telegraf-coin.conf +++ b/coin/provisioning/common/macos/telegraf-coin.conf @@ -24,7 +24,13 @@ # collect_cpu_time = false # report_active = false [[inputs.disk]] - ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + # macOS needs two extra filesystem exceptions: + # + cd9660: This is a read-only filesystem, and always 100% full + # + msdos: Sometimes at boot a short-lived unique mount of type "msdos" + # is performed, at the random directory + # /Volumes/firmwaresyncd.XXXXXX. + # The uniqueness of the path bloats the cardinality of InfluxDB. + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs", "cd9660", "msdos"] #[[inputs.diskio]] # linux-only [[inputs.kernel]] [[inputs.mem]] From 5d431ecc974f9d010dce17bdd4cd15e46a56622a Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 5 Feb 2020 10:21:27 +0100 Subject: [PATCH 15/18] Refactor python3 installation in SLES-15 provisioning + Use DownloadURL to download from local cache + Verify download SHA + Silence the tarball extraction + Install some generic packages in 02-zypper.sh + Use some other generic facilities from common.sourced.sh + Other minor changes Task-number: QTQAINFRA-3528 Change-Id: Ic085d43381e220a0a4a52ef6d5bb0cf8a9bf7032 Reviewed-by: Heikki Halmet --- .../02-zypperpackages.sh | 3 +-- .../qtci-linux-SLES-15-x86_64/08-pythondev.sh | 26 +++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/coin/provisioning/qtci-linux-SLES-15-x86_64/02-zypperpackages.sh b/coin/provisioning/qtci-linux-SLES-15-x86_64/02-zypperpackages.sh index 12f3a673..3ea67ef0 100755 --- a/coin/provisioning/qtci-linux-SLES-15-x86_64/02-zypperpackages.sh +++ b/coin/provisioning/qtci-linux-SLES-15-x86_64/02-zypperpackages.sh @@ -35,10 +35,9 @@ set -ex - sudo zypper -nq install elfutils binutils -sudo zypper -nq install git gcc gcc-c++ +sudo zypper -nq install git gcc gcc-c++ make patch wget tar sudo zypper -nq install bison flex gperf \ zlib-devel \ diff --git a/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh index ab35a009..8211dd96 100755 --- a/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh +++ b/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh @@ -39,21 +39,31 @@ set -ex -sudo zypper -nq install python-devel +PROVISIONING_DIR="$(dirname "$0")/../" +. "$PROVISIONING_DIR"/common/unix/common.sourced.sh +. "$PROVISIONING_DIR"/common/unix/DownloadURL.sh + + +# Python 2 +$CMD_PKG_INSTALL python-devel # Selected installation instructions coming from: # https://raw.githubusercontent.com/linux-on-ibm-z/scripts/master/Python3/build_python3.sh export PACKAGE_NAME="python" export PACKAGE_VERSION="3.7.2" +export PACKAGE_SHA=d83fe8ce51b1bb48bbcf0550fd265b9a75cdfdfa93f916f9e700aef8444bf1bb ( - sudo zypper install -y gcc gcc-c++ make ncurses patch wget tar zlib-devel zlib libffi-devel libopenssl-devel + $CMD_PKG_INSTALL ncurses zlib-devel libffi-devel libopenssl-devel - printf -- 'Configuration and Installation started \n' + echo 'Configuration and Installation started' - #Downloading Source code - wget "https://www.python.org/ftp/${PACKAGE_NAME}/${PACKAGE_VERSION}/Python-${PACKAGE_VERSION}.tar.xz" - tar -xvf "Python-${PACKAGE_VERSION}.tar.xz" + #Download Source code + DownloadURL \ + http://ci-files01-hki.intra.qt.io/input/python/Python-${PACKAGE_VERSION}.tar.xz \ + https://www.python.org/ftp/${PACKAGE_NAME}/${PACKAGE_VERSION}/Python-${PACKAGE_VERSION}.tar.xz \ + $PACKAGE_SHA + tar -xf "Python-${PACKAGE_VERSION}.tar.xz" #Configure and Build cd "Python-${PACKAGE_VERSION}" @@ -61,14 +71,14 @@ export PACKAGE_VERSION="3.7.2" make sudo make install - export PATH="/usr/local/bin:${PATH}" - printf -- '\nInstalled python successfully \n' + echo 'Installed python successfully' #Cleanup cd - rm "Python-${PACKAGE_VERSION}.tar.xz" #Verify python installation + export PATH="/usr/local/bin:${PATH}" if command -V "$PACKAGE_NAME"${PACKAGE_VERSION:0:1} >/dev/null then printf -- "%s installation completed. Please check the Usage to start the service.\n" "$PACKAGE_NAME" From 0d208f77ec8c18f5907fbacc94a2b420e2710b3a Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Wed, 5 Feb 2020 15:52:08 +0100 Subject: [PATCH 16/18] Update python3 version on SLES-15 We update to the latest 3.7.x maintenance release for security/stability. Change-Id: Ic2c251ab8e0db2c5e622f685e54e5cdd715723fc Reviewed-by: Heikki Halmet --- coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh index 8211dd96..2da45c2b 100755 --- a/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh +++ b/coin/provisioning/qtci-linux-SLES-15-x86_64/08-pythondev.sh @@ -50,8 +50,8 @@ $CMD_PKG_INSTALL python-devel # Selected installation instructions coming from: # https://raw.githubusercontent.com/linux-on-ibm-z/scripts/master/Python3/build_python3.sh export PACKAGE_NAME="python" -export PACKAGE_VERSION="3.7.2" -export PACKAGE_SHA=d83fe8ce51b1bb48bbcf0550fd265b9a75cdfdfa93f916f9e700aef8444bf1bb +export PACKAGE_VERSION="3.7.6" +export PACKAGE_SHA=55a2cce72049f0794e9a11a84862e9039af9183603b78bc60d89539f82cf533f ( $CMD_PKG_INSTALL ncurses zlib-devel libffi-devel libopenssl-devel From d757c6d0ece7823a28c8601a56f567580aa8cf16 Mon Sep 17 00:00:00 2001 From: Dimitrios Apostolou Date: Thu, 6 Feb 2020 17:19:55 +0100 Subject: [PATCH 17/18] Try to catch all errors when sdkmanager installs Task-number: QTQAINFRA-3531 Change-Id: I008c31a824287c60e67a665d0c2a3aebec863929 Reviewed-by: Heikki Halmet --- coin/provisioning/common/linux/android_linux.sh | 7 ++++++- coin/provisioning/common/windows/android.ps1 | 3 +++ coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh | 7 +++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/coin/provisioning/common/linux/android_linux.sh b/coin/provisioning/common/linux/android_linux.sh index 44bd2281..ca9110b8 100755 --- a/coin/provisioning/common/linux/android_linux.sh +++ b/coin/provisioning/common/linux/android_linux.sh @@ -37,6 +37,8 @@ # It also runs update for SDK API, latest SDK tools, latest platform-tools and build-tools version +set -e + # shellcheck source=../unix/DownloadURL.sh source "${BASH_SOURCE%/*}/../unix/DownloadURL.sh" # shellcheck source=../unix/check_and_set_proxy.sh @@ -80,8 +82,11 @@ else sudo chown -R qt:users "$targetFolder" fi -# Run the following command under `eval` or `sh -c` so that the shell properly splits it +# Stop the sdkmanager from printing thousands of lines of #hashmarks. +# Run the following command under `eval` or `sh -c` so that the shell properly splits it. sdkmanager_no_progress_bar_cmd="tr '\r' '\n' | grep -v '^\[[ =]*\]'" +# But don't let the pipeline hide sdkmanager failures. +set -o pipefail echo "Running SDK manager for platforms;$sdkApiLevel, platform-tools and build-tools;$sdkBuildToolsVersion." # shellcheck disable=SC2031 diff --git a/coin/provisioning/common/windows/android.ps1 b/coin/provisioning/common/windows/android.ps1 index 0ac07f8a..2cf18cdc 100644 --- a/coin/provisioning/common/windows/android.ps1 +++ b/coin/provisioning/common/windows/android.ps1 @@ -95,6 +95,9 @@ Out-File -FilePath C:\Utils\Android\licenses\android-sdk-license -Encoding utf8 # Get a PATH where Java's path is defined from previous provisioning [Environment]::SetEnvironmentVariable("PATH", [Environment]::GetEnvironmentVariable("PATH", "Machine"), "Process") +# Attempt to catch all errors of sdkmanager.bat, even when hidden behind a pipeline. +$ErrorActionPreference = "Stop" + cd $toolsFolder\bin\ $sdkmanager_args += "platforms;$sdkApiLevel", "platform-tools", "build-tools;$sdkBuildToolsVersion" $command = 'for($i=0;$i -lt 6;$i++) { $response += "y`n"}; $response | .\sdkmanager.bat @sdkmanager_args | Out-Null' diff --git a/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh b/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh index aa0935f0..bc123111 100755 --- a/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh +++ b/coin/provisioning/qtci-macos-10.13-x86_64/30-android.sh @@ -37,7 +37,7 @@ # It also runs update for SDK API, latest SDK tools, latest platform-tools and build-tools version -set -ex +set -e # shellcheck source=../common/unix/SetEnvVar.sh source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh" @@ -68,8 +68,11 @@ sudo unzip -q "$toolsSourceFile" -d "$sdkTargetFolder" echo "Changing ownership of Android files." sudo chown -R qt:wheel "$targetFolder" -# Run the following command under `eval` or `sh -c` so that the shell properly splits it +# Stop the sdkmanager from printing thousands of lines of #hashmarks. +# Run the following command under `eval` or `sh -c` so that the shell properly splits it. sdkmanager_no_progress_bar_cmd="tr '\r' '\n' | grep -v '^\[[ =]*\]'" +# But don't let the pipeline hide sdkmanager failures. +set -o pipefail echo "Running SDK manager for platforms;$sdkApiLevel, platform-tools and build-tools;$sdkBuildToolsVersion." (echo "y"; echo "y") | "$sdkTargetFolder/tools/bin/sdkmanager" \ From fdb88edba04d175b1eadd8d6142e5e8132d414f8 Mon Sep 17 00:00:00 2001 From: Heikki Halmet Date: Thu, 30 Jan 2020 13:24:53 +0200 Subject: [PATCH 18/18] Provisioning: Make sure that Perl's Data:Dumper is installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Data:dumper is critical for release script build.pl. Currently data:dumper is installed by devtoolset as a dependency for autoconf. But, it seems, that in earlier version of devtoolset the autoconf wasn't installed by devtoolset which caused the failure with build.pl. Change-Id: I487a17ca0eb37c5984eb2f195e1024ed019b80d2 Reviewed-by: Tony Sarajärvi --- .../qtci-linux-RHEL-7.6-x86_64/04-install-packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coin/provisioning/qtci-linux-RHEL-7.6-x86_64/04-install-packages.sh b/coin/provisioning/qtci-linux-RHEL-7.6-x86_64/04-install-packages.sh index d3dcc420..b3ecceea 100755 --- a/coin/provisioning/qtci-linux-RHEL-7.6-x86_64/04-install-packages.sh +++ b/coin/provisioning/qtci-linux-RHEL-7.6-x86_64/04-install-packages.sh @@ -120,6 +120,8 @@ installPackages+=(xz-devel) installPackages+=(zlib-devel) installPackages+=(libffi-devel) installPackages+=(libsqlite3x-devel) +# Build.pl +installPackages+=(perl-Data-Dumper) sudo yum -y install "${installPackages[@]}"