Android: Begin initial alignment of Linux and macOS provisioning

The Android provisioning scripts on macOS and Linux perform many of the
same tasks, but with slightly differing approaches. Future patches will
make us run Android emulators on ARM64, which will further make these
scripts similar.

This patch performs some modifications to the macOS provisioning script
to make it more similar to the Linux one. This will help us combine
parts of these scripts in the future.

Task-number: QTQAINFRA-7487
Pick-to: 6.11 6.10 6.8
Change-Id: Idba30d07607780d1ef8c60cc5e038a034c821632
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>

coin/provisioning/common/linux/android_linux.sh

@@ -6,7 +6,7 @@
 
 # It also runs update for SDK API, latest SDK tools, latest platform-tools and build-tools version
 
-set -e
+set -ex
 
 # shellcheck source=../unix/DownloadURL.sh
 source "${BASH_SOURCE%/*}/../unix/DownloadURL.sh"
@@ -145,7 +145,9 @@ cat >>~/versions.txt <<EOB
 Android SDK tools = $toolsVersion
 Android SDK Build Tools = $sdkBuildToolsVersion
 Android SDK API level = $sdkApiLevel
-Android NDK = $ndkVersion
+Android NDK latest = $ndkVersionLatest
+Android NDK nightly1 = $ndkVersionNightly1
+Android NDK nightly2 = $ndkVersionNightly2
 EOB
 
 cd "$sdkTargetFolder/cmdline-tools/tools/bin"

coin/provisioning/common/macos/android.sh

@@ -8,79 +8,114 @@
 
 set -ex
 
+# shellcheck source=../unix/DownloadURL.sh
+source "${BASH_SOURCE%/*}/../unix/DownloadURL.sh"
+# shellcheck source=../unix/check_and_set_proxy.sh
+source "${BASH_SOURCE%/*}/../unix/check_and_set_proxy.sh"
 # shellcheck source=../unix/SetEnvVar.sh
 source "${BASH_SOURCE%/*}/../unix/SetEnvVar.sh"
 
 targetFolder="/opt/android"
 sdkTargetFolder="$targetFolder/sdk"
 
-basePath="/net/ci-files01-hki.ci.qt.io/hdd/www/input/android"
+sudo mkdir -p "$sdkTargetFolder"
+
+basePath="http://ci-files01-hki.ci.qt.io/input/android"
 
 toolsVersion="2.1"
-# toolsFile dertermines tools version
 toolsFile="commandlinetools-mac-6609375_latest.zip"
+sdkBuildToolsVersion="35.0.1"
+sdkApiLevel="android-35"
+toolsSha1="c0c6bb9f1970bfadeaac623dfabf22c3e56baaec"
+
+ndkVersionLatest="r27c"
+ndkSha1Latest="0217c10ffbec496bb9fbfbb3c6fc2477c6b77297"
+
+# Preview NDK that is in alpha/beta/RC state
+ndkVersionPreview="r29-beta2"
+ndkSha1Preview="09be4f8fb626a9c93415198ea8e75d8d82f528fa"
 
 # Non-latest (but still supported by the qt/qt5 branch) NDKs are installed for nightly targets in:
 # coin/platform_configs/nightly_android.yaml
 
-ndkVersionLatest="r27c"
+ndkVersionNightly1=$ndkVersionLatest  # Set as same version as latest = skip NDK install in provisioning
-ndkVersionPreview="r29-beta2"
+ndkSha1Nightly1=$ndkSha1Latest
-ndkVersionNightly1="$ndkVersionLatest"  # If same version as latest = skip NDK install for nightly
-ndkVersionNightly2="$ndkVersionLatest"
 
-sdkBuildToolsVersion="35.0.1"
+ndkVersionNightly2=$ndkVersionLatest
-# this is compile sdk version
+ndkSha1Nightly2=$ndkSha1Latest
-sdkApiLevel="android-35"
 
+toolsTargetFile="/tmp/$toolsFile"
 toolsSourceFile="$basePath/$toolsFile"
 
+echo "Download and unzip Android SDK"
+DownloadURL "$toolsSourceFile" "$toolsSourceFile" "$toolsSha1" "$toolsTargetFile"
+echo "Unzipping Android Tools to '$sdkTargetFolder'"
+sudo unzip -q "$toolsTargetFile" -d "$sdkTargetFolder"
+rm "$toolsTargetFile"
+
 function InstallNdk() {
 
     ndkVersion=$1
+    ndkSha1=$2
 
-    if [[ ! -d "${targetFolder}/android-ndk-${ndkVersion}" ]]; then
+    ndkFile="android-ndk-$ndkVersion-darwin.zip"
-        echo "Unzipping Android NDK $ndkVersion to '${targetFolder}'"
+    ndkTargetFile="/tmp/$ndkFile"
-        ndkSourceFile="$basePath/android-ndk-$ndkVersion-darwin*.zip"
+    ndkSourceFile="$basePath/$ndkFile"
-        sudo unzip -q "$ndkSourceFile" -d "$targetFolder"
-    fi
 
+    DownloadURL "$ndkSourceFile" "$ndkSourceFile" "$ndkSha1" "$ndkTargetFile"
+    echo "Unzipping Android NDK to '$targetFolder'"
+    # Get the package base directory name as string
+    zipBase=$(sudo zipinfo -1 "$ndkTargetFile" 2>/dev/null | awk '!seen {sub("/.*",""); print; seen=1}')
+    sudo unzip -q "$ndkTargetFile" -d "$targetFolder"
+    rm "$ndkTargetFile"
+    androidNdkRoot="${targetFolder}/${zipBase}"
 }
 
-InstallNdk $ndkVersionLatest
+InstallNdk $ndkVersionLatest $ndkSha1Latest
-SetEnvVar "ANDROID_NDK_ROOT_LATEST" "$targetFolder/android-ndk-$ndkVersionLatest"
+SetEnvVar "ANDROID_NDK_ROOT_LATEST" "$androidNdkRoot"
 
 if [ "$ndkVersionPreview" != "$ndkVersionLatest" ]; then
-    InstallNdk $ndkVersionPreview
+    InstallNdk $ndkVersionPreview $ndkSha1Preview
-    SetEnvVar "ANDROID_NDK_ROOT_PREVIEW" "$targetFolder/android-ndk-$ndkVersionPreview"
+    SetEnvVar "ANDROID_NDK_ROOT_PREVIEW" "$androidNdkRoot"
 fi
 
 if [ "$ndkVersionNightly1" != "$ndkVersionLatest" ]; then
-    InstallNdk $ndkVersionNightly1
+    InstallNdk $ndkVersionNightly1 $ndkSha1Nightly1
-    SetEnvVar "ANDROID_NDK_ROOT_NIGHTLY1" "$targetFolder/android-ndk-$ndkVersionNightly1"
+    SetEnvVar "ANDROID_NDK_ROOT_NIGHTLY1" "$androidNdkRoot"
 fi
 
 if [ "$ndkVersionNightly2" != "$ndkVersionLatest" ]; then
-    InstallNdk $ndkVersionNightly2
+    InstallNdk $ndkVersionNightly2 $ndkSha1Nightly2
-    SetEnvVar "ANDROID_NDK_ROOT_NIGHTLY2" "$targetFolder/android-ndk-$ndkVersionNightly2"
+    SetEnvVar "ANDROID_NDK_ROOT_NIGHTLY2" "$androidNdkRoot"
 fi
 
-echo "Unzipping Android Tools to '$sdkTargetFolder'"
-sudo unzip -q "$toolsSourceFile" -d "$sdkTargetFolder"
-
 echo "Changing ownership of Android files."
 sudo chown -R qt:wheel "$targetFolder"
 sudo chmod -R 755 $targetFolder
 
-# Run the following command under `eval` or `sh -c` so that the shell properly splits it
+# Stop the sdkmanager from printing thousands of lines of #hashmarks.
+# Run the following command under `eval` or `sh -c` so that the shell properly splits it.
 sdkmanager_no_progress_bar_cmd="tr '\r' '\n'  |  grep -v '^\[[ =]*\]'"
+# But don't let the pipeline hide sdkmanager failures.
+set -o pipefail
 
 sudo mkdir "$sdkTargetFolder/cmdline-tools"
 sudo mv "$sdkTargetFolder/tools" "$sdkTargetFolder/cmdline-tools"
 
 echo "Running SDK manager for platforms;$sdkApiLevel, platform-tools and build-tools;$sdkBuildToolsVersion."
-(echo "y"; echo "y") | "$sdkTargetFolder/cmdline-tools/tools/bin/sdkmanager" "--sdk_root=$sdkTargetFolder" \
+# shellcheck disable=SC2031
-    "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion"  \
+if [ "$http_proxy" != "" ]; then
-    | eval "$sdkmanager_no_progress_bar_cmd"
+    proxy_host=$(echo "$proxy" | cut -d'/' -f3 | cut -d':' -f1)
+    proxy_port=$(echo "$proxy" | cut -d':' -f3)
+    echo "y" | "$sdkTargetFolder/cmdline-tools/tools/bin/sdkmanager" --sdk_root=$sdkTargetFolder  \
+                   --no_https --proxy=http --proxy_host="$proxy_host" --proxy_port="$proxy_port"  \
+                   "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion"  \
+        | eval "$sdkmanager_no_progress_bar_cmd"
+else
+    echo "y" | "$sdkTargetFolder/cmdline-tools/tools/bin/sdkmanager" --sdk_root=$sdkTargetFolder  \
+                   "platforms;$sdkApiLevel" "platform-tools" "build-tools;$sdkBuildToolsVersion"  \
+        | eval "$sdkmanager_no_progress_bar_cmd"
+fi
 
 echo "Checking the contents of Android SDK..."
 ls -l "$sdkTargetFolder"