Provisioning: Remove the special wheel cache for python3 packages

It should be replaced with a global pip cache in the long run.
Removing it should make the maintenance of the required packages easier
as it can be done within the modules instead of in the global
provisioing scripts.

Change-Id: Ief9832a8a640ce98325cda015d0a9e89d6cb16af
Reviewed-by: Simo Fält <simo.falt@qt.io>
(cherry picked from commit b87b44168c)
Reviewed-by: Daniel Smith <Daniel.Smith@qt.io>

coin/provisioning/common/macos/python3.sh

@@ -63,10 +63,6 @@ InstallPip python3.9
 SetEnvVar "PYTHON3_PATH" "/Library/Frameworks/Python.framework/Versions/3.9/bin"
 SetEnvVar "PIP3_PATH" "/Library/Frameworks/Python.framework/Versions/3.9/bin"
 
-# Install all needed packages in a special wheel cache directory
-/Library/Frameworks/Python.framework/Versions/3.9/bin/pip3 wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../shared/requirements.txt
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
-
 # Install Python certificates. Required at least for emsdk installation
 open /Applications/Python\ 3.9/Install\ Certificates.command
 

coin/provisioning/common/shared/requirements.txt

@@ -1,19 +0,0 @@
-# This file holds all python3 packages need for a certain qt module
-# When adding new packages here, first add a comment with the name of the module
-# Then all the packages. Duplicates are wanted as every module should install
-# all packages it needs and shouldn't depend on a package from another module
-#
-# qtinterfaceframework
-antlr4-python3-runtime==4.9.2
-argh==0.26.2
-click==8.0.1
-coloredlogs==15.0.1
-humanfriendly==10.0
-Jinja2==3.0.1
-MarkupSafe==2.0.1
-path.py==12.5.0
-pathtools==0.1.2
-PyYAML==5.4.1
-six==1.16.0
-typing==3.7.4
-watchdog==2.1.5

coin/provisioning/common/windows/python3.ps1

@@ -98,17 +98,6 @@ Run-Executable "$install_path\python.exe" "-m pip install --upgrade pip"
 
 Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install virtualenv wheel html5lib"
 
-# Install all needed packages in a special wheel cache directory
-$python3_wheel_dir="$install_path\python3-wheels"
-Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args wheel --wheel-dir $python3_wheel_dir -r $PSScriptRoot\..\shared\requirements.txt"
-
-Set-EnvironmentVariable "PYTHON3_WHEEL_CACHE-$version-$archVer" "$python3_wheel_dir"
-# PYTHON3_WHEEL_CACHE is already in use so we should keep it pointing to 64 bit default
-# wheel cache
-if (($setDefault) -And ($archVer -eq 64)) {
-    Set-EnvironmentVariable "PYTHON3_WHEEL_CACHE" "$python3_wheel_dir"
-}
-
 # Install PyPDF2 for QSR documentation
 Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install PyPDF2"
 

coin/provisioning/qtci-linux-Debian-11.6-aarch64/02-apt.sh

@@ -285,11 +285,6 @@ waitLoop
 sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install "${installPackages[@]}"
 sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install cmake apt-cacher-ng -t bullseye-backports
 
-# Install all needed packages in a special wheel cache directory
-#pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-#source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-#SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
 # SetEnvVar "PATH" "/usr/lib/nodejs-mozilla/bin:\$PATH"
 
 #OpenSSLVersion="$(openssl version |cut -b 9-14)"

coin/provisioning/qtci-linux-RHEL-8.2-x86_64/04-install-packages.sh

@@ -167,12 +167,6 @@ sudo pip3 install --upgrade pip
 sudo pip3 install virtualenv wheel
 
 sudo /usr/bin/pip3 install wheel
-# Install all needed packages in a special wheel cache directory
-/usr/bin/pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# shellcheck source=../common/unix/SetEnvVar.sh
-source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
 
 OpenSSLVersion="$(openssl version |cut -b 9-14)"
 echo "OpenSSL = $OpenSSLVersion" >> ~/versions.txt

coin/provisioning/qtci-linux-RHEL-8.4-x86_64/04-install-packages.sh

@@ -179,15 +179,6 @@ sudo pip3 install virtualenv wheel
 sudo python3.8 -m pip install virtualenv wheel
 
 sudo /usr/bin/pip3 install wheel
-# Install all needed packages in a special wheel cache directory
-/usr/bin/pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# Install 3.8 compatible wheels
-python3.8 -m pip wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# shellcheck source=../common/unix/SetEnvVar.sh
-source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
 
 OpenSSLVersion="$(openssl3 version |cut -b 9-14)"
 echo "OpenSSL = $OpenSSLVersion" >> ~/versions.txt

coin/provisioning/qtci-linux-RHEL-8.6-x86_64/04-install-packages.sh

@@ -155,15 +155,6 @@ sudo pip3 install virtualenv wheel
 sudo python3.8 -m pip install virtualenv wheel
 
 sudo /usr/bin/pip3 install wheel
-# Install all needed packages in a special wheel cache directory
-/usr/bin/pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# Install 3.8 compatible wheels
-python3.8 -m pip wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# shellcheck source=../common/unix/SetEnvVar.sh
-source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
 
 OpenSSLVersion="$(openssl3 version |cut -b 9-14)"
 echo "OpenSSL = $OpenSSLVersion" >> ~/versions.txt

coin/provisioning/qtci-linux-RHEL-8.8-x86_64/04-install-packages.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2023 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+set -ex
+
+# Remove update notifications and packagekit running in the background
+sudo yum -y remove PackageKit gnome-software
+
+sudo yum -y update
+
+installPackages=()
+installPackages+=(git)
+installPackages+=(zlib-devel)
+installPackages+=(glib2-devel)
+installPackages+=(openssl3)
+installPackages+=(openssl3-devel)
+installPackages+=(freetype-devel)
+installPackages+=(fontconfig-devel)
+installPackages+=(curl-devel)
+installPackages+=(expat-devel)
+installPackages+=(gettext-devel)
+installPackages+=(perl-devel)
+installPackages+=(dh-autoreconf)
+# cmake build
+installPackages+=(ninja-build)
+installPackages+=(pcre2-devel)
+installPackages+=(double-conversion-devel)
+installPackages+=(zstd)
+# update kernel
+installPackages+=(kernel)
+installPackages+=(kernel-tools)
+installPackages+=(kernel-devel)
+installPackages+=(kernel-core)
+installPackages+=(kernel-modules)
+installPackages+=(kernel-headers)
+# EGL support
+# mesa-libraries need to use older version than 22.1.5-2 which cause Xorg to crash
+installPackages+=(mesa-libEGL-devel-21.3.4-1.el8)
+installPackages+=(mesa-libGL-devel-21.3.4-1.el8)
+installPackages+=(mesa-dri-drivers-21.3.4-1.el8.x86_64)
+installPackages+=(mesa-libgbm-21.3.4-1.el8.x86_64)
+installPackages+=(mesa-vulkan-drivers-21.3.4-1.el8.x86_64)
+installPackages+=(libxkbfile-devel)
+# Xinput2
+installPackages+=(libXi-devel)
+installPackages+=(mysql-server)
+installPackages+=(mysql)
+installPackages+=(mysql-devel)
+installPackages+=(postgresql-devel)
+installPackages+=(cups-devel)
+installPackages+=(dbus-devel)
+# gstreamer 1 for QtMultimedia
+# Note! gstreamer1-plugins-bad-free needs to be upgraded or it will conflicts with gstreamer1-plugins-base-devel
+installPackages+=(gstreamer1-plugins-bad-free)
+installPackages+=(gstreamer1-devel)
+installPackages+=(gstreamer1-plugins-base-devel)
+# yasm for QtMultimedia
+installPackages+=(yasm)
+# gtk3 style for QtGui/QStyle
+installPackages+=(gtk3-devel)
+# libusb1 for tqtc-boot2qt/qdb
+installPackages+=(libusbx-devel)
+# speech-dispatcher-devel for QtSpeech, otherwise it has no backend on Linux
+installPackages+=(speech-dispatcher-devel)
+# Python 3.8 for pyside. Qt for Python support for Python 3.6 will be deprecated in within pyside6.3
+installPackages+=(python38)
+installPackages+=(python38-devel)
+# WebEngine
+installPackages+=(bison)
+installPackages+=(flex)
+installPackages+=(gperftools-libs)
+installPackages+=(gperf)
+installPackages+=(alsa-lib-devel)
+installPackages+=(pulseaudio-libs-devel)
+installPackages+=(libXtst-devel)
+installPackages+=(libxshmfence-devel)
+installPackages+=(nspr-devel)
+installPackages+=(nss-devel)
+installPackages+=(python3-html5lib)
+# For Android builds
+installPackages+=(java-11-openjdk-devel)
+# For receiving shasum
+installPackages+=(perl-Digest-SHA)
+# INTEGRITY requirements
+installPackages+=(glibc.i686)
+# Enable Qt Bluetooth
+installPackages+=(bluez-libs-devel)
+# QtNfc
+installPackages+=(pcsc-lite-devel)
+# QtWebKit
+installPackages+=(libxml2-devel)
+installPackages+=(libxslt-devel)
+# For building Wayland from source
+installPackages+=(libffi-devel)
+# QtWayland
+#installPackages+=(mesa-libwayland-egl)
+#installPackages+=(mesa-libwayland-egl-devel)
+installPackages+=(libwayland-client)
+installPackages+=(libwayland-cursor)
+installPackages+=(libwayland-server)
+# Jenkins
+installPackages+=(chrpath)
+# libxkbcommon
+installPackages+=(libxkbcommon-devel)
+installPackages+=(libxkbcommon-x11-devel)
+# xcb-util-* libraries
+installPackages+=(xcb-util)
+installPackages+=(xcb-util-image-devel)
+installPackages+=(xcb-util-keysyms-devel)
+installPackages+=(xcb-util-wm-devel)
+installPackages+=(xcb-util-renderutil-devel)
+installPackages+=(xcb-util-cursor)
+installPackages+=(xcb-util-cursor-devel)
+
+# ODBC support
+installPackages+=(unixODBC-devel)
+installPackages+=(unixODBC)
+# Vulkan support
+installPackages+=(vulkan-devel)
+installPackages+=(vulkan-tools)
+# Conan: For Python build
+installPackages+=(xz-devel)
+installPackages+=(zlib-devel)
+installPackages+=(libffi-devel)
+installPackages+=(libsqlite3x-devel)
+# Build.pl
+installPackages+=(perl-Data-Dumper)
+# In RedHat these come with Devtoolset
+installPackages+=(gcc)
+installPackages+=(gcc-c++)
+installPackages+=(make)
+# Open source VMware Tools
+installPackages+=(open-vm-tools)
+
+sudo yum -y install "${installPackages[@]}"
+
+sudo dnf -y module install nodejs:16
+
+# We shouldn't use yum to install virtualenv. The one found from package repo is not
+# working, but we can use installed pip
+sudo pip3 install --upgrade pip
+sudo pip3 install virtualenv wheel
+# Just make sure we have virtualenv to run with python3.8 -m virtualenv
+sudo python3.8 -m pip install virtualenv wheel
+
+sudo /usr/bin/pip3 install wheel
+
+OpenSSLVersion="$(openssl3 version |cut -b 9-14)"
+echo "OpenSSL = $OpenSSLVersion" >> ~/versions.txt

coin/provisioning/qtci-linux-RHEL-9.0-x86_64/04-install-packages.sh

@@ -185,18 +185,9 @@ sudo pip3 install virtualenv wheel
 sudo python -m pip install virtualenv wheel
 
 sudo /usr/bin/pip3 install wheel
-# Install all needed packages in a special wheel cache directory
-/usr/bin/pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# Install compatible wheels
-python -m pip wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
 
 # Make FindPython3.cmake to find python3
 sudo ln -s /usr/bin/python3 /usr/local/bin/python3
 
-# shellcheck source=../common/unix/SetEnvVar.sh
-source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
-
 OpenSSLVersion="$(openssl version |cut -b 9-14)"
 echo "OpenSSL = $OpenSSLVersion" >> ~/versions.txt

coin/provisioning/qtci-linux-SLES-15_SP4-x86_64/08-pythondev.sh

@@ -108,10 +108,6 @@ python3 --version | fgrep "$python3Version"
 pip3 install --user wheel
 pip3 install --user virtualenv
 
-# Install all needed packages in a special wheel cache directory
-pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
 # shellcheck source=../common/unix/SetEnvVar.sh
 source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
 SetEnvVar "PYTHON3_PATH" "/usr/local/bin"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"

coin/provisioning/qtci-linux-Ubuntu-18.04-x86_64/02-apt.sh

@@ -201,12 +201,6 @@ echo "Installing packages"
 waitLoop
 sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install "${installPackages[@]}"
 
-# Install all needed packages in a special wheel cache directory
-pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
-
 OpenSSLVersion="$(openssl version |cut -b 9-14)"
 echo "OpenSSL = $OpenSSLVersion" >> ~/versions.txt
 

coin/provisioning/qtci-linux-Ubuntu-20.04-x86_64/02-apt.sh

@@ -251,11 +251,7 @@ echo "Installing packages"
 waitLoop
 sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install "${installPackages[@]}"
 
-# Install all needed packages in a special wheel cache directory
-pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
 source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
 SetEnvVar "PATH" "/usr/lib/nodejs-mozilla/bin:\$PATH"
 
 OpenSSLVersion="$(openssl version |cut -b 9-14)"

coin/provisioning/qtci-linux-Ubuntu-22.04-x86_64/02-apt.sh

@@ -256,11 +256,7 @@ echo "Installing packages"
 waitLoop
 sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Lock::Timeout=300 install "${installPackages[@]}"
 
-# Install all needed packages in a special wheel cache directory
-pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
 source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
 # SetEnvVar "PATH" "/usr/lib/nodejs-mozilla/bin:\$PATH"
 
 OpenSSLVersion="$(openssl version |cut -b 9-14)"

coin/provisioning/qtci-linux-openSUSE-15.3-x86_64/08-pythondev.sh

@@ -10,9 +10,3 @@ sudo zypper -nq install python-devel python-xml
 # install python3
 sudo zypper -nq install python3-base python3 python3-pip python3-devel python3-virtualenv python3-wheel
 
-# Install all needed packages in a special wheel cache directory
-pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# shellcheck source=../common/unix/SetEnvVar.sh
-source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"

coin/provisioning/qtci-linux-openSUSE-15.4-x86_64/08-pythondev.sh

@@ -43,9 +43,3 @@ sudo zypper -nq install python-devel python-xml
 # install python3
 sudo zypper -nq install python3-base python3 python3-pip python3-devel python3-virtualenv python3-wheel
 
-# Install all needed packages in a special wheel cache directory
-pip3 wheel --wheel-dir "$HOME/python3-wheels" -r "${BASH_SOURCE%/*}/../common/shared/requirements.txt"
-
-# shellcheck source=../common/unix/SetEnvVar.sh
-source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"

coin/provisioning/qtci-macos-11-arm/25-python3.sh

@@ -56,8 +56,4 @@ SetEnvVar "PIP3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
 # Use 3.9 as a default python
 SetEnvVar "PATH" "\$PYTHON3_PATH:\$PATH"
 
-# Install all needed packages in a special wheel cache directory
-/Users/qt/.pyenv/versions/3.9.7/bin/pip3 wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
-
 echo "python3 = 3.9.7" >> ~/versions.txt

coin/provisioning/qtci-macos-12-arm/25-python.sh

@@ -56,10 +56,6 @@ SetEnvVar "PIP3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
 # Use 3.9 as a default python
 SetEnvVar "PATH" "\$PYTHON3_PATH:\$PATH"
 
-# Install all needed packages in a special wheel cache directory
-/Users/qt/.pyenv/versions/3.9.7/bin/pip3 wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
-
 # QtWebengine still requires python2
 pyenv install 2.7.18
 SetEnvVar "PYTHON2_PATH" "/Users/qt/.pyenv/versions/2.7.18/bin/"

coin/provisioning/qtci-macos-13-arm/25-python.sh

@@ -56,10 +56,6 @@ SetEnvVar "PIP3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
 # Use 3.9 as a default python
 SetEnvVar "PATH" "\$PYTHON3_PATH:\$PATH"
 
-# Install all needed packages in a special wheel cache directory
-/Users/qt/.pyenv/versions/3.9.7/bin/pip3 wheel --wheel-dir $HOME/python3-wheels -r ${BASH_SOURCE%/*}/../common/shared/requirements.txt
-SetEnvVar "PYTHON3_WHEEL_CACHE" "$HOME/python3-wheels"
-
 # QtWebengine still requires python2
 pyenv install 2.7.18
 SetEnvVar "PYTHON2_PATH" "/Users/qt/.pyenv/versions/2.7.18/bin/"