From 1f2fb6312c1a8c92ab6700faa785e046aca2ecf9 Mon Sep 17 00:00:00 2001
From: Alexandru Croitor <alexandru.croitor@qt.io>
Date: Thu, 16 May 2024 12:30:43 +0200
Subject: [PATCH] coin: Add python packages to allow validating Qt generated
 SBOM files

Add a sbom_requirements.txt file and install it using the
various platform specific pip installations.

Partially restores b87b44168cc5ab3b11b03c1015022d69763745a9 but
without the wheel cache.

Pick-to: 6.8
Task-number: QTBUG-122899
Change-Id: I8002898866acbc226ffebb59dcf5b3a58237ea34
Reviewed-by: Toni Saario <toni.saario@qt.io>
---
 coin/provisioning/common/macos/python3.sh                   | 1 +
 coin/provisioning/common/shared/sbom_requirements.txt       | 5 +++++
 coin/provisioning/common/windows/python3.ps1                | 6 ++++++
 coin/provisioning/qtci-linux-Debian-11.6-aarch64/02-apt.sh  | 2 ++
 .../qtci-linux-RHEL-8.8-x86_64/04-install-packages.sh       | 2 ++
 .../qtci-linux-RHEL-9.2-x86_64/04-install-packages.sh       | 2 ++
 .../qtci-linux-SLES-15_SP5-x86_64/08-pythondev.sh           | 1 +
 coin/provisioning/qtci-linux-Ubuntu-22.04-x86_64/02-apt.sh  | 1 +
 coin/provisioning/qtci-linux-Ubuntu-24.04-aarch64/02-apt.sh | 3 +++
 coin/provisioning/qtci-linux-Ubuntu-24.04-x86_64/02-apt.sh  | 3 +++
 .../qtci-linux-openSUSE-15.5-x86_64/08-pythondev.sh         | 1 +
 coin/provisioning/qtci-macos-12-arm/25-python.sh            | 1 +
 coin/provisioning/qtci-macos-13-arm/25-python.sh            | 1 +
 coin/provisioning/qtci-macos-14-arm/25-python.sh            | 1 +
 14 files changed, 30 insertions(+)
 create mode 100644 coin/provisioning/common/shared/sbom_requirements.txt

diff --git a/coin/provisioning/common/macos/python3.sh b/coin/provisioning/common/macos/python3.sh
index 1a8809d6..ff74bd4a 100755
--- a/coin/provisioning/common/macos/python3.sh
+++ b/coin/provisioning/common/macos/python3.sh
@@ -24,6 +24,7 @@ echo "Configure pip"
 /Library/Frameworks/Python.framework/Versions/3.11/bin/pip3 config --user set global.extra-index-url https://pypi.org/simple/
 
 /Library/Frameworks/Python.framework/Versions/3.11/bin/pip3 install virtualenv wheel html5lib
+/Library/Frameworks/Python.framework/Versions/3.11/bin/pip3 install -r ${BASH_SOURCE%/*}/../shared/sbom_requirements.txt
 
 SetEnvVar "PYTHON3_PATH" "/Library/Frameworks/Python.framework/Versions/3.11/bin"
 SetEnvVar "PIP3_PATH" "/Library/Frameworks/Python.framework/Versions/3.11/bin"
diff --git a/coin/provisioning/common/shared/sbom_requirements.txt b/coin/provisioning/common/shared/sbom_requirements.txt
new file mode 100644
index 00000000..0c72d654
--- /dev/null
+++ b/coin/provisioning/common/shared/sbom_requirements.txt
@@ -0,0 +1,5 @@
+# Python packages used to validate and audit Qt SBOM informaiton
+spdx-tools>=0.8.0
+ntia-conformance-checker
+sbomaudit
+sbom2doc
diff --git a/coin/provisioning/common/windows/python3.ps1 b/coin/provisioning/common/windows/python3.ps1
index 46f75d7c..13637c6c 100644
--- a/coin/provisioning/common/windows/python3.ps1
+++ b/coin/provisioning/common/windows/python3.ps1
@@ -78,6 +78,12 @@ Run-Executable "$install_path\python.exe" "-m pip config --user set global.index
 Run-Executable "$install_path\python.exe" "-m pip config --user set global.extra-index-url https://pypi.org/simple/"
 Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install virtualenv wheel html5lib"
 
+# Check if python version is higher than 3.8.
+# ntia-conformance-checker requires at least 3.8
+if ([version]::Parse($version) -gt [version]::Parse("3.8")) {
+    Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install -r $PSScriptRoot\..\shared\sbom_requirements.txt"
+}
+
 # Install PyPDF2 for QSR documentation
 Run-Executable "$install_path\Scripts\pip3.exe" "$pip_args install PyPDF2"
 
diff --git a/coin/provisioning/qtci-linux-Debian-11.6-aarch64/02-apt.sh b/coin/provisioning/qtci-linux-Debian-11.6-aarch64/02-apt.sh
index 2ce4aef9..2f175f47 100755
--- a/coin/provisioning/qtci-linux-Debian-11.6-aarch64/02-apt.sh
+++ b/coin/provisioning/qtci-linux-Debian-11.6-aarch64/02-apt.sh
@@ -256,6 +256,8 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install cmake apt-cacher-ng -t
 # Disable keyring password prompt
 keyring --disable
 
+pip install --user -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt"
+
 # SetEnvVar "PATH" "/usr/lib/nodejs-mozilla/bin:\$PATH"
 
 #OpenSSLVersion="$(openssl version |cut -b 9-14)"
diff --git a/coin/provisioning/qtci-linux-RHEL-8.8-x86_64/04-install-packages.sh b/coin/provisioning/qtci-linux-RHEL-8.8-x86_64/04-install-packages.sh
index d1d600da..e6912724 100755
--- a/coin/provisioning/qtci-linux-RHEL-8.8-x86_64/04-install-packages.sh
+++ b/coin/provisioning/qtci-linux-RHEL-8.8-x86_64/04-install-packages.sh
@@ -164,9 +164,11 @@ sudo pip config --user set global.extra-index-url https://pypi.org/simple/
 sudo pip3 install virtualenv wheel
 # Just make sure we have virtualenv to run with python3.8 -m virtualenv
 sudo python3.11 -m pip install virtualenv wheel
+sudo python3.11 -m pip install -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt"
 
 sudo /usr/bin/pip3 install wheel
 sudo /usr/bin/pip3 install dataclasses
+# No sbom_requirements.txt, because it requires Python 3.9 for poetry_core -> spdx_tools and we have 3.8
 
 OpenSSLVersion="$(openssl3 version |cut -b 9-14)"
 echo "System's OpenSSL = $OpenSSLVersion" >> ~/versions.txt
diff --git a/coin/provisioning/qtci-linux-RHEL-9.2-x86_64/04-install-packages.sh b/coin/provisioning/qtci-linux-RHEL-9.2-x86_64/04-install-packages.sh
index 279d66a5..fd454ed2 100755
--- a/coin/provisioning/qtci-linux-RHEL-9.2-x86_64/04-install-packages.sh
+++ b/coin/provisioning/qtci-linux-RHEL-9.2-x86_64/04-install-packages.sh
@@ -176,8 +176,10 @@ sudo pip config --user set global.extra-index-url https://pypi.org/simple/
 sudo pip3 install virtualenv wheel
 # Just make sure we have virtualenv to run with python3.8 -m virtualenv
 sudo python -m pip install virtualenv wheel
+sudo python -m pip install -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt"
 
 sudo /usr/bin/pip3 install wheel
+sudo /usr/bin/pip3 install -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt"
 
 # Make FindPython3.cmake to find python3
 sudo ln -s /usr/bin/python3 /usr/local/bin/python3
diff --git a/coin/provisioning/qtci-linux-SLES-15_SP5-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-SLES-15_SP5-x86_64/08-pythondev.sh
index 476c5dc3..79d0f1b2 100644
--- a/coin/provisioning/qtci-linux-SLES-15_SP5-x86_64/08-pythondev.sh
+++ b/coin/provisioning/qtci-linux-SLES-15_SP5-x86_64/08-pythondev.sh
@@ -71,6 +71,7 @@ python3 --version | grep -F "$python3Version"
 
 pip3 install --user wheel
 pip3 install --user virtualenv
+pip3 install --user -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt"
 
 # shellcheck source=../common/unix/SetEnvVar.sh
 source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
diff --git a/coin/provisioning/qtci-linux-Ubuntu-22.04-x86_64/02-apt.sh b/coin/provisioning/qtci-linux-Ubuntu-22.04-x86_64/02-apt.sh
index 22bb3e50..11b34572 100755
--- a/coin/provisioning/qtci-linux-Ubuntu-22.04-x86_64/02-apt.sh
+++ b/coin/provisioning/qtci-linux-Ubuntu-22.04-x86_64/02-apt.sh
@@ -245,6 +245,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Lock::Timeout=300 ins
 # Configure pip
 pip config --user set global.index https://ci-files01-hki.ci.qt.io/input/python_module_cache
 pip config --user set global.extra-index-url https://pypi.org/simple/
+pip install --user -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt"
 
 source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
 # SetEnvVar "PATH" "/usr/lib/nodejs-mozilla/bin:\$PATH"
diff --git a/coin/provisioning/qtci-linux-Ubuntu-24.04-aarch64/02-apt.sh b/coin/provisioning/qtci-linux-Ubuntu-24.04-aarch64/02-apt.sh
index dfe7aa9b..c9106f88 100644
--- a/coin/provisioning/qtci-linux-Ubuntu-24.04-aarch64/02-apt.sh
+++ b/coin/provisioning/qtci-linux-Ubuntu-24.04-aarch64/02-apt.sh
@@ -239,6 +239,9 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Lock::Timeout=300 ins
 # Configure pip
 pip config --user set global.index https://ci-files01-hki.ci.qt.io/input/python_module_cache
 pip config --user set global.extra-index-url https://pypi.org/simple/
+# Ubuntu 24.04 comes with a newer pip that disallows installing into the system site-packages,
+# so we explicitly ask it to allow it.
+pip install --user -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt" --break-system-packages
 
 source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
 # SetEnvVar "PATH" "/usr/lib/nodejs-mozilla/bin:\$PATH"
diff --git a/coin/provisioning/qtci-linux-Ubuntu-24.04-x86_64/02-apt.sh b/coin/provisioning/qtci-linux-Ubuntu-24.04-x86_64/02-apt.sh
index a8c5894b..f128e095 100644
--- a/coin/provisioning/qtci-linux-Ubuntu-24.04-x86_64/02-apt.sh
+++ b/coin/provisioning/qtci-linux-Ubuntu-24.04-x86_64/02-apt.sh
@@ -250,6 +250,9 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Lock::Timeout=300 ins
 # Configure pip
 pip config --user set global.index https://ci-files01-hki.ci.qt.io/input/python_module_cache
 pip config --user set global.extra-index-url https://pypi.org/simple/
+# Ubuntu 24.04 comes with a newer pip that disallows installing into the system site-packages,
+# so we explicitly ask it to allow it.
+pip install --user -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt" --break-system-packages
 
 source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
 # SetEnvVar "PATH" "/usr/lib/nodejs-mozilla/bin:\$PATH"
diff --git a/coin/provisioning/qtci-linux-openSUSE-15.5-x86_64/08-pythondev.sh b/coin/provisioning/qtci-linux-openSUSE-15.5-x86_64/08-pythondev.sh
index 82101c9b..7f7d867a 100755
--- a/coin/provisioning/qtci-linux-openSUSE-15.5-x86_64/08-pythondev.sh
+++ b/coin/provisioning/qtci-linux-openSUSE-15.5-x86_64/08-pythondev.sh
@@ -14,5 +14,6 @@ sudo zypper -nq install python-devel python-xml
 # install python3
 sudo zypper -nq install python311-base python311-devel python311-pip python311-virtualenv python311-wheel
 python3.11 -m pip install selenium netifaces scache webdriver-manager
+python3.11 -m pip install -r "${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt"
 
 SetEnvVar "PYTHON3_EXECUTABLE" "/usr/bin/python3.11"
diff --git a/coin/provisioning/qtci-macos-12-arm/25-python.sh b/coin/provisioning/qtci-macos-12-arm/25-python.sh
index 9bfb6384..0e05f593 100755
--- a/coin/provisioning/qtci-macos-12-arm/25-python.sh
+++ b/coin/provisioning/qtci-macos-12-arm/25-python.sh
@@ -11,6 +11,7 @@ source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
 pyenv install 3.9.7
 
 /Users/qt/.pyenv/versions/3.9.7/bin/pip3 install --user install virtualenv wheel html5lib
+/Users/qt/.pyenv/versions/3.9.7/bin/pip3 install --user -r ${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt
 
 SetEnvVar "PYTHON3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
 SetEnvVar "PIP3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
diff --git a/coin/provisioning/qtci-macos-13-arm/25-python.sh b/coin/provisioning/qtci-macos-13-arm/25-python.sh
index 9bfb6384..0e05f593 100755
--- a/coin/provisioning/qtci-macos-13-arm/25-python.sh
+++ b/coin/provisioning/qtci-macos-13-arm/25-python.sh
@@ -11,6 +11,7 @@ source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
 pyenv install 3.9.7
 
 /Users/qt/.pyenv/versions/3.9.7/bin/pip3 install --user install virtualenv wheel html5lib
+/Users/qt/.pyenv/versions/3.9.7/bin/pip3 install --user -r ${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt
 
 SetEnvVar "PYTHON3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
 SetEnvVar "PIP3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
diff --git a/coin/provisioning/qtci-macos-14-arm/25-python.sh b/coin/provisioning/qtci-macos-14-arm/25-python.sh
index 5a0df739..761fcf01 100755
--- a/coin/provisioning/qtci-macos-14-arm/25-python.sh
+++ b/coin/provisioning/qtci-macos-14-arm/25-python.sh
@@ -11,6 +11,7 @@ source "${BASH_SOURCE%/*}/../common/unix/SetEnvVar.sh"
 pyenv install 3.9.7
 
 /Users/qt/.pyenv/versions/3.9.7/bin/pip3 install --user install virtualenv wheel html5lib
+/Users/qt/.pyenv/versions/3.9.7/bin/pip3 install --user -r ${BASH_SOURCE%/*}/../common/shared/sbom_requirements.txt
 
 SetEnvVar "PYTHON3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"
 SetEnvVar "PIP3_PATH" "/Users/qt/.pyenv/versions/3.9.7/bin/"