CMake: Add RunCMake test to configure and build Qt in various ways

Coin CI only tests 'prefix' builds of Qt, because that's what we use
for packaging.

But many developers configure and build Qt in various ways which are
not covered by tests.

Introduce a new suite of RunCMake tests that configure and build Qt
in all the different permutations that we know people use.

This includes various combinations of:
- prefix vs no-prefix
- out-of-source vs in-source
- per-repo vs top-level
- building tests and examples in-tree vs out-of-tree
- building examples as in-tree vs external projects
- building more than one repo

The aim is to run all or some subset of these tests in a nightly Coin
CI run.

It can also be a useful way to test "risky" unmerged changes in an
automated way, instead of manually doing the various builds.

The current default set of repos that are built are:
qtbase, qtshadertools and qtdeclarative.

The submodules that are built can be controlled by setting various
cmake or env vars when configuring or running the test:

- QT_CI_BUILD_QT_SYNC_MODULE - the main repo that should be checked
  out. Its dependencies will also be checked out, based on its
  dependencies.yaml info.

- QT_CI_BUILD_QT_PIN_GIT_REF - the git sha1 or ref of the main repo
  that should be checked out

- QT_CI_BUILD_QT_SKIP_SUBMODULES - a list of submodules that should be
  skipped, can be useful to skip optional dependencies

- QT_CI_BUILD_QT_EXTRA_CHECKOUT_CHANGES - a list of gerrit commit
  sha1s or refs for each submodule to be checked out to specifically.

- QT_CI_BUILD_QT_EXTRA_CHERRYPICK_CHANGES - a list of gerrit commit
  sha1s or refs to be cherry-picked on top of whatever commits the
  submodules were synced to.

- QT_CI_BUILD_QT_FILTER - a list of filters to include or exclude
  test cases to run.

Sample usage:
mkdir build && cd build
cmake ~/qt5/tests/manual/RunCMake

 # Run regular tests
ctest -V -R RunCMake.ConfigureBuildQt

 # Skip some optional submodules of qtdeclarative
export QT_CI_BUILD_QT_SYNC_MODULE=qtdeclarative
export QT_CI_BUILD_QT_SKIP_SUBMODULES='qtimageformats,qtlanguageserver"
ctest -V -R RunCMake.ConfigureBuildQt

 # Cherry-pick extra changes on top of the synced commits
 # repos are split by '|', commits by ','
export QT_CI_BUILD_QT_EXTRA_CHERRYPICK_CHANGES='qtbase aaabbbccc|qtshadertools a12,b23,b45,refs/changes/57/628457/2'
ctest -V -R RunCMake.ConfigureBuildQt

 # Checkout specific commits for each submodule
export QT_CI_BUILD_QT_EXTRA_CHECKOUT_CHANGES='qtbase refs/changes/54/634554/5|qtshadertools 3bb8a41c44fa69c30f6887685ad70ef7e84c10a6'
ctest -V -R RunCMake.ConfigureBuildQt

 # Only run top-level no-prefix, but not in-source source builds
export QT_CI_BUILD_QT_FILTER='top_level,no_prefix,-per_repo,-in_source'
ctest -V -R RunCMake.ConfigureBuildQt

Change-Id: I355084081dd7b48cdf75c03eb001b64ab7ba96fb
Reviewed-by: Alexey Edelev <alexey.edelev@qt.io>
Reviewed-by: Cristian Le <cristian.le@qt.io>

tests/manual/RunCMake/CMakeLists.txt

@@ -10,3 +10,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/Common.cmake")
 add_RunCMake_test(InitRepository
     -DEXTRA_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
 )
+add_RunCMake_test(ConfigureBuildQt
+    -DEXTRA_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
+)

tests/manual/RunCMake/ConfigureBuildQt/RunCMakeTest.cmake

@@ -0,0 +1,171 @@
+# Copyright (C) 2025 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+cmake_minimum_required(VERSION 3.16)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")
+list(APPEND CMAKE_MODULE_PATH "${EXTRA_MODULE_PATH}")
+include(Common)
+
+# The file is included separately from Common.cmake because it has side-effects
+# that we want to apply only in the RunCMake part of the test.
+include(RunCMake)
+
+# Include the test specific utilities.
+include(Utils)
+
+macro(test_per_repo_prefix_qt)
+    add_test_case(
+        TEST_NAME "per_repo_prefix"
+        TEST_GROUPS per_repo prefix
+        TEST_ARGS
+            BUILD_STANDALONE_TESTS
+            BUILD_STANDALONE_EXAMPLES_IN_TREE
+            BUILD_STANDALONE_EXAMPLES_AS_EXTERNAL_PROJECTS
+            RECONFIGURE_WITHOUT_ARGS_IMMEDIATELY
+    )
+endmacro()
+
+macro(test_per_repo_no_prefix_qt)
+    add_test_case(
+        TEST_NAME "per_repo_no_prefix"
+        TEST_GROUPS per_repo no_prefix
+        TEST_ARGS
+            NO_PREFIX
+            BUILD_STANDALONE_TESTS
+            BUILD_STANDALONE_EXAMPLES_IN_TREE
+            BUILD_STANDALONE_EXAMPLES_AS_EXTERNAL_PROJECTS
+            RECONFIGURE_WITHOUT_ARGS_AFTER_BUILD
+            BUILD_AFTER_RECONFIGURE
+    )
+endmacro()
+
+macro(test_per_repo_prefix_in_tree_tests_and_examples_qt)
+    add_test_case(
+        TEST_NAME "per_repo_prefix_in_tree_tests_and_examples"
+        TEST_GROUPS per_repo prefix in_tree_tests_and_examples
+        TEST_ARGS
+            BUILD_IN_TREE_TESTS
+            BUILD_IN_TREE_EXAMPLES
+    )
+endmacro()
+
+macro(test_per_repo_no_prefix_in_tree_tests_and_examples_qt)
+    add_test_case(
+        TEST_NAME "per_repo_no_prefix_in_tree_tests_and_examples"
+        TEST_GROUPS per_repo no_prefix in_tree_tests_and_examples
+        TEST_ARGS
+            NO_PREFIX
+            BUILD_IN_TREE_TESTS
+            BUILD_IN_TREE_EXAMPLES
+    )
+endmacro()
+
+macro(test_per_repo_no_prefix_in_source)
+    add_test_case(
+        TEST_NAME "per_repo_no_prefix_in_source"
+        TEST_GROUPS per_repo no_prefix in_source
+        TEST_ARGS
+            NO_PREFIX
+            IN_SOURCE
+    )
+endmacro()
+
+macro(test_top_level_prefix_qt)
+    add_test_case(
+        TEST_NAME "top_level_prefix"
+        TEST_GROUPS top_level prefix
+        TEST_ARGS
+            TOP_LEVEL
+            BUILD_STANDALONE_TESTS
+            BUILD_STANDALONE_EXAMPLES_IN_TREE
+            BUILD_STANDALONE_EXAMPLES_AS_EXTERNAL_PROJECTS
+    )
+endmacro()
+
+macro(test_top_level_no_prefix_qt)
+    add_test_case(
+        TEST_NAME "top_level_no_prefix"
+        TEST_GROUPS top_level no_prefix
+        TEST_ARGS
+            TOP_LEVEL
+            NO_PREFIX
+            BUILD_STANDALONE_TESTS
+            BUILD_STANDALONE_EXAMPLES_IN_TREE
+            BUILD_STANDALONE_EXAMPLES_AS_EXTERNAL_PROJECTS
+    )
+endmacro()
+
+macro(test_top_level_no_prefix_in_source)
+    add_test_case(
+        TEST_NAME "top_level_no_prefix_in_source"
+        TEST_GROUPS top_level no_prefix in_source
+        TEST_ARGS
+            TOP_LEVEL
+            NO_PREFIX
+            IN_SOURCE
+    )
+endmacro()
+
+macro(test_top_level_prefix_in_tree_tests_and_examples_qt)
+    add_test_case(
+        TEST_NAME "top_level_prefix_in_tree_tests_and_examples"
+        TEST_GROUPS top_level prefix in_tree_tests_and_examples
+        TEST_ARGS
+            TOP_LEVEL
+            BUILD_IN_TREE_TESTS
+            BUILD_IN_TREE_EXAMPLES
+    )
+endmacro()
+
+macro(test_top_level_no_prefix_in_tree_tests_and_examples_qt)
+    add_test_case(
+        TEST_NAME "top_level_no_prefix_in_tree_tests_and_examples"
+        TEST_GROUPS top_level no_prefix in_tree_tests_and_examples
+        TEST_ARGS
+            TOP_LEVEL
+            NO_PREFIX
+            BUILD_IN_TREE_TESTS
+            BUILD_IN_TREE_EXAMPLES
+    )
+endmacro()
+
+# Collect all test cases and groups.
+macro(collect_tests)
+    set(TEST_CASES "")
+    set(TEST_GROUPS "")
+
+    test_per_repo_no_prefix_qt()
+    test_top_level_no_prefix_qt()
+
+    # This usually tested in regular CI as well.
+    test_per_repo_prefix_qt()
+
+    test_top_level_prefix_qt()
+
+    # TODO: These don't work atm due to some failed include(Targets) files.
+    #test_per_repo_no_prefix_in_tree_tests_and_examples_qt()
+    #test_top_level_no_prefix_in_tree_tests_and_examples_qt()
+    #test_per_repo_prefix_in_tree_tests_and_examples_qt()
+    #test_top_level_prefix_in_tree_tests_and_examples_qt()
+
+    test_per_repo_no_prefix_in_source()
+    test_top_level_no_prefix_in_source()
+
+    # TODO: Cross-builds.
+    # TODO: qt5.git builds with all submodules. Current limitation is that the sync-to
+    # script can't handle multiple modules at once, nor an "all repos" case.
+    # so we might have to call init-repository in that case.
+    # TODO: Unix Makefile builds.
+    # TODO: Build examples and tests, not only configure them.
+    # TODO: Perhaps run some of the cmake auto tests in configs that are not tested in CI
+    # like no-prefix builds.
+
+    message(STATUS "Available test cases: ${TEST_CASES}")
+    message(STATUS "Available test groups: ${TEST_GROUPS}")
+    foreach(group IN LISTS TEST_GROUPS)
+        message(STATUS "Available test cases for group ${group}: ${TEST_GROUPS_${group}}")
+    endforeach()
+endmacro()
+
+run_tests()

tests/manual/RunCMake/ConfigureBuildQt/check.cmake

@@ -0,0 +1,9 @@
+# This file is include()d by run_cmake right after it calls execute_process.
+# The msg var is set to a non-empty value when the was an error of
+# some kind. Record failure with a separate variable in the parent scope,
+# so we can FATAL_ERROR at the end.
+if(NOT "${msg}" STREQUAL "")
+    set(should_error_out TRUE PARENT_SCOPE)
+else()
+    set(should_error_out FALSE PARENT_SCOPE)
+endif()