diff --git a/cmake/QtSynchronizeRepo.cmake b/cmake/QtSynchronizeRepo.cmake index eabd5c7c..3876fc0a 100644 --- a/cmake/QtSynchronizeRepo.cmake +++ b/cmake/QtSynchronizeRepo.cmake @@ -3,13 +3,62 @@ # This script is to be called (ideally from a git-sync-to alias script): # cmake -DSYNC_TO_MODULE="$1" -DSYNC_TO_BRANCH="$2" -P cmake/QtSynchronizeRepo.cmake + # Or as follows (ideally from a git-qt-foreach alias script): # cmake -DQT_FOREACH=TRUE "-DARGS=$*" -P cmake/QtSynchronizeRepo.cmake +# +# The script can take additional options. +# +# SYNC_REF_SPEC - an alias for SYNC_TO_BRANCH, can be a tag, branch or commit sha1. +# +# REMOTE_NAME - remote name to use for fetching, default is origin. +# +# GIT_DEPTH - corresponds to git's --depth option, will be passed to git clone and git submodule +# update --init operations. +# +# SHOW_PROGRESS - passes --progress to git submodule update operations +# +# VERBOSE - enables more verbose output +# +# The script also takes the following environment variables: +# +# QT_TL_SUBMODULE_UPDATE_FLAGS - additional flags to pass to git submodule update calls. +# +# To run the script in full debug mode use: +# cmake -DSYNC_TO_MODULE="$1" -DSYNC_TO_BRANCH="$2" -DSHOW_PROGRESS=1 -DVERBOSE=1 +# -P cmake/QtSynchronizeRepo.cmake --log-level=DEBUG --trace-redirect=log.txt --trace-expand cmake_policy(VERSION 3.16) -include(cmake/QtTopLevelHelpers.cmake) +include("${CMAKE_CURRENT_LIST_DIR}/QtTopLevelHelpers.cmake") if(QT_FOREACH) qt_internal_foreach_repo_run(ARGS ${ARGS}) else() - qt_internal_sync_to(${SYNC_TO_MODULE} ${SYNC_TO_BRANCH}) + set(args "") + + if(SYNC_REF_SPEC) + set(ref_spec "${SYNC_REF_SPEC}") + elseif(SYNC_TO_BRANCH) + set(ref_spec "${SYNC_TO_BRANCH}") + endif() + + if(REMOTE_NAME) + list(APPEND args REMOTE_NAME "${REMOTE_NAME}") + endif() + + if(GIT_DEPTH) + list(APPEND args GIT_DEPTH "${GIT_DEPTH}") + endif() + + if(SHOW_PROGRESS) + list(APPEND args SHOW_PROGRESS) + endif() + + if(VERBOSE) + list(APPEND args VERBOSE) + endif() + + qt_internal_sync_to(${SYNC_TO_MODULE} + SYNC_REF ${ref_spec} + ${args} + ) endif() diff --git a/cmake/QtTopLevelHelpers.cmake b/cmake/QtTopLevelHelpers.cmake index 7fe21e4f..69873d18 100644 --- a/cmake/QtTopLevelHelpers.cmake +++ b/cmake/QtTopLevelHelpers.cmake @@ -379,16 +379,560 @@ function(qt_internal_sort_module_dependencies modules out_all_ordered) set(${out_all_ordered} "${ordered}" PARENT_SCOPE) endfunction() -# does what it says, but also updates submodules -function(qt_internal_checkout module revision) +# Checks whether any unparsed arguments have been passed to the function at the call site. +# Use this right after `cmake_parse_arguments`. +function(qt_internal_tl_validate_all_args_are_parsed prefix) + if(DEFINED ${prefix}_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unknown arguments: (${${prefix}_UNPARSED_ARGUMENTS})") + endif() +endfunction() + +# If VERBOSE is not set or FALSE in the parent or root scopes, swallow the git output. +# If VERBOSE is true, echo the stdout output, as well as the command run. +function(qt_internal_tl_handle_verbose_git_operations) set(swallow_output "") # unless VERBOSE, eat git output, show it in case of error if (NOT VERBOSE) list(APPEND swallow_output "OUTPUT_VARIABLE" "git_output" "ERROR_VARIABLE" "git_output") + else() + list(APPEND swallow_output COMMAND_ECHO STDOUT) endif() + set(swallow_output "${swallow_output}" PARENT_SCOPE) +endfunction() + +# Returns true if the current working directory is a super module with a .gitmodules file. +# Likely means it's the qt5.git super repo. +function(qt_internal_tl_is_super_repo out_var) + execute_process( + COMMAND "git" "rev-parse" "--show-toplevel" + RESULT_VARIABLE git_result + OUTPUT_VARIABLE top_level_path + ERROR_VARIABLE git_stderr + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + if(NOT git_result AND top_level_path AND EXISTS "${top_level_path}/.gitmodules") + set(result TRUE) + else() + set(result FALSE) + endif() + + set(${out_var} ${result} PARENT_SCOPE) +endfunction() + +# Returns whether the given git repo is shallow (cloned with --depth arg). +function(qt_internal_tl_is_git_repo_shallow out_var working_directory) + message(DEBUG "Checking if repo in '${working_directory}' is shallow") + + execute_process( + COMMAND "git" "rev-parse" "--is-shallow-repository" + RESULT_VARIABLE git_result + OUTPUT_VARIABLE git_stdout + ERROR_VARIABLE git_stderr + WORKING_DIRECTORY "${working_directory}" + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + if(git_result) + message(FATAL_ERROR + "Failed to check if repo is shallow in '${working_directory}'\n" + "stdout: ${git_stdout}\n" + "stderr: ${git_stderr}") + endif() + + string(STRIP "${git_stdout}" git_stdout) + if(git_stdout) + set(value TRUE) + else() + set(value FALSE) + endif() + set(${out_var} "${value}" PARENT_SCOPE) +endfunction() + +# Returns whether the given refspec is known to the repo in the given working directory. +function(qt_internal_tl_is_git_ref_spec_known out_var refspec working_directory) + # The funny ^{commit} syntax means the refpsec resolves to a commit, as opposed to a blob or a + # tree. + execute_process( + COMMAND "git" "cat-file" "-e" "${refspec}^{commit}" + RESULT_VARIABLE git_result + OUTPUT_VARIABLE git_stdout + ERROR_VARIABLE git_stderr + WORKING_DIRECTORY "${working_directory}" + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + # A non-0 exit code means it doesn't exist. + if(git_result) + set(value FALSE) + else() + set(value TRUE) + endif() + + set(${out_var} "${value}" PARENT_SCOPE) +endfunction() + +# Unshallow the given git repo. +function(qt_internal_tl_git_unshallow_repo) + set(opt_args + SHOW_PROGRESS + ) + set(single_args + WORKING_DIRECTORY + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + if(NOT arg_WORKING_DIRECTORY) + message(FATAL_ERROR "WORKING_DIRECTORY is required") + endif() + + set(args "") + if(arg_SHOW_PROGRESS) + list(APPEND args --progress) + endif() + + execute_process( + COMMAND "git" "fetch" "--unshallow" ${args} + RESULT_VARIABLE git_result + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" + ${swallow_output} + ) + + if(git_result) + message(FATAL_ERROR + "Failed to unshallow repo in '${arg_WORKING_DIRECTORY}': ${git_stderr}") + endif() +endfunction() + +# Fetches a shallow ref spec (--depth 1) from the given remote. +function(qt_internal_tl_git_fetch_shallow_ref_spec) + set(opt_args + SHOW_PROGRESS + FATAL + ) + set(single_args + REMOTE + REF_SPEC + WORKING_DIRECTORY + OUT_VAR_RESULT + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + if(NOT arg_REF_SPEC) + message(FATAL_ERROR "REF_SPEC is required") + endif() + + if(NOT arg_REMOTE) + message(FATAL_ERROR "REMOTE is required") + endif() + + if(NOT arg_WORKING_DIRECTORY) + message(FATAL_ERROR "WORKING_DIRECTORY is required") + endif() + + if(NOT arg_OUT_VAR_RESULT) + message(FATAL_ERROR "OUT_VAR_RESULT is required") + endif() + + set(args "") + if(arg_SHOW_PROGRESS) + list(APPEND args --progress) + endif() + + execute_process( + COMMAND "git" "fetch" "--depth" "1" "${arg_REMOTE}" "${arg_REF_SPEC}" ${args} + RESULT_VARIABLE git_result + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" + ${swallow_output} + ) + + if(git_result) + set(result FALSE) + if(arg_FATAL) + set(message_type FATAL_ERROR) + else() + set(message_type DEBUG) + endif() + message(${message_type} + "Failed to fetch shallow ref spec '${arg_REF_SPEC}' " + "in '${arg_WORKING_DIRECTORY}': ${git_stderr}") + else() + set(result TRUE) + endif() + + if(arg_OUT_VAR_RESULT) + set("${arg_OUT_VAR_RESULT}" "${result}" PARENT_SCOPE) + endif() +endfunction() + +# Detects if a repo is shallow. If it is, checks if the given ref spec is known. If not, it will +# try to fetch it. If it's still unknown, will try to unshallow the repo. +function(qt_internal_tl_handle_shallow_repo) + set(opt_args + SHOW_PROGRESS + ) + set(single_args + REF_SPEC + REMOTE_NAME + WORKING_DIRECTORY + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + if(NOT arg_REF_SPEC) + message(FATAL_ERROR "REF_SPEC is required") + endif() + + if(NOT arg_REMOTE_NAME) + message(FATAL_ERROR "REMOTE_NAME is required") + endif() + + if(NOT arg_WORKING_DIRECTORY) + message(FATAL_ERROR "WORKING_DIRECTORY is required") + endif() + + qt_internal_tl_is_git_repo_shallow(is_shallow "${arg_WORKING_DIRECTORY}") + if(NOT is_shallow) + return() + endif() + + qt_internal_tl_is_git_ref_spec_known(is_known "${arg_REF_SPEC}" "${arg_WORKING_DIRECTORY}") + if(is_known) + return() + endif() + + set(remote "${arg_REMOTE_NAME}") + message(DEBUG + "Fetching with --depth 1 from '${remote}' due to unknown refspec '${arg_REF_SPEC}'") + + set(args "") + if(arg_SHOW_PROGRESS) + list(APPEND args SHOW_PROGRESS) + endif() + + qt_internal_tl_git_fetch_shallow_ref_spec( + REF_SPEC "${arg_REF_SPEC}" + REMOTE "${remote}" + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" + OUT_VAR_RESULT shallow_fetch_succeeded + ${args} + ) + + # Attempt to unshallow a repo if a ref spec that we are meant to check out to, + # is not known. + if(shallow_fetch_succeeded) + return() + endif() + + message(DEBUG + "Unshallowing repo in ${arg_WORKING_DIRECTORY} due to unknown " + "refspec ${arg_REF_SPEC}") + + qt_internal_tl_git_unshallow_repo( + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" + ${args} + ) +endfunction() + +# Checks if given string looks like a sha1. +function(qt_internal_tl_is_sha1_ish value out_var) + set(hex_digit "[0-9a-fA-F]") + string(REPEAT "${hex_digit}" 5 hex_5) + + set(hex_digit_maybe "${hex_digit}?") + string(REPEAT "${hex_digit_maybe}" 35 hex_35) + + # A sha1 would have at least 5 hex digits followed by 35 optional hex digits. + set(sha1_regex "^${hex_5}${hex_35}$") + + if("${value}" MATCHES "${sha1_regex}") + set(result TRUE) + else() + set(result FALSE) + endif() + + set(${out_var} "${result}" PARENT_SCOPE) +endfunction() + +# Directly updates the submodule sha in the supermodule, without having to checkout the submodule +# first. +# This is useful to be able to clone a specific revision with --depth 1 when using the "sync to +# module" feature. +# Causes the super module to have a "staged" change for the given submodule. +# Can only be used with a sha1, not a branch or tag or other refspec, because there might not be +# any repo info yet to resolve that refspec. +function(qt_internal_tl_modify_submodule_sha module revision working_directory) + qt_internal_tl_handle_verbose_git_operations() + + # This mode means 'treat path as a git submodule'. + set(mode "160000") + + execute_process( + COMMAND git update-index --add --cacheinfo "${mode},${revision},${module}" + RESULT_VARIABLE git_result + WORKING_DIRECTORY "${working_directory}" + ${swallow_output} + ) + if(git_result) + message(FATAL_ERROR "Failed to set initial submodule revision for '${module}'") + endif() +endfunction() + +# Unstages a previously staged change to the submodule sha in the supermodule. +# This should be run after qt_internal_tl_modify_submodule_sha and the submodule update operation, +# to not accidentally stage the change to the supermodule. +# It might still lieave the worktree dirty, because the checked out revision might be different +# from the one the supermodule expects, but that's fine, that's the point of the sync-to script. +function(qt_internal_tl_unstage_submodule_sha module working_directory) + qt_internal_tl_handle_verbose_git_operations() + + execute_process( + COMMAND git restore --staged "${module}" + RESULT_VARIABLE git_result + WORKING_DIRECTORY "${working_directory}" + ${swallow_output} + ) + if(git_result) + message(FATAL_ERROR "Failed to unstage submodule revision change for '${module}'") + endif() +endfunction() + +# Transforms a refspec into a commit sha1. +# Useful for git commands that can't take a refspec. +function(qt_internal_tl_get_refspec_as_sha) + set(opt_args + FATAL + ) + set(single_args + REF_SPEC + WORKING_DIRECTORY + OUT_VAR + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + if(NOT arg_REF_SPEC) + message(FATAL_ERROR "REF_SPEC is required") + endif() + + if(NOT arg_WORKING_DIRECTORY) + message(FATAL_ERROR "WORKING_DIRECTORY is required") + endif() + + if(NOT arg_OUT_VAR) + message(FATAL_ERROR "OUT_VAR is required") + endif() + + execute_process( + COMMAND "git" "rev-parse" "${arg_REF_SPEC}" + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" + RESULT_VARIABLE git_result + OUTPUT_VARIABLE git_stdout + ERROR_VARIABLE git_stderr + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(git_result) + message(WARNING "${git_stdout}") + if(arg_FATAL) + set(message_type FATAL_ERROR) + else() + set(message_type WARNING) + endif() + message("${message_type}" + "Failed to get sha1 of ${arg_REF_SPEC} in '${arg_WORKING_DIRECTORY}': ${git_stderr}") + endif() + + string(STRIP "${git_stdout}" git_stdout) + set(${arg_OUT_VAR} "${git_stdout}" PARENT_SCOPE) +endfunction() + +# Runs `submodule update --init` for a submodule. +# +# If REF_SPEC is passed and is a valid commit sha1, sets the current active submodule sha1 in the +# super module to the given sha1. This is useful for cloning a specific sha1 with --depth 1. +# +# GIT_DEPTH passes the given --depth for the submodule update operation. +# +# SHOW_PROGRESS passes --progress to the submodule update operation. +# +# OUT_VAR_RESULT - set to TRUE or FALSE depending on whether the submodule update init succeeded. + +function(qt_internal_tl_run_submodule_update_init module) + set(opt_args + FAILURE_IS_WARNING + SHOW_PROGRESS + ) + set(single_args + REF_SPEC + GIT_DEPTH + FAILURE_MESSAGE + WORKING_DIRECTORY + OUT_VAR_RESULT + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 1 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + qt_internal_tl_handle_verbose_git_operations() + + if(NOT arg_WORKING_DIRECTORY) + message(FATAL_ERROR "WORKING_DIRECTORY is required") + endif() + + qt_internal_tl_is_sha1_ish("${arg_REF_SPEC}" is_sha1_revision) + + # We can only modify the submodule sha1 if we are given a sha1 reference, not any kind of + # refspec. + if(arg_REF_SPEC AND is_sha1_revision) + qt_internal_tl_modify_submodule_sha("${module}" "${arg_REF_SPEC}" + "${arg_WORKING_DIRECTORY}") + endif() + + set(args "") + set(extra_flags "$ENV{QT_TL_SUBMODULE_UPDATE_FLAGS}") + if(extra_flags) + list(APPEND args ${extra_flags}) + endif() + if(arg_GIT_DEPTH) + list(APPEND args --depth "${arg_GIT_DEPTH}") + endif() + if(arg_SHOW_PROGRESS) + list(APPEND args --progress) + endif() + + execute_process( + COMMAND "git" "submodule" "update" "--init" ${args} "${module}" + RESULT_VARIABLE git_result + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" + ${swallow_output} + ) + + if(arg_REF_SPEC AND is_sha1_revision) + qt_internal_tl_unstage_submodule_sha("${module}" "${arg_WORKING_DIRECTORY}") + endif() + + if(git_result) + set(result FALSE) + + if(arg_FAILURE_IS_WARNING) + set(message_type WARNING) + else() + set(message_type FATAL_ERROR) + endif() + + message(${message_type} "${arg_FAILURE_MESSAGE}") + else() + set(result TRUE) + endif() + + if(arg_OUT_VAR_RESULT) + set("${arg_OUT_VAR_RESULT}" "${result}" PARENT_SCOPE) + endif() +endfunction() + +# Clones a 'qt/${REPO_NAME}' repo from code.qt.io. +function(qt_internal_tl_git_clone_repo) + set(opt_args + SHOW_PROGRESS + ) + set(single_args + REPO_NAME + GIT_DEPTH + REMOTE_URL_BASE + WORKING_DIRECTORY + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + qt_internal_tl_handle_verbose_git_operations() + + if(NOT arg_WORKING_DIRECTORY) + message(FATAL_ERROR "WORKING_DIRECTORY is required") + endif() + + if(NOT arg_REPO_NAME) + message(FATAL_ERROR "REPO_NAME is required") + endif() + + if(arg_REMOTE_URL_BASE) + set(remote_url_base "${arg_REMOTE_URL_BASE}") + else() + set(remote_url_base "https://code.qt.io/qt/") + endif() + + set(remote_url "${remote_url_base}${arg_REPO_NAME}.git") + + message(NOTICE "Cloning '${arg_REPO_NAME}' from '${remote_url}'") + + set(clone_args "") + if(arg_GIT_DEPTH) + list(APPEND clone_args --depth "${arg_GIT_DEPTH}") + endif() + if(arg_SHOW_PROGRESS) + list(APPEND clone_args --progress) + endif() + + # Note that cloning does not allow fetching a specific sha1 directly if --depth is + # specified. It can only take a branch or tag name. So we don't pass REF_SPEC here. + execute_process( + COMMAND "git" "clone" "${remote_url}" ${clone_args} + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}" + RESULT_VARIABLE git_result + ${swallow_output} + ) + if(git_result) + message(FATAL_ERROR + "Failed to clone '${module}' from '${remote_url}': ${git_output}") + endif() +endfunction() + +# Checks out a submodule to a given refspec, and runs 'git submodule update' in the submodule +# directory. +# If a regular checkout does not work, a detached checkout is attempted. +function(qt_internal_checkout module revision) + set(opt_args + SHOW_PROGRESS + ) + set(single_args + REMOTE_NAME + WORKING_DIRECTORY + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 2 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + qt_internal_tl_handle_verbose_git_operations() + + if(NOT arg_WORKING_DIRECTORY) + message(FATAL_ERROR "WORKING_DIRECTORY is required") + endif() + + if(NOT arg_REMOTE_NAME) + message(FATAL_ERROR "REMOTE_NAME is required") + endif() + + set(shallow_args "") + if(arg_SHOW_PROGRESS) + list(APPEND shallow_args SHOW_PROGRESS) + endif() + + qt_internal_tl_handle_shallow_repo( + REF_SPEC "${revision}" + REMOTE_NAME "${arg_REMOTE_NAME}" + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}/${module}" + ${shallow_args} + ) + message(NOTICE "Checking '${module}' out to revision '${revision}'") execute_process( COMMAND "git" "checkout" "${revision}" - WORKING_DIRECTORY "./${module}" + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}/${module}" RESULT_VARIABLE git_result ${swallow_output} ) @@ -396,7 +940,7 @@ function(qt_internal_checkout module revision) message(WARNING "${git_output}, trying detached checkout") execute_process( COMMAND "git" "checkout" "--detach" "${revision}" - WORKING_DIRECTORY "./${module}" + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}/${module}" RESULT_VARIABLE git_result ${swallow_output} ) @@ -404,45 +948,89 @@ function(qt_internal_checkout module revision) if (git_result) message(FATAL_ERROR "Failed to check '${module}' out to '${revision}': ${git_output}") endif() + + set(args "") + set(extra_flags "$ENV{QT_TL_SUBMODULE_UPDATE_FLAGS}") + if(extra_flags) + list(APPEND args ${extra_flags}) + endif() + if(arg_SHOW_PROGRESS) + list(APPEND args --progress) + endif() + execute_process( - COMMAND "git" "submodule" "update" - WORKING_DIRECTORY "./${module}" + COMMAND "git" "submodule" "update" ${args} + WORKING_DIRECTORY "${arg_WORKING_DIRECTORY}/${module}" RESULT_VARIABLE git_result - OUTPUT_VARIABLE git_stdout - ERROR_VARIABLE git_stderr + ${swallow_output} ) endfunction() -# clones or creates a worktree for $dependency, using the source of $dependent +# Clones or creates a worktree, or initializes a submodule for $dependency, using the source of +# $dependent. +# Example dependent: qtdeclarative +# Example dependency: qtbase function(qt_internal_get_dependency dependent dependency) - set(swallow_output "") # unless VERBOSE, eat git output, show it in case of error - if (NOT VERBOSE) - list(APPEND swallow_output "OUTPUT_VARIABLE" "git_output" "ERROR_VARIABLE" "git_output") + set(opt_args + SHOW_PROGRESS + ) + set(single_args + GIT_DEPTH + REMOTE_NAME + REF_SPEC + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 2 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + qt_internal_tl_handle_verbose_git_operations() + + if(NOT arg_REMOTE_NAME) + message(FATAL_ERROR "REMOTE_NAME is required") endif() + set(show_progress_args "") + if(arg_SHOW_PROGRESS) + set(show_progress_args SHOW_PROGRESS) + endif() + + # This will hold the path to parent dir of the main ${dependent} worktree, regardless if it's a + # clone or a git worktree. + # So if dependent is 'src/qt6/qtshadertools' + # gitdir will be 'src/qt6' + # If dependent is 'worktrees/6.8-worktree/qtshadertools' + # gitdir will still be 'src/qt6', not 'worktrees/6.8-worktree' set(gitdir "") + + # The remote url. set(remote "") - # try to read the worktree source + # Worktree of dependent, aka who depends on dependency. + set(dependent_path "${CMAKE_CURRENT_SOURCE_DIR}/${dependent}") + + # Try to get the dependent worktree git dir. execute_process( - COMMAND "git" "rev-parse" "--git-dir" - WORKING_DIRECTORY "./${dependent}" + COMMAND "git" "rev-parse" "--absolute-git-dir" + WORKING_DIRECTORY "${dependent_path}" RESULT_VARIABLE git_result OUTPUT_VARIABLE git_stdout ERROR_VARIABLE git_stderr OUTPUT_STRIP_TRAILING_WHITESPACE ) + message(DEBUG "Original gitdir for '${dependent_path}' is '${git_stdout}'") + string(FIND "${git_stdout}" "${module}" index) string(SUBSTRING "${git_stdout}" 0 ${index} gitdir) string(FIND "${gitdir}" ".git/modules" index) if(index GREATER -1) # submodules have not been absorbed string(SUBSTRING "${gitdir}" 0 ${index} gitdir) endif() - message(DEBUG "Will look for clones in ${gitdir}") + + message(DEBUG "Will check computed '${gitdir}' for worktrees and clones.") execute_process( - COMMAND "git" "remote" "get-url" "origin" - WORKING_DIRECTORY "./${dependent}" + COMMAND "git" "remote" "get-url" "${arg_REMOTE_NAME}" + WORKING_DIRECTORY "${dependent_path}" RESULT_VARIABLE git_result OUTPUT_VARIABLE git_stdout ERROR_VARIABLE git_stderr @@ -450,67 +1038,200 @@ function(qt_internal_get_dependency dependent dependency) ) string(FIND "${git_stdout}" "${dependent}.git" index) string(SUBSTRING "${git_stdout}" 0 ${index} remote) - message(DEBUG "Will clone from ${remote}") - if(EXISTS "${gitdir}.gitmodules" AND NOT EXISTS "${gitdir}${dependency}/.git") - # super repo exists, but the submodule we need does not - try to initialize - message(NOTICE "Initializing submodule '${dependency}' from ${gitdir}") - execute_process( - COMMAND "git" "submodule" "update" "--init" "${dependency}" - WORKING_DIRECTORY "${gitdir}" - RESULT_VARIABLE git_result - ${swallow_output} - ) - if (git_result) - # ignore errors, fall back to an independent clone instead - message(WARNING "Failed to initialize submodule '${dependency}' from ${gitdir}") + message(DEBUG "Original remote for '${dependent_path}' is '${git_stdout}'") + + set(maybe_super_module_path "${gitdir}") + set(maybe_submodule_path "${maybe_super_module_path}${dependency}") + set(maybe_submodule_git_path "${maybe_submodule_path}/.git") + + set(maybe_existing_worktree_path "${maybe_submodule_path}") + + if(EXISTS "${maybe_super_module_path}.gitmodules" AND NOT EXISTS "${maybe_submodule_git_path}") + set(use_submodule_init TRUE) + message(DEBUG + "Will attempt to initialize submodule using supermodule ${maybe_super_module_path}") + else() + set(use_submodule_init FALSE) + if(EXISTS "${maybe_existing_worktree_path}") + message(DEBUG "Will attempt to use worktree from ${maybe_existing_worktree_path}") + else() + message(DEBUG "Will clone from ${remote}") endif() endif() - if(EXISTS "${gitdir}${dependency}") + if(use_submodule_init) + # super repo exists, but the submodule we need does not - try to initialize + message(NOTICE "Initializing submodule '${dependency}' from ${maybe_super_module_path}") + + set(args + FAILURE_MESSAGE + "Failed to initialize submodule '${dependency}' from ${maybe_super_module_path}" + + # Ignore errors, fall back to an independent clone below. + FAILURE_IS_WARNING + + OUT_VAR_RESULT submodule_update_init_result + WORKING_DIRECTORY "${gitdir}" + ${show_progress_args} + ) + if(arg_GIT_DEPTH) + list(APPEND args + GIT_DEPTH "${arg_GIT_DEPTH}" + REF_SPEC "${arg_REF_SPEC}" + ) + endif() + qt_internal_tl_run_submodule_update_init("${dependency}" ${args}) + endif() + + # If the submodule was initialized in the super repo in the code above, and the location where + # we're supposed to clone the dependency is the same, skip trying to clone the dependency or + # setting up a worktree, because it's already there. + set(new_dependency_path "${CMAKE_CURRENT_SOURCE_DIR}/${dependency}") + if(EXISTS "${new_dependency_path}" AND + "${new_dependency_path}" STREQUAL "${maybe_existing_worktree_path}") + return() + endif() + + if(EXISTS "${maybe_existing_worktree_path}") # for the module we want, there seems to be a clone parallel to what we have message(NOTICE "Adding worktree for ${dependency} from ${gitdir}${dependency}") execute_process( - COMMAND "git" "worktree" "add" "--detach" "${CMAKE_CURRENT_SOURCE_DIR}/${dependency}" - WORKING_DIRECTORY "${gitdir}/${dependency}" + COMMAND "git" "worktree" "add" "--detach" "${new_dependency_path}" + WORKING_DIRECTORY "${maybe_existing_worktree_path}" RESULT_VARIABLE git_result ${swallow_output} ) - if (git_result) - message(FATAL_ERROR "Failed to check '${module}' out to '${revision}': ${git_output}") + if(git_result) + message(FATAL_ERROR + "Failed to add worktree '${module}' from '${new_dependency_path}': ${git_output}") endif() else() - # we don't find the existing clone, so clone from the same remote - message(NOTICE "Cloning ${dependency} from ${remote}${dependency}.git") - execute_process( - COMMAND "git" "clone" "${remote}${dependency}.git" - WORKING_DIRECTORY "." - RESULT_VARIABLE git_result - ${swallow_output} - ) - if (git_result) - message(FATAL_ERROR "Failed to check '${module}' out to '${revision}': ${git_output}") + # We didn't find an existing clone or worktree, so clone from the same remote. + set(clone_args "") + if(arg_GIT_DEPTH) + list(APPEND clone_args GIT_DEPTH "${arg_GIT_DEPTH}") endif() + if(arg_SHOW_PROGRESS) + list(APPEND clone_args SHOW_PROGRESS) + endif() + + qt_internal_tl_git_clone_repo( + REPO_NAME "${dependency}" + REMOTE_URL_BASE "${remote}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ${clone_args} + ) endif() endfunction() -# evaluates the dependencies for $module, and checks all dependencies -# out so that it is a consistent set +# Syncs a submodule to a given refspec, collects its dependencies, and then checks out +# the submodule and its dependencies to a consistent set, according to the submodule +# dependencies.yaml file. +# +# A special case is when the module is ".", in which case all submodules are checked out to the +# given refspec, e.g. check out everything to origin/dev/HEAD. +# +# Initializes the submodule and any of its dependencies if they are not already initialized, when +# executed in a qt5.git checkout. +# +# Clones the specified submodule from code.qt.io if it missing, and not in a qt5.git checkout. function(qt_internal_sync_to module) - if(ARGN) - set(revision "${ARGV1}") - # special casing "." as the target module - checkout all out to $revision - if("${module}" STREQUAL ".") - qt_internal_find_modules(modules) - foreach(module IN LISTS modules) - qt_internal_checkout("${module}" "${revision}") - endforeach() - return() - endif() + set(opt_args + VERBOSE + SHOW_PROGRESS + ) + set(single_args + SYNC_REF + REMOTE_NAME + GIT_DEPTH + ) + set(multi_args "") + cmake_parse_arguments(PARSE_ARGV 1 arg "${opt_args}" "${single_args}" "${multi_args}") + qt_internal_tl_validate_all_args_are_parsed(arg) + + if(arg_VERBOSE) + # This is meant to trickle into scopes of other functions as well. + set(VERBOSE TRUE) + endif() + + set(show_progress_args "") + if(arg_SHOW_PROGRESS) + set(show_progress_args SHOW_PROGRESS) + endif() + + if(arg_REMOTE_NAME) + set(remote_name "${arg_REMOTE_NAME}") else() + set(remote_name "origin") + endif() + + set(revision "${arg_SYNC_REF}") + + # Special casing "." as the target module - checkout all initialized submodules to $revision. + # If revision is unset, check out to dev. + if("${module}" STREQUAL ".") + if(NOT revision) + set(revision "dev") + endif() + + qt_internal_find_modules(modules) + foreach(module IN LISTS modules) + qt_internal_checkout("${module}" "${revision}" ${show_progress_args} + REMOTE_NAME "${remote_name}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + endforeach() + return() + endif() + + # If no revision given, checkout to the HEAD as specified by the super module. + if(NOT revision) set(revision "HEAD") endif() - qt_internal_checkout("${module}" "${revision}") + + set(submodule_path "${CMAKE_CURRENT_SOURCE_DIR}/${module}") + set(submodule_git_path "${submodule_path}/.git") + + # We are in a qt5.git dir, but the requested submodule is not initialized yet, try to + # initialize it. + qt_internal_tl_is_super_repo(is_super_repo) + if(is_super_repo AND NOT EXISTS "${submodule_git_path}") + message(NOTICE "Initializing submodule '${module}' within supermodule.") + + set(args + FAILURE_MESSAGE "Failed to initialize initial submodule '${module}'" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ${show_progress_args} + ) + if(arg_GIT_DEPTH) + list(APPEND args + GIT_DEPTH "${arg_GIT_DEPTH}" + REF_SPEC "${revision}" + ) + endif() + qt_internal_tl_run_submodule_update_init("${module}" ${args}) + endif() + + # If we were in a qt5.git dir, the submodule should have been initialized by now. + # If we were in some random src/ dir, we need to manually clone the repo. + if(NOT EXISTS "${submodule_path}") + qt_internal_tl_git_clone_repo( + REPO_NAME "${module}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ${show_progress_args} + ) + endif() + + if(NOT EXISTS "${submodule_git_path}") + message(FATAL_ERROR "No worktree for '${module}' found in '${submodule_path}'") + endif() + + # Check out the submodule to the given refspec. + qt_internal_checkout("${module}" "${revision}" ${show_progress_args} + REMOTE_NAME "${remote_name}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) qt_internal_resolve_module_dependencies(${module} initial_dependencies initial_revisions) if(initial_dependencies) @@ -524,12 +1245,12 @@ function(qt_internal_sync_to module) endif() set(revision "") - set(checkedout "1") + set(should_visit_dependencies "1") # Load all dependencies for $module, then iterate over the dependencies in reverse order, # and check out the first that isn't already at the required revision. # Repeat everything (we need to reload dependencies after each checkout) until no more checkouts # are done. - while(${checkedout}) + while(${should_visit_dependencies}) qt_internal_resolve_module_dependencies(${module} dependencies revisions) message(DEBUG "${module} dependencies: ${dependencies}") message(DEBUG "${module} revisions : ${revisions}") @@ -541,7 +1262,7 @@ function(qt_internal_sync_to module) endif() math(EXPR count "${count} - 1") - set(checkedout 0) + set(should_visit_dependencies 0) foreach(i RANGE ${count} 0 -1 ) list(GET dependencies ${i} dependency) list(GET revisions ${i} revision) @@ -550,32 +1271,53 @@ function(qt_internal_sync_to module) continue() endif() - if(NOT EXISTS "./${dependency}") - message(DEBUG "No worktree for '${dependency}' found in '${CMAKE_CURRENT_SOURCE_DIR}'") - qt_internal_get_dependency("${module}" "${dependency}") + # When in a super module, the dependency directory might exist, but is empty if the + # submodule was not yet initiallized. Check its existence and initialization state + # by looking at the existence of the .git file or directory. + if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${dependency}/.git") + message(DEBUG + "No worktree for '${dependency}' found in '${CMAKE_CURRENT_SOURCE_DIR}'. " + "Trying to acquire it." + ) + + set(args ${show_progress_args}) + if(arg_GIT_DEPTH) + list(APPEND args GIT_DEPTH "${arg_GIT_DEPTH}") + endif() + qt_internal_get_dependency("${module}" "${dependency}" + REF_SPEC "${revision}" + REMOTE_NAME "${remote_name}" + ${args} + ) + + set(should_visit_dependencies 1) endif() - execute_process( - COMMAND "git" "rev-parse" "HEAD" - WORKING_DIRECTORY "./${dependency}" - RESULT_VARIABLE git_result - OUTPUT_VARIABLE git_stdout - ERROR_VARIABLE git_stderr - OUTPUT_STRIP_TRAILING_WHITESPACE + qt_internal_tl_get_refspec_as_sha( + REF_SPEC "HEAD" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${dependency}" + OUT_VAR head_ref ) - if (git_result) - message(WARNING "${git_stdout}") - message(FATAL_ERROR "Failed to get current HEAD of '${dependency}': ${git_stderr}") - endif() - if ("${git_stdout}" STREQUAL "${revision}") + + if("${head_ref}" STREQUAL "${revision}") + message(DEBUG + "The dependency ${dependency} is already checked out to ${revision}. " + "Continuing to next dependency." + ) continue() endif() - qt_internal_checkout("${dependency}" "${revision}") - set(checkedout 1) + qt_internal_checkout("${dependency}" "${revision}" ${show_progress_args} + REMOTE_NAME "${remote_name}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + set(should_visit_dependencies 1) + + # Start revisiting the dependencies in the while loop. break() endforeach() endwhile() + message(DEBUG "Module syncing finished.") endfunction() # Runs user specified command for all qt repositories in qt directory.