Build & install protobuf on windows

Change-Id: Ic66bf6afba3ca53e13bfcd079471658eacfb528e Reviewed-by: Dimitrios Apostolou <jimis@qt.io>
2026-02-01 11:26:04 +08:00 · 2022-05-13 15:05:42 +02:00
parent 08ba3ff882
commit b09cac4448
7 changed files with 178 additions and 14 deletions
--- a/coin/platform_configs/cmake_platforms.yaml
+++ b/coin/platform_configs/cmake_platforms.yaml
@@ -52,7 +52,9 @@ Configurations:
        'CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\.. -DPostgreSQL_ROOT={{.Env.POSTGRESQL_LIB_x64}}\..',
        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}',
        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}',
-        'CONAN_PROFILE=coin/conan/profiles/ci-windows-x86_64-msvc2019_64']
+        'CONAN_PROFILE=coin/conan/profiles/ci-windows-x86_64-msvc2019_64',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
+    ]
 -
    Template: 'qtci-windows-11_21H2-x86_64-52'
    Compiler: 'MSVC2019'
@@ -61,7 +63,9 @@ Configurations:
    Environment variables: [
        'CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\.. -DPostgreSQL_ROOT={{.Env.POSTGRESQL_LIB_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}',
-        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}']
+        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
+    ]
 -
    Template: 'qtci-windows-10_21H2-x86_64-51'
    Compiler: 'MSVC2022'
@@ -70,7 +74,8 @@ Configurations:
    Environment variables: [
        'CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\.. -DPostgreSQL_ROOT={{.Env.POSTGRESQL_LIB_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}',
-        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}'
+        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
    ]
 -
    Template: 'qtci-windows-11_21H2-x86_64-52'
@@ -80,7 +85,8 @@ Configurations:
    Environment variables: [
        'CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\.. -DPostgreSQL_ROOT={{.Env.POSTGRESQL_LIB_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}',
-        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}'
+        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
    ]
 -
    Id: 'Windows10_21H2-MSVC2019-x64-host'
@@ -91,7 +97,8 @@ Configurations:
    Environment variables: [
        'CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\..',
        'CONAN_PROFILE=coin/conan/profiles/ci-windows-x86_64-msvc2019_64-developer',
-        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}'
+        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
    ]
 -
    Template: 'qtci-windows-10_21H2-x86_64-51'
@@ -106,7 +113,8 @@ Configurations:
        'CONAN_PROFILE=coin/conan/profiles/ci-windows-arm64-msvc2019_64',
        'OPENSSL_CONF_x64={{.Env.OPENSSL_CONF_x64_arm64}}',
        'OPENSSL_INCLUDE_x64={{.Env.OPENSSL_INCLUDE_x64_arm64}}',
-        'OPENSSL_LIB_x64={{.Env.OPENSSL_LIB_x64_arm64}}'
+        'OPENSSL_LIB_x64={{.Env.OPENSSL_LIB_x64_arm64}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
    ]
 -
    Id: 'Windows11_21H2-MSVC2019-x64-host'
@@ -116,7 +124,8 @@ Configurations:
    Configure arguments: '-debug-and-release -make examples -developer-build -force-debug-info -qt-zlib'
    Environment variables: [
        'CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\..',
-        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}'
+        'NON_QTBASE_CMAKE_ARGS=-DFFMPEG_DIR={{.Env.FFMPEG_DIR_MSVC}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
    ]
 -
    Template: 'qtci-windows-11_21H2-x86_64-52'
@@ -128,7 +137,8 @@ Configurations:
        'TARGET_CONFIGURE_ARGS=-debug-and-release -force-debug-info -platform win32-arm64-msvc',
        'TARGET_CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON',
        'CONAN_HOST_PROFILE=coin/conan/profiles/ci-windows-x86_64-msvc2019_64',
-        'CONAN_PROFILE=coin/conan/profiles/ci-windows-arm64-msvc2019_64'
+        'CONAN_PROFILE=coin/conan/profiles/ci-windows-arm64-msvc2019_64',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
    ]
 -
    Id: 'RHEL-8.4-host'
@@ -166,7 +176,8 @@ Configurations:
        'CMAKE_ARGS=-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\.. -DPostgreSQL_ROOT={{.Env.POSTGRESQL_LIB_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFEATURE_system_tiff=OFF -DFFMPEG_DIR={{.Env.FFMPEG_DIR_MINGW}}',
        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_mingw}}',
-        'CONAN_PROFILE=coin/conan/profiles/ci-windows-x86_64-mingw'
+        'CONAN_PROFILE=coin/conan/profiles/ci-windows-x86_64-mingw',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_mingw}}'
    ]
 -
    Id: 'Windows10_21H2-LLVM-x64-host'
@@ -178,7 +189,8 @@ Configurations:
        'Path=C:\llvm-mingw\bin;{{.Env.Path}}',
        'CMAKE_ARGS=-DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\.. -DPostgreSQL_ROOT={{.Env.POSTGRESQL_LIB_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFEATURE_system_tiff=OFF',
-        'LLVM_INSTALL_DIR=C:\llvm-mingw'
+        'LLVM_INSTALL_DIR=C:\llvm-mingw',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_llvm_mingw}}'
    ]
 -
    Id: 'Windows11_21H2-LLVM-x64-host'
@@ -190,7 +202,8 @@ Configurations:
        'Path=C:\llvm-mingw\bin;{{.Env.Path}}',
        'CMAKE_ARGS=-DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\.. -DPostgreSQL_ROOT={{.Env.POSTGRESQL_LIB_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFEATURE_system_tiff=OFF',
-        'LLVM_INSTALL_DIR=C:\llvm-mingw'
+        'LLVM_INSTALL_DIR=C:\llvm-mingw',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_llvm_mingw}}'
    ]
 -
    Template: 'qtci-windows-10_21H2-x86_64-51'
@@ -201,7 +214,8 @@ Configurations:
        'Path={{.Env.MINGW1120}}\bin;{{.Env.Path}}',
        'CMAKE_ARGS=-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFEATURE_system_tiff=OFF -DFFMPEG_DIR={{.Env.FFMPEG_DIR_MINGW}}',
-        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_mingw}}'
+        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_mingw}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_mingw}}'
    ]
 -
    Template: 'qtci-windows-11_21H2-x86_64-52'
@@ -212,5 +226,6 @@ Configurations:
        'Path={{.Env.MINGW1120}}\bin;{{.Env.Path}}',
        'CMAKE_ARGS=-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\..',
        'NON_QTBASE_CMAKE_ARGS=-DFEATURE_system_tiff=OFF -DFFMPEG_DIR={{.Env.FFMPEG_DIR_MINGW}}',
-        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_mingw}}'
+        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_mingw}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_mingw}}'
    ]
--- a/coin/platform_configs/cmake_platforms_static_tests.yaml
+++ b/coin/platform_configs/cmake_platforms_static_tests.yaml
@@ -16,7 +16,8 @@ Configurations:
    Configure arguments: '-release -force-debug-info -static -nomake examples -qt-zlib'
    Environment variables: [
        'CMAKE_ARGS=-DFEATURE_msvc_obj_debug_info=ON -DOPENSSL_ROOT_DIR={{.Env.OPENSSL_INCLUDE_x64}}\..',
-        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}'
+        'LLVM_INSTALL_DIR={{.Env.LLVM_INSTALL_DIR_msvc}}',
+        'Protobuf_ROOT={{.Env.Protobuf_ROOT_msvc}}'
    ]
 -
    Template: 'qtci-macos-11-x86_64-104'
--- a/coin/provisioning/common/windows/protobuf.ps1
+++ b/coin/provisioning/common/windows/protobuf.ps1
@@ -0,0 +1,140 @@
+# Copyright (C) 2022 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+. "$PSScriptRoot\helpers.ps1"
+
+# Here we build protobuf libraries for MinGW and MSVC.
+# Since it's a c++ library we need both msvc and mingw because they mangle symbols differently.
+# For MSVC it builds with both debug and release configurations because of the visual c++ runtime.
+# For MinGW we only need one, so we only build with release.
+# The function below takes care of the common part of building - invoking cmake,
+# calling ninja and installing it to a directory which we set an environment variable to.
+# Because we have two compilers we also have two env. vars. and then each
+# config in CI has the Protobuf_ROOT set to the appropriate one.
+function build-install-protobuf {
+    param(
+        [string]$CC,
+        [string]$CXX,
+        [string]$BuildType,
+        [string]$Postfix, # Used for install-path and the environment variable name
+        [string[]]$ExtraArguments = @()
+    )
+    $installPrefix = "C:\Utils\protobuf"
+    $installPath = "${installPrefix}-$Postfix"
+    Write-Output "Configuring and building protobuf for $CXX"
+    $oldCC = $env:CC
+    $oldCXX = $env:CXX
+    $env:CC = $CC
+    $env:CXX = $CXX
+    mkdir build
+    Push-Location build
+    cmake .. -G"Ninja Multi-Config" -DCMAKE_CONFIGURATION_TYPES="$BuildType" -DCMAKE_INSTALL_PREFIX="$installPath" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_WITH_ZLIB=OFF $ExtraArguments
+    # ninja install:all # This is broken and does not work
+    foreach ($config in $BuildType.split(";")) {
+        ninja -f "build-$config.ninja" install
+    }
+    $env:CC = $oldCC
+    $env:CXX = $oldCXX
+    Set-EnvironmentVariable "Protobuf_ROOT_$Postfix" "$InstallPath"
+    Pop-Location
+    Remove build
+}
+
+function Find-Tool {
+    param(
+        [string]$Name,
+        [string]$Path
+    )
+    # Is tool missing from path?
+    if (!(Get-Command $Name -ErrorAction SilentlyContinue)) {
+        # Is tool in the $Path directory?
+        if (Test-Path "$Path\$Name") {
+            $env:Path += ";$Path"
+        }
+        else {
+            throw "Cannot find $Name in path or $Name in $Name, something is configured wrong"
+        }
+    }
+}
+# This script is fairly late in provisioning so both of these should be present!
+Find-Tool -Name "cmake.exe" -Path "C:\CMake\bin"
+Find-Tool -Name "ninja.exe" -Path "C:\Utils\Ninja"
+
+$version = "3.20.1"
+$sha1 = "d89de6b747a264c0ddca3294b5095b785f1bd4f0"
+$internalUrl = "http://ci-files01-hki.intra.qt.io/input/protobuf/protobuf-all-$version.zip"
+$externalUrl = "https://github.com/protocolbuffers/protobuf/releases/download/v$version/protobuf-all-$version.zip"
+
+$targetDir = "$env:HOMEDRIVE\$env:HOMEPATH\protobuf-$version"
+$targetFile = "$targetDir.zip"
+Download  $externalUrl $internalUrl $targetFile
+Verify-Checksum $targetFile $sha1
+Extract-7Zip $targetFile (Join-Path $env:HOMEDRIVE $env:HOMEPATH)
+Remove $targetFile
+
+
+# cd into the cmake directory where the CMakeLists.txt file is located
+# then we build in a build\ subfolder there for simplicity's sake
+Push-Location $targetDir\cmake
+
+### MinGW
+
+# Check if mingw is where we expect it to be and add it to path:
+$mingwPath = "C:\MINGW1120\mingw64\bin"
+if (!(Test-Path $mingwPath)) {
+    throw "Cannot find mingw in $mingwPath, something is configured wrong"
+}
+
+$oldPath = $env:Path
+$env:Path = "$mingwPath;$env:Path"
+build-install-protobuf -CC "gcc" -CXX "g++" -BuildType "Release" -Postfix "mingw"
+$env:Path = $oldPath
+
+### LLVM MinGW
+
+$llvmMingwPath = "C:\llvm-mingw"
+if (!(Test-Path $llvmMingwPath)) {
+    throw "Cannot find llvm-mingw in $llvmMingwPath, something is configured wrong"
+}
+
+$oldPath = $env:Path
+$env:Path = "$llvmMingwPath\bin;$env:Path"
+build-install-protobuf -CC "clang" -CXX "clang++" -BuildType "Release" -Postfix "llvm_mingw"
+$env:Path = $oldPath
+
+### MSVC
+
+# Add cl to path if it is not already there
+if (!(Get-Command cl.exe -ErrorAction SilentlyContinue)) {
+    $vswhere = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"
+    $vc_component = "Microsoft.VisualStudio.Component.VC.CoreIde"
+    # We pick the oldest build tools we can find and use that to be compatible with it and any newer version:
+    # If MSVC has an ABI break this will stop working, and yet another build must be added.
+    $vs_location = (& $vswhere -nologo -products * -requires $vc_component -sort -format value -property installationPath | Select-Object -Last 1)
+    $vcvars_location = Join-Path $vs_location "VC\Auxiliary\Build\"
+
+    Push-Location $vcvars_location
+
+    # This snippet was stolen from https://stackoverflow.com/a/2124759
+    # Grabs all the environment variables that the script has set and assigns it
+    # to environment variables in PowerShell (calling batch files creates a
+    # cmd.exe instance which does not propagate environment variables)
+    cmd /c "vcvarsall.bat $env:PROCESSOR_ARCHITECTURE & set" |
+    ForEach-Object {
+        if ($_ -match "=") {
+            $v = $_.Split("=");
+            Set-Item -Force -Path "ENV:\$($v[0])" -Value "$($v[1])"
+        }
+    }
+
+    Pop-Location
+}
+
+# We pass along an extra argument to stop protobuf linking with the static runtime
+build-install-protobuf -CC "cl" -CXX "cl" -BuildType "Release" -Postfix "msvc" -ExtraArguments @("-Dprotobuf_MSVC_STATIC_RUNTIME=OFF")
+
+$env:Path = $oldPath
+Pop-Location
+Remove $targetDir
+
+Write-Output "Protobuf = $version" >> ~/versions.txt
--- a/coin/provisioning/qtci-windows-10-x86/92-protobuf.ps1
+++ b/coin/provisioning/qtci-windows-10-x86/92-protobuf.ps1
@@ -0,0 +1,2 @@
+. "$PSScriptRoot\..\common\windows\protobuf.ps1"
+
--- a/coin/provisioning/qtci-windows-10-x86_64/92-protobuf.ps1
+++ b/coin/provisioning/qtci-windows-10-x86_64/92-protobuf.ps1
@@ -0,0 +1,2 @@
+. "$PSScriptRoot\..\common\windows\protobuf.ps1"
+
--- a/coin/provisioning/qtci-windows-10_21H2-x86_64/92-protobuf.ps1
+++ b/coin/provisioning/qtci-windows-10_21H2-x86_64/92-protobuf.ps1
@@ -0,0 +1,2 @@
+. "$PSScriptRoot\..\common\windows\protobuf.ps1"
+
--- a/coin/provisioning/qtci-windows-11_21H2-x86_64/92-protobuf.ps1
+++ b/coin/provisioning/qtci-windows-11_21H2-x86_64/92-protobuf.ps1
@@ -0,0 +1,2 @@
+. "$PSScriptRoot\..\common\windows\protobuf.ps1"
+
				`@@ -0,0 +1,2 @@`
				`. "$PSScriptRoot\..\common\windows\protobuf.ps1"`