vcpkg/scripts/azure-pipelines/test-modified-ports.ps1

178 lines
6.5 KiB
PowerShell
Raw Normal View History

# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: MIT
#
<#
.SYNOPSIS
Runs the 'Test Modified Ports' part of the vcpkg CI system for all platforms.
.PARAMETER Triplet
The triplet to test.
.PARAMETER WorkingRoot
The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories.
.PARAMETER ArtifactStagingDirectory
The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.
.PARAMETER ArchivesRoot
Equivalent to '-BinarySourceStub "files,$ArchivesRoot"'
.PARAMETER BinarySourceStub
The type and parameters of the binary source. Shared across runs of this script. If
this parameter is not set, binary caching will not be used. Example: "files,W:\"
.PARAMETER BuildReason
The reason Azure Pipelines is running this script. For invocations caused by `PullRequest`,
modified ports are identified by changed hashes with regard to git HEAD~1 (subject to NoParentHashes),
and ports marked as failing in the CI baseline (or which depend on such ports) are skipped.
If BinarySourceStub is set and this parameter is set to a non-empty value other than `PullRequest`,
binary caching will be in write-only mode.
.PARAMETER NoParentHashes
Indicates to not use parent hashes even for pull requests.
.PARAMETER PassingIsPassing
Indicates that 'Passing, remove from fail list' results should not be emitted as failures. (For example, this is used
when using vcpkg to test a prerelease MSVC++ compiler)
#>
[CmdletBinding(DefaultParameterSetName="ArchivesRoot")]
Param(
[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
[string]$Triplet,
[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
$WorkingRoot,
[ValidateNotNullOrEmpty()]
$ArtifactStagingDirectory = '.',
[Parameter(ParameterSetName='ArchivesRoot')]
$ArchivesRoot = $null,
[Parameter(ParameterSetName='BinarySourceStub')]
$BinarySourceStub = $null,
[String]$BuildReason = $null,
[switch]$NoParentHashes = $false,
[switch]$PassingIsPassing = $false
)
if (-Not ((Test-Path "triplets/$Triplet.cmake") -or (Test-Path "triplets/community/$Triplet.cmake"))) {
Write-Error "Incorrect triplet '$Triplet', please supply a valid triplet."
throw
}
if ((-Not [string]::IsNullOrWhiteSpace($ArchivesRoot))) {
if ((-Not [string]::IsNullOrWhiteSpace($BinarySourceStub))) {
Write-Error "Only one binary caching setting may be used."
throw
[opentracing] Update VMs 2020-12 (#15151) * Add meson from https://github.com/microsoft/vcpkg/pull/12860/ * Add autoconf-archive from https://github.com/microsoft/vcpkg/pull/13081/ * Add kf5windowsystem libs from https://github.com/microsoft/vcpkg/pull/13467/ * Open the FTP and SFTP ports from https://github.com/microsoft/vcpkg/pull/14412/ * Add libxcb-util0-dev from https://github.com/microsoft/vcpkg/pull/14678/ * Add libasound2-dev from https://github.com/microsoft/vcpkg/pull/14774 * Remove no longer necessary apt-mark calls. * Update nasm on Linux. * Fix longstanding bug where Storage was publicly accessible and change to generate SAS token rather than File Share * Delete no longer necessary azure storage firewall rules. * Install the newer Windows SDK with the VS installer instead of manually. * Install the VS2015 and VS2017 compilers. * Update Powershell-Core to 7.1.0. * Update source of WDK. * Update pools. * [opentracing] Repair arm64-windows failures caused by mojibake in `expected.hpp` and errors in opentracing-cpp's lint for arm64 where it thinks exceptions are disabled when they are enabled. Fixes: C:\Dev\vcpkg\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\3rd_party\include\opentracing/expected/expected.hpp(1): warning C4828: The file contains a character starting at offset 0x4a77 that is illegal in the current source character set (codepage 65001). Fixes: D:\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\include\opentracing/tracer.h:223:5: error: cannot use 'try' with exceptions disabled [clang-diagnostic-error] try { ^ * [mmloader] Patch out overrides of CMAKE_C_FLAGS and CMAKE_CXX_FLAGS that inserted /WX. Note that this port sets /GS-, possibly because it may be for authoring shellcode. * Bump storage API version to 2020-04-08.
2020-12-23 06:52:58 +08:00
}
$BinarySourceStub = "files,$ArchivesRoot"
[opentracing] Update VMs 2020-12 (#15151) * Add meson from https://github.com/microsoft/vcpkg/pull/12860/ * Add autoconf-archive from https://github.com/microsoft/vcpkg/pull/13081/ * Add kf5windowsystem libs from https://github.com/microsoft/vcpkg/pull/13467/ * Open the FTP and SFTP ports from https://github.com/microsoft/vcpkg/pull/14412/ * Add libxcb-util0-dev from https://github.com/microsoft/vcpkg/pull/14678/ * Add libasound2-dev from https://github.com/microsoft/vcpkg/pull/14774 * Remove no longer necessary apt-mark calls. * Update nasm on Linux. * Fix longstanding bug where Storage was publicly accessible and change to generate SAS token rather than File Share * Delete no longer necessary azure storage firewall rules. * Install the newer Windows SDK with the VS installer instead of manually. * Install the VS2015 and VS2017 compilers. * Update Powershell-Core to 7.1.0. * Update source of WDK. * Update pools. * [opentracing] Repair arm64-windows failures caused by mojibake in `expected.hpp` and errors in opentracing-cpp's lint for arm64 where it thinks exceptions are disabled when they are enabled. Fixes: C:\Dev\vcpkg\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\3rd_party\include\opentracing/expected/expected.hpp(1): warning C4828: The file contains a character starting at offset 0x4a77 that is illegal in the current source character set (codepage 65001). Fixes: D:\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\include\opentracing/tracer.h:223:5: error: cannot use 'try' with exceptions disabled [clang-diagnostic-error] try { ^ * [mmloader] Patch out overrides of CMAKE_C_FLAGS and CMAKE_CXX_FLAGS that inserted /WX. Note that this port sets /GS-, possibly because it may be for authoring shellcode. * Bump storage API version to 2020-04-08.
2020-12-23 06:52:58 +08:00
}
$env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads'
$buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
$installRoot = Join-Path $WorkingRoot 'installed'
$packagesRoot = Join-Path $WorkingRoot 'packages'
$commonArgs = @(
"--x-buildtrees-root=$buildtreesRoot",
"--x-install-root=$installRoot",
"--x-packages-root=$packagesRoot",
"--overlay-ports=scripts/test_ports"
)
$cachingArgs = @()
$skipFailuresArg = @()
if ([string]::IsNullOrWhiteSpace($BinarySourceStub)) {
$cachingArgs = @('--no-binarycaching')
} else {
$cachingArgs = @('--binarycaching')
$binaryCachingMode = 'readwrite'
if ([string]::IsNullOrWhiteSpace($BuildReason)) {
Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.'
}
elseif ($BuildReason -eq 'PullRequest') {
Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.'
$skipFailuresArg = @('--skip-failures')
}
else {
Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
$binaryCachingMode = 'write'
}
[opentracing] Update VMs 2020-12 (#15151) * Add meson from https://github.com/microsoft/vcpkg/pull/12860/ * Add autoconf-archive from https://github.com/microsoft/vcpkg/pull/13081/ * Add kf5windowsystem libs from https://github.com/microsoft/vcpkg/pull/13467/ * Open the FTP and SFTP ports from https://github.com/microsoft/vcpkg/pull/14412/ * Add libxcb-util0-dev from https://github.com/microsoft/vcpkg/pull/14678/ * Add libasound2-dev from https://github.com/microsoft/vcpkg/pull/14774 * Remove no longer necessary apt-mark calls. * Update nasm on Linux. * Fix longstanding bug where Storage was publicly accessible and change to generate SAS token rather than File Share * Delete no longer necessary azure storage firewall rules. * Install the newer Windows SDK with the VS installer instead of manually. * Install the VS2015 and VS2017 compilers. * Update Powershell-Core to 7.1.0. * Update source of WDK. * Update pools. * [opentracing] Repair arm64-windows failures caused by mojibake in `expected.hpp` and errors in opentracing-cpp's lint for arm64 where it thinks exceptions are disabled when they are enabled. Fixes: C:\Dev\vcpkg\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\3rd_party\include\opentracing/expected/expected.hpp(1): warning C4828: The file contains a character starting at offset 0x4a77 that is illegal in the current source character set (codepage 65001). Fixes: D:\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\include\opentracing/tracer.h:223:5: error: cannot use 'try' with exceptions disabled [clang-diagnostic-error] try { ^ * [mmloader] Patch out overrides of CMAKE_C_FLAGS and CMAKE_CXX_FLAGS that inserted /WX. Note that this port sets /GS-, possibly because it may be for authoring shellcode. * Bump storage API version to 2020-04-08.
2020-12-23 06:52:58 +08:00
$cachingArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode")
}
if ($Triplet -eq 'x64-linux') {
$env:HOME = '/home/agent'
$executableExtension = [string]::Empty
}
elseif ($Triplet -eq 'x64-osx') {
$executableExtension = [string]::Empty
}
else {
$executableExtension = '.exe'
}
$xmlResults = Join-Path $ArtifactStagingDirectory 'xml-results'
mkdir $xmlResults
$xmlFile = Join-Path $xmlResults "$Triplet.xml"
$failureLogs = Join-Path $ArtifactStagingDirectory 'failure-logs'
if ($IsWindows)
{
mkdir empty
cmd /c "robocopy.exe empty `"$buildtreesRoot`" /MIR /NFL /NDL /NC /NP > nul"
cmd /c "robocopy.exe empty `"$packagesRoot`" /MIR /NFL /NDL /NC /NP > nul"
cmd /c "robocopy.exe empty `"$installRoot`" /MIR /NFL /NDL /NC /NP > nul"
rmdir empty
}
& "./vcpkg$executableExtension" x-ci-clean @commonArgs
if ($LASTEXITCODE -ne 0)
{
throw "vcpkg clean failed"
}
$parentHashes = @()
if (($BuildReason -eq 'PullRequest') -and -not $NoParentHashes)
{
# Prefetch tools for better output
foreach ($tool in @('cmake', 'ninja', 'git')) {
& "./vcpkg$executableExtension" fetch $tool
if ($LASTEXITCODE -ne 0)
{
throw "Failed to fetch $tool"
}
}
Write-Host "Determining parent hashes using HEAD~1"
$parentHashesFile = Join-Path $ArtifactStagingDirectory 'parent-hashes.json'
$parentHashes = @("--parent-hashes=$parentHashesFile")
& git revert -n -m 1 HEAD | Out-Null
# The vcpkg.cmake toolchain file is not part of ABI hashing,
# but changes must trigger at least some testing.
Copy-Item "scripts/buildsystems/vcpkg.cmake" -Destination "scripts/test_ports/cmake"
Copy-Item "scripts/buildsystems/vcpkg.cmake" -Destination "scripts/test_ports/cmake-user"
Update vcpkg-tool to 2022-03-25 (#23757) * Update vcpkg-tool to 2022-03-24 * Hook up https://github.com/microsoft/vcpkg-tool/pull/345 * Hook up https://github.com/microsoft/vcpkg-tool/pull/442 * Update vcpkg-tool to 2022-03-25 * Analysis of failures. * [Most recent nightly build failed](https://dev.azure.com/vcpkg/public/_build/results?buildId=69427) * [Validation of this tool update failed](https://dev.azure.com/vcpkg/public/_build/results?buildId=69417) ## Common to both: PASSING, REMOVE FROM FAIL LIST: chartdir:x64-windows (.\scripts\ci.baseline.txt) PASSING, REMOVE FROM FAIL LIST: chartdir:x64-windows-static-md (.\scripts\ci.baseline.txt) PASSING, REMOVE FROM FAIL LIST: chartdir:x86-windows (.\scripts\ci.baseline.txt) Probably fixed by https://github.com/microsoft/vcpkg/pull/23701 PASSING, REMOVE FROM FAIL LIST: gmp:x64-uwp (.\scripts\ci.baseline.txt) PASSING, REMOVE FROM FAIL LIST: gmp:x64-windows-static-md (.\scripts\ci.baseline.txt) Probably fixed by https://github.com/microsoft/vcpkg/pull/23466 ? REGRESSION: colmap:x64-windows-static-md failed with BUILD_FAILED. If expected, add colmap:x64-windows-static-md=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. I don't know exactly what changed. I observe that * this thing depends on a *lot* of stuff * on March 14 we didn't even attempt to build this * the x64-windows ones are already in the baseline so I skipped it. REGRESSION: qtdeclarative:x64-windows. If expected, add qtdeclarative:x64-windows=fail to .\scripts\ci.baseline.txt. REGRESSION: qtdeclarative:x64-windows failed with BUILD_FAILED. If expected, add qtdeclarative:x64-windows=fail to C:\a\1\s\scripts\azure-pipelines/../ci.baseline.txt. REGRESSION: qtdeclarative:x64-windows failed with BUILD_FAILED. If expected, add qtdeclarative:x64-windows=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. REGRESSION: qtdeclarative:x64-windows failed with BUILD_FAILED. If expected, add qtdeclarative:x64-windows=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. REGRESSION: qtdeclarative:x64-windows failed with BUILD_FAILED. If expected, add qtdeclarative:x64-windows=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. REGRESSION: qtdeclarative:x64-windows failed with BUILD_FAILED. If expected, add qtdeclarative:x64-windows=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. REGRESSION: qtdeclarative:x64-windows failed with BUILD_FAILED. If expected, add qtdeclarative:x64-windows=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. REGRESSION: qtdeclarative:x64-windows failed with BUILD_FAILED. If expected, add qtdeclarative:x64-windows=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. This is a reporting change: The new world order also includes host build failures which is why it's duplicated. See also https://github.com/microsoft/vcpkg/issues/23714 See also https://github.com/microsoft/vcpkg/issues/23490 I'm nervous about baslining this because it seems most of the qt world is built on top of this port I filed https://github.com/microsoft/vcpkg/issues/23824 about this and @Neumann-A indicated this should be fixed by https://github.com/microsoft/vcpkg/pull/23755 REGRESSION: nettle:x64-uwp. If expected, add nettle:x64-uwp=fail to .\scripts\ci.baseline.txt. REGRESSION: nettle:x64-windows-static-md. If expected, add nettle:x64-windows-static-md=fail to .\scripts\ci.baseline.txt. REGRESSION: nettle:x64-uwp failed with BUILD_FAILED. If expected, add nettle:x64-uwp=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. REGRESSION: nettle:x64-windows-static-md failed with POST_BUILD_CHECKS_FAILED. If expected, add nettle:x64-windows-static-md=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. Didn't analyze, probably fixed by https://github.com/microsoft/vcpkg/pull/23519 ? REGRESSION: libgpg-error:x64-uwp. If expected, add libgpg-error:x64-uwp=fail to .\scripts\ci.baseline.txt. REGRESSION: libgpg-error:x64-uwp failed with BUILD_FAILED. If expected, add libgpg-error:x64-uwp=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. This was broken by VS2022 update: ``` C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Microsoft\VisualStudio\v17.0\AppxPackage\Microsoft.AppXPackage.Targets(892,25): error MSB4086: A numeric comparison was attempted on "$(TargetPlatformMinVersion)" that evaluates to "" instead of a number, in condition "'$(TargetPlatformMinVersion)' >= '10.0.17200.0'". [C:\Dev\vcpkg\buildtrees\libgpg-error\x64-uwp-rel\error-1.42-2324ddbc71.clean\SMP\libgpg-error_winrt.vcxproj] ``` REGRESSION: libmikmod:x64-osx. If expected, add libmikmod:x64-osx=fail to .\scripts\ci.baseline.txt. REGRESSION: libmikmod:x64-osx failed with BUILD_FAILED. If expected, add libmikmod:x64-osx=fail to /Users/vagrant/Data/work/2/s/scripts/azure-pipelines/../ci.baseline.txt. Broken between [2022-03-16](https://dev.azure.com/vcpkg/public/_build/results?buildId=68947) and [2022-03-18](https://dev.azure.com/vcpkg/public/_build/results?buildId=69051). Unfortunately I don't see obvious reasons why. Nothing else depends on this and nobody has noticed in 2 weeks, so I'm baslining it for now. (Will investigate shortly...) ## Only broken in tool update: REGRESSION: mesa:x64-windows failed with BUILD_FAILED. If expected, add mesa:x64-windows=fail to C:\a\2\s\scripts\azure-pipelines/../ci.baseline.txt. ``` -- Downloading https://gitlab.freedesktop.org/mesa/mesa/-/archive/mesa-21.2.5/mesa-mesa-21.2.5.tar.gz -> mesa-mesa-mesa-21.2.5-1.tar.gz... -- Extracting source /Users/vagrant/Data/downloads/mesa-mesa-mesa-21.2.5-1.tar.gz -- Applying patch swravx512-post-static-link.patch -- Applying patch swr-msvc-2.patch -- Applying patch swr-llvm13.patch -- Applying patch radv-msvc-llvm13-2.patch -- Applying patch d3d10sw.patch -- Using source at /Users/vagrant/Data/buildtrees/mesa/src/esa-21.2.5-2df234d2b1.clean Traceback (most recent call last): File "<string>", line 1, in <module> ModuleNotFoundError: No module named 'mako' CMake Error at ports/mesa/portfile.cmake:85 (message): Python package 'mako' needs to be installed for port 'mesa'. Complete list of required python packages: setuptools;mako Call Stack (most recent call first): ports/mesa/portfile.cmake:91 (vcpkg_get_python_package) scripts/ports.cmake:145 (include) ``` Looks like this is being tracked by https://github.com/microsoft/vcpkg/pull/23089 ; perhaps that we don't have as aggressive a recycling strategy for macos boxes as we do for the others has let different machines give different results? ## Only broken without tool update: REGRESSION: chromium-base:x64-osx. If expected, add chromium-base:x64-osx=fail to .\scripts\ci.baseline.txt. This one has been constantly flaky; I baselined it. REGRESSION: libxml2:x64-osx. If expected, add libxml2:x64-osx=fail to .\scripts\ci.baseline.txt. This port uses vcpkg_from_git and the upstream server was down during the build. * Restore chartdir to the baseline, I thought https://github.com/microsoft/vcpkg/pull/23732 had been merged.
2022-03-29 04:17:35 +08:00
& "./vcpkg$executableExtension" ci "--triplet=$Triplet" --dry-run "--ci-baseline=$PSScriptRoot/../ci.baseline.txt" @commonArgs --no-binarycaching "--output-hashes=$parentHashesFile"
Write-Host "Running CI using parent hashes"
& git reset --hard HEAD
}
# The vcpkg.cmake toolchain file is not part of ABI hashing,
# but changes must trigger at least some testing.
Copy-Item "scripts/buildsystems/vcpkg.cmake" -Destination "scripts/test_ports/cmake"
Copy-Item "scripts/buildsystems/vcpkg.cmake" -Destination "scripts/test_ports/cmake-user"
& "./vcpkg$executableExtension" ci "--triplet=$Triplet" --x-xunit=$xmlFile --failure-logs=$failureLogs "--ci-baseline=$PSScriptRoot/../ci.baseline.txt" @commonArgs @cachingArgs @parentHashes @skipFailuresArg
$failureLogsEmpty = (-Not (Test-Path $failureLogs) -Or ((Get-ChildItem $failureLogs).count -eq 0))
Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$failureLogsEmpty"
if ($LASTEXITCODE -ne 0)
{
throw "vcpkg ci failed"
}