[vcpkg] Commonize pipelines build powershell and don't use binary caching in CI (rather than PR) builds (#12082)

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -32,28 +32,16 @@ jobs:
       VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
-  - powershell: |
-      $env:VCPKG_DOWNLOADS = '/mnt/vcpkg-ci/downloads'
-      $commonArgs = @(
-        '--binarycaching',
-        '--x-binarysource=clear;files,/archives,upload',
-        '--x-buildtrees-root=/mnt/vcpkg-ci/buildtrees',
-        '--x-install-root=/mnt/vcpkg-ci/install',
-        '--x-packages-root=/mnt/vcpkg-ci/packages'
-      )
-
-      ./vcpkg x-ci-clean @commonArgs
-      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
-      mkdir $(System.ArtifactsDirectory)/xml-results
-      $env:HOME = '/home/agent'
-      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList @commonArgs
-    displayName: '** Test Modified Ports **'
   - task: PowerShell@2
-    displayName: 'Analyze results and prepare test logs'
+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
-      filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'
+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "/archives/fail" -triplets "x64-linux" -errorOnRegression'
+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot /mnt/vcpkg-ci -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-linux.xml'
+    condition: always()
   - bash: |
       df -h
     displayName: 'Report on Disk Space After Build'

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -36,27 +36,16 @@ jobs:
       VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
-  - powershell: |
-      $env:VCPKG_DOWNLOADS = '/Users/vagrant/Data/downloads'
-      $commonArgs = @(
-        '--binarycaching',
-        '--x-binarysource=clear;files,/Users/vagrant/Data/archives,upload',
-        '--x-buildtrees-root=/Users/vagrant/Data/buildtrees',
-        '--x-install-root=/Users/vagrant/Data/installed',
-        '--x-packages-root=/Users/vagrant/Data/packages'
-      )
-
-      ./vcpkg x-ci-clean @commonArgs
-      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
-      mkdir $(System.ArtifactsDirectory)/xml-results
-      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList @commonArgs
-    displayName: '** Test Modified Ports **'
   - task: PowerShell@2
-    displayName: 'Analyze results and prepare test logs'
+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
-      filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'
+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir /Users/vagrant/Data/archives/fail -triplets "x64-osx" -errorOnRegression'
+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot /Users/vagrant/Data/archives -WorkingRoot /Users/vagrant/Data -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-osx.xml'
+    condition: always()
   - bash: |
       df -h
     displayName: 'Report on Disk Space After Build'

scripts/azure-pipelines/test-modified-ports.ps1

@@ -0,0 +1,102 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+<#
+.SYNOPSIS
+Runs the 'Test Modified Ports' part of the vcpkg CI system for all platforms.
+
+.PARAMETER Triplet
+The triplet to test.
+
+.PARAMETER ArchivesRoot
+The location where the binary caching archives are stored. Shared across runs of this script.
+
+.PARAMETER WorkingRoot
+The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories.
+
+.PARAMETER ArtifactStagingDirectory
+The Azure Pipelines artifact staging directory. If not supplied, defaults to the current directory.
+
+.PARAMETER ArtifactsDirectory
+The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.
+
+.PARAMETER BuildReason
+The reason Azure Pipelines is running this script (controls whether Binary Caching is used). If not
+supplied, binary caching will be used.
+#>
+
+[CmdletBinding()]
+Param(
+    [Parameter(Mandatory=$true)]
+    [ValidateNotNullOrEmpty()]
+    [string]$Triplet,
+    [Parameter(Mandatory=$true)]
+    [ValidateNotNullOrEmpty()]
+    $ArchivesRoot,
+    [Parameter(Mandatory=$true)]
+    [ValidateNotNullOrEmpty()]
+    $WorkingRoot,
+    [ValidateNotNullOrEmpty()]
+    $ArtifactStagingDirectory = '.',
+    [ValidateNotNullOrEmpty()]
+    $ArtifactsDirectory = '.',
+    $BuildReason = $null
+)
+
+if (-Not (Test-Path "triplets/$Triplet.cmake")) {
+    Write-Error "Incorrect triplet '$Triplet', please supply a valid triplet."
+}
+
+$env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads'
+$buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
+$installRoot = Join-Path $WorkingRoot 'installed'
+$packagesRoot = Join-Path $WorkingRoot 'packages'
+$commonArgs = @(
+    "--x-buildtrees-root=$buildtreesRoot",
+    "--x-install-root=$installRoot",
+    "--x-packages-root=$packagesRoot"
+)
+
+$binaryCaching = $false
+if ([string]::IsNullOrWhiteSpace($BuildReason)) {
+    Write-Host 'Build reason not specified, defaulting to using binary caching.'
+    $binaryCaching = $true
+} elseif ($BuildReason -eq 'PullRequest') {
+    Write-Host 'Build reason was Pull Request, using binary caching.'
+    $binaryCaching = $true
+}
+
+if ($binaryCaching) {
+    $commonArgs += @(
+        '--binarycaching',
+        "--x-binarysource=clear;files,$ArchivesRoot,upload"
+    )
+}
+
+if ($Triplet -eq 'x64-linux') {
+    $env:HOME = '/home/agent'
+    $executableExtension = [string]::Empty
+} elseif ($Triplet -eq 'x64-osx') {
+    $executableExtension = [string]::Empty
+} else {
+    $executableExtension = '.exe'
+}
+
+$xmlResults = Join-Path $ArtifactsDirectory 'xml-results'
+mkdir $xmlResults
+$xmlFile = Join-Path $xmlResults "$Triplet.xml"
+
+& "./vcpkg$executableExtension" x-ci-clean @commonArgs
+$skipList = . "$PSScriptRoot/generate-skip-list.ps1" -Triplet $Triplet -BaselineFile "$PSScriptRoot/../ci.baseline.txt"
+
+# WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.
+# Install them so the CI succeeds:
+if ($Triplet -in @('x64-uwp', 'arm64-windows', 'arm-uwp')) {
+    .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs
+}
+
+& "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList @commonArgs
+& "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults -outputDir $ArtifactStagingDirectory `
+    -failureLogDir (Join-Path $ArchivesRoot 'fail') -triplets $Triplet -errorOnRegression `
+    -baselineFile .\scripts\ci.baseline.txt

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -38,33 +38,16 @@ jobs:
         ninja.exe -C build.x86.debug
         build.x86.debug\vcpkg-test.exe
       failOnStderr: true
-  - powershell: |
-      $env:VCPKG_DOWNLOADS='D:\downloads'
-      $commonArgs = @(
-        '--binarycaching',
-        '--x-binarysource=clear;files,W:\,upload',
-        '--x-buildtrees-root=D:\buildtrees',
-        '--x-install-root=D:\install',
-        '--x-packages-root=D:\packages'
-      )
-
-      .\vcpkg.exe x-ci-clean @commonArgs
-      $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt
-      mkdir $(System.ArtifactsDirectory)\xml-results
-      # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.
-      # Install them so the CI succeeds:
-      if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) {
-          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs
-      }
-
-      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList @commonArgs
-    displayName: '** Test Modified Ports **'
   - task: PowerShell@2
-    displayName: 'Analyze results and prepare test logs'
+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
-      filePath: 'scripts/azure-pipelines/analyze-test-results.ps1'
+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-baselineFile .\scripts\ci.baseline.txt -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "W:\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"'
+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot D:\ -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/${{ parameters.triplet }}.xml'
+    condition: always()
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: ${{ parameters.triplet }} port build failure logs'
     inputs:

toolsrc/src/vcpkg/commands.ci.cpp

@@ -391,8 +391,14 @@ namespace vcpkg::Commands::CI
             System::print2(System::Color::warning, "Warning: Running ci without binary caching!\n");
         }
 
-        auto binaryprovider =
+        std::unique_ptr<IBinaryProvider> binaryproviderStorage;
-            create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);
+        if (args.binary_caching_enabled())
+        {
+            binaryproviderStorage =
+                create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);
+        }
+
+        IBinaryProvider& binaryprovider = binaryproviderStorage ? *binaryproviderStorage : null_binary_provider();
 
         const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
 
@@ -458,14 +464,13 @@ namespace vcpkg::Commands::CI
                 return FullPackageSpec{spec, std::move(default_features)};
             });
 
-            auto split_specs =
+            auto split_specs = find_unknown_ports_for_ci(paths,
-                find_unknown_ports_for_ci(paths,
+                                                         exclusions_set,
-                                          exclusions_set,
+                                                         provider,
-                                          provider,
+                                                         var_provider,
-                                          var_provider,
+                                                         all_default_full_specs,
-                                          all_default_full_specs,
+                                                         purge_tombstones,
-                                          purge_tombstones,
+                                                         binaryprovider);
-                                          args.binary_caching_enabled() ? *binaryprovider : null_binary_provider());
             PortFileProvider::MapPortFileProvider new_default_provider(split_specs->default_feature_provider);
 
             Dependencies::CreateInstallPlanOptions serialize_options;
@@ -510,7 +515,7 @@ namespace vcpkg::Commands::CI
             {
                 auto collection_timer = Chrono::ElapsedTimer::create_started();
                 auto summary = Install::perform(
-                    action_plan, Install::KeepGoing::YES, paths, status_db, *binaryprovider, var_provider);
+                    action_plan, Install::KeepGoing::YES, paths, status_db, binaryprovider, var_provider);
                 auto collection_time_elapsed = collection_timer.elapsed();
 
                 // Adding results for ports that were built or pulled from an archive