[vcpkg] Add vcpkg export to E2E tests. Enable E2E tests on all platforms. (#12198)

* [vcpkg] Add `vcpkg export` to E2E tests. Enable E2E tests on MacOS.

* [vcpkg] Fix export --raw --output-dir=/path/ by changing directory to new export root

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>

scripts/azure-pipelines/azure-pipelines.yml

@@ -23,6 +23,9 @@ stages:
   displayName: Check the formatting of port manifests
   pool: $(windows-pool)
   dependsOn: []
+  variables:
+  - name: VCPKG_DOWNLOADS
+    value: D:\downloads
   jobs:
   - job:
     workspace:
@@ -32,7 +35,7 @@ stages:
       displayName: 'Check port manifest Formatting'
       inputs:
         filePath: 'scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1'
-        arguments: '-Root . -Downloads D:\Downloads'
+        arguments: '-Root .'
 - stage: run_port_ci
   displayName: 'Run the Port CI'
   dependsOn:

scripts/azure-pipelines/end-to-end-tests.ps1

@@ -158,8 +158,13 @@ Require-FileNotExists "$installRoot/$Triplet/include/rapidjson/rapidjson.h"
 Require-FileNotExists "$buildtreesRoot/rapidjson/src"
 Require-FileExists "$TestingRoot/packages.config"
 
-& $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"
-Throw-IfFailed
+if ($IsLinux -or $IsMacOS) {
+    mono $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"
+    Throw-IfFailed
+} else {
+    & $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"
+    Throw-IfFailed
+}
 
 Remove-Item -Recurse -Force $NuGetRoot -ErrorAction SilentlyContinue
 mkdir $NuGetRoot
@@ -177,3 +182,15 @@ Require-FileExists "$buildtreesRoot/tinyxml/src"
 if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 1) {
     throw "In '$CurrentTest': did not create exactly 1 NuGet package"
 }
+
+# Test export
+$args = $commonArgs + @("export","rapidjson","tinyxml","--nuget","--nuget-id=vcpkg-export","--nuget-version=1.0.0","--output=vcpkg-export-output","--raw","--zip","--output-dir=$TestingRoot")
+$CurrentTest = "./vcpkg $($args -join ' ')"
+Write-Host $CurrentTest
+Require-FileNotExists "$TestingRoot/vcpkg-export-output"
+Require-FileNotExists "$TestingRoot/vcpkg-export.1.0.0.nupkg"
+Require-FileNotExists "$TestingRoot/vcpkg-export-output.zip"
+./vcpkg @args
+Require-FileExists "$TestingRoot/vcpkg-export-output"
+Require-FileExists "$TestingRoot/vcpkg-export.1.0.0.nupkg"
+Require-FileExists "$TestingRoot/vcpkg-export-output.zip"

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -9,6 +9,12 @@ jobs:
   workspace:
     clean: resources
   timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: /mnt/vcpkg-ci
+  - name: VCPKG_DOWNLOADS
+    value: /mnt/vcpkg-ci/downloads
+
   steps:
   - bash: df -h
     displayName: 'Report on Disk Space'
@@ -19,17 +25,15 @@ jobs:
     displayName: 'Create /home/agent'
     # Note: /mnt is the Azure machines' temporary disk.
   - bash: |
-      sudo mkdir /mnt/vcpkg-ci -m=777
-      sudo mkdir /mnt/vcpkg-ci/downloads -m=777
+      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
       exit 0
-    displayName: 'Create /mnt/vcpkg-ci/downloads'
+    displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'
   - task: Bash@3
     displayName: 'Build vcpkg'
     inputs:
       filePath: bootstrap-vcpkg.sh
       arguments: "-buildTests"
-    env:
-      VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
   - task: PowerShell@2
@@ -37,7 +41,7 @@ jobs:
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot /mnt/vcpkg-ci -ArtifactsDirectory $(System.ArtifactsDirectory)'
+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)'
   - bash: |
       df -h
     displayName: 'Report on Disk Space After Build'

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -10,6 +10,12 @@ jobs:
   workspace:
     clean: resources
   timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: /Users/vagrant/Data
+  - name: VCPKG_DOWNLOADS
+    value: /Users/vagrant/Data/downloads
+
   steps:
   - bash: |
       df -h
@@ -22,28 +28,32 @@ jobs:
       brew list libtool || brew install libtool
       brew list bison || brew install bison
       brew list gfortran || brew cask install gfortran
+      brew list mono || brew install mono
       brew list yasm || brew install yasm
     displayName: 'Install brew dependencies'
   - bash: |
-      sudo mkdir /Users/vagrant/Data/downloads || 0
-      sudo chmod 777 /Users/vagrant/Data/downloads || 0
+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0
+      sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0
       exit 0
-    displayName: 'Create /Users/vagrant/Data/downloads'
+    displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'
   - task: Bash@3
     displayName: 'Build vcpkg'
     inputs:
       filePath: bootstrap-vcpkg.sh
       arguments: '-buildTests'
-    env:
-      VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
+  - task: PowerShell@2
+    displayName: 'Run vcpkg end-to-end tests'
+    inputs:
+      filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1'
+      arguments: '-Triplet x64-osx -WorkingRoot ${{ variables.WORKING_ROOT }}'
   - task: PowerShell@2
     displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot /Users/vagrant/Data/archives -WorkingRoot /Users/vagrant/Data -ArtifactsDirectory $(System.ArtifactsDirectory)'
+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)'
   - bash: |
       df -h
     displayName: 'Report on Disk Space After Build'

scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1

@@ -3,8 +3,6 @@ Param(
     [Parameter(Mandatory=$True)]
     [string]$Root,
     [Parameter()]
-    [string]$DownloadsDirectory,
-    [Parameter()]
     [switch]$IgnoreErrors # allows one to just format
 )
 
@@ -16,11 +14,6 @@ if (-not (Test-Path "$Root/.vcpkg-root"))
     throw
 }
 
-if (-not [string]::IsNullOrEmpty($DownloadsDirectory))
-{
-    $env:VCPKG_DOWNLOADS = $DownloadsDirectory
-}
-
 if (-not (Test-Path "$Root/vcpkg.exe"))
 {
     & "$Root/bootstrap-vcpkg.bat"

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -9,6 +9,11 @@ jobs:
   workspace:
     clean: resources
   timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: D:\
+  - name: VCPKG_DOWNLOADS
+    value: D:\downloads
 
   steps:
   - task: PowerShell@2
@@ -21,35 +26,35 @@ jobs:
     inputs:
       filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
   # Note: D: is the Azure machines' temporary disk.
-  - task: CmdLine@2
+  - script: .\bootstrap-vcpkg.bat
     displayName: 'Build vcpkg'
-    inputs:
-      script: |
-        set VCPKG_DOWNLOADS=D:\downloads
-        .\bootstrap-vcpkg.bat
   - task: CmdLine@2
     displayName: "Build vcpkg with CMake and Run Tests"
     condition: eq('${{ parameters.triplet }}', 'x86-windows')
     inputs:
       script: |
         :: TRANSITION, get these tools on the VMs next time we roll them
-        set VCPKG_DOWNLOADS=D:\downloads
         .\vcpkg.exe fetch cmake
         .\vcpkg.exe fetch ninja
-        set PATH=D:\downloads\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;D:\downloads\tools\ninja-1.10.0-windows;%PATH%
+        set PATH=${{ variables.VCPKG_DOWNLOADS }}\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;${{ variables.VCPKG_DOWNLOADS }}\tools\ninja-1.10.0-windows;%PATH%
         call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86
         rmdir /s /q build.x86.debug > nul 2> nul
         cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=ON -B build.x86.debug -S toolsrc
         ninja.exe -C build.x86.debug
         build.x86.debug\vcpkg-test.exe
-        powershell.exe -NoProfile -ExecutionPolicy Bypass "scripts\azure-pipelines\end-to-end-tests.ps1 -WorkingRoot \"%cd%\testing\" -triplet x86-windows"
       failOnStderr: true
+  - task: PowerShell@2
+    displayName: 'Run vcpkg end-to-end tests'
+    condition: eq('${{ parameters.triplet }}', 'x86-windows')
+    inputs:
+      filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1'
+      arguments: '-Triplet ${{ parameters.triplet }} -WorkingRoot ${{ variables.WORKING_ROOT }}'
   - task: PowerShell@2
     displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot D:\ -ArtifactsDirectory $(System.ArtifactsDirectory)'
+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)'
   - task: PowerShell@2
     displayName: 'Report on Disk Space After Build'
     condition: always()
@@ -68,7 +73,6 @@ jobs:
     inputs:
       targetType: inline
       script: |
-        $env:VCPKG_DOWNLOADS = "D:\downloads"
         ./vcpkg.exe fetch python3
         & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\
   - task: PublishBuildArtifacts@1

toolsrc/include/vcpkg/base/system.process.h

@@ -27,6 +27,12 @@ namespace vcpkg::System
     {
         CmdLineBuilder& path_arg(const fs::path& p) { return string_arg(p.u8string()); }
         CmdLineBuilder& string_arg(StringView s);
+        CmdLineBuilder& ampersand()
+        {
+            buf.push_back('&');
+            buf.push_back('&');
+            return *this;
+        }
         std::string extract() noexcept { return std::move(buf); }
 
         operator ZStringView() const { return buf; }

toolsrc/src/vcpkg/base/files.cpp

@@ -1041,7 +1041,7 @@ namespace vcpkg::Files
                 return res;
             }
 
-            System::printf("Waiting to take filesystem lock on %s...\n", path.u8string());
+            Debug::print("Waiting to take filesystem lock on ", path.u8string(), "...\n");
             auto wait = std::chrono::milliseconds(100);
             // waits, at most, a second and a half.
             while (wait < std::chrono::milliseconds(1000))

toolsrc/src/vcpkg/export.cpp

@@ -150,13 +150,19 @@ namespace vcpkg::Export
         fs.write_contents(nuspec_file_path, nuspec_file_content, VCPKG_LINE_INFO);
 
         // -NoDefaultExcludes is needed for ".vcpkg-root"
-        const auto cmd_line = Strings::format(R"("%s" pack -OutputDirectory "%s" "%s" -NoDefaultExcludes)",
-                                              nuget_exe.u8string(),
-                                              output_dir.u8string(),
-                                              nuspec_file_path.u8string());
+        System::CmdLineBuilder cmd;
+#ifndef _WIN32
+        cmd.path_arg(paths.get_tool_exe(Tools::MONO));
+#endif
+        cmd.path_arg(nuget_exe)
+            .string_arg("pack")
+            .path_arg(nuspec_file_path)
+            .string_arg("-OutputDirectory")
+            .path_arg(output_dir)
+            .string_arg("-NoDefaultExcludes");
 
         const int exit_code =
-            System::cmd_execute_and_capture_output(cmd_line, System::get_clean_environment()).exit_code;
+            System::cmd_execute_and_capture_output(cmd.extract(), System::get_clean_environment()).exit_code;
         Checks::check_exit(VCPKG_LINE_INFO, exit_code == 0, "Error: NuGet package creation failed");
 
         const fs::path output_path = output_dir / (nuget_id + "." + nuget_version + ".nupkg");
@@ -206,14 +212,26 @@ namespace vcpkg::Export
             Strings::format("%s.%s", exported_dir_filename, format.extension());
         const fs::path exported_archive_path = (output_dir / exported_archive_filename);
 
-        // -NoDefaultExcludes is needed for ".vcpkg-root"
-        const auto cmd_line = Strings::format(R"("%s" -E tar "cf" "%s" --format=%s -- "%s")",
-                                              cmake_exe.u8string(),
-                                              exported_archive_path.u8string(),
-                                              format.cmake_option(),
-                                              raw_exported_dir.u8string());
+        System::CmdLineBuilder cmd;
+        cmd.string_arg("cd").path_arg(raw_exported_dir.parent_path());
+        cmd.ampersand();
+        cmd.path_arg(cmake_exe)
+            .string_arg("-E")
+            .string_arg("tar")
+            .string_arg("cf")
+            .path_arg(exported_archive_path)
+            .string_arg(Strings::concat("--format=", format.cmake_option()))
+            .string_arg("--")
+            .path_arg(raw_exported_dir);
 
-        const int exit_code = System::cmd_execute_clean(cmd_line);
+        auto cmdline = cmd.extract();
+#ifdef WIN32
+        // Invoke through `cmd` to support `&&`
+        cmdline.insert(0, "cmd /c \"");
+        cmdline.push_back('"');
+#endif
+
+        const int exit_code = System::cmd_execute_clean(cmdline);
         Checks::check_exit(
             VCPKG_LINE_INFO, exit_code == 0, "Error: %s creation failed", exported_archive_path.generic_string());
         return exported_archive_path;
@@ -262,6 +280,7 @@ namespace vcpkg::Export
         bool all_installed = false;
 
         Optional<std::string> maybe_output;
+        fs::path output_dir;
 
         Optional<std::string> maybe_nuget_id;
         Optional<std::string> maybe_nuget_version;
@@ -273,6 +292,7 @@ namespace vcpkg::Export
     };
 
     static constexpr StringLiteral OPTION_OUTPUT = "output";
+    static constexpr StringLiteral OPTION_OUTPUT_DIR = "output-dir";
     static constexpr StringLiteral OPTION_DRY_RUN = "dry-run";
     static constexpr StringLiteral OPTION_RAW = "raw";
     static constexpr StringLiteral OPTION_NUGET = "nuget";
@@ -314,8 +334,9 @@ namespace vcpkg::Export
         {OPTION_ALL_INSTALLED, "Export all installed packages"},
     }};
 
-    static constexpr std::array<CommandSetting, 15> EXPORT_SETTINGS = {{
+    static constexpr std::array<CommandSetting, 16> EXPORT_SETTINGS = {{
         {OPTION_OUTPUT, "Specify the output name (used to construct filename)"},
+        {OPTION_OUTPUT_DIR, "Specify the output directory for produced artifacts"},
         {OPTION_NUGET_ID, "Specify the id for the exported NuGet package (overrides --output)"},
         {OPTION_NUGET_VERSION, "Specify the version for the exported NuGet package"},
         {OPTION_IFW_REPOSITORY_URL, "Specify the remote repository URL for the online installer"},
@@ -342,7 +363,8 @@ namespace vcpkg::Export
         nullptr,
     };
 
-    static ExportArguments handle_export_command_arguments(const VcpkgCmdArguments& args,
+    static ExportArguments handle_export_command_arguments(const VcpkgPaths& paths,
+                                                           const VcpkgCmdArguments& args,
                                                            Triplet default_triplet,
                                                            const StatusParagraphs& status_db)
     {
@@ -361,6 +383,15 @@ namespace vcpkg::Export
         ret.prefab_options.enable_maven = options.switches.find(OPTION_PREFAB_ENABLE_MAVEN) != options.switches.cend();
         ret.prefab_options.enable_debug = options.switches.find(OPTION_PREFAB_ENABLE_DEBUG) != options.switches.cend();
         ret.maybe_output = maybe_lookup(options.settings, OPTION_OUTPUT);
+        auto maybe_output_dir = maybe_lookup(options.settings, OPTION_OUTPUT_DIR);
+        if (auto output_dir = maybe_output_dir.get())
+        {
+            ret.output_dir = Files::combine(paths.original_cwd, fs::u8path(*output_dir));
+        }
+        else
+        {
+            ret.output_dir = paths.root;
+        }
         ret.all_installed = options.switches.find(OPTION_ALL_INSTALLED) != options.switches.end();
 
         if (ret.all_installed)
@@ -479,8 +510,7 @@ namespace vcpkg::Export
                                         const VcpkgPaths& paths)
     {
         Files::Filesystem& fs = paths.get_filesystem();
-        const fs::path export_to_path = paths.root;
-        const fs::path raw_exported_dir_path = export_to_path / export_id;
+        const fs::path raw_exported_dir_path = opts.output_dir / export_id;
         fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO);
 
         // TODO: error handling
@@ -538,7 +568,7 @@ namespace vcpkg::Export
             const std::string nuget_id = opts.maybe_nuget_id.value_or(raw_exported_dir_path.filename().string());
             const std::string nuget_version = opts.maybe_nuget_version.value_or("1.0.0");
             const fs::path output_path =
-                do_nuget_export(paths, nuget_id, nuget_version, raw_exported_dir_path, export_to_path);
+                do_nuget_export(paths, nuget_id, nuget_version, raw_exported_dir_path, opts.output_dir);
             System::print2(System::Color::success, "NuGet package exported at: ", output_path.u8string(), "\n");
 
             System::printf(R"(
@@ -554,7 +584,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
         {
             System::print2("Creating zip archive...\n");
             const fs::path output_path =
-                do_archive_export(paths, raw_exported_dir_path, export_to_path, ArchiveFormatC::ZIP);
+                do_archive_export(paths, raw_exported_dir_path, opts.output_dir, ArchiveFormatC::ZIP);
             System::print2(System::Color::success, "Zip archive exported at: ", output_path.u8string(), "\n");
             print_next_step_info("[...]");
         }
@@ -563,7 +593,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
         {
             System::print2("Creating 7zip archive...\n");
             const fs::path output_path =
-                do_archive_export(paths, raw_exported_dir_path, export_to_path, ArchiveFormatC::SEVEN_ZIP);
+                do_archive_export(paths, raw_exported_dir_path, opts.output_dir, ArchiveFormatC::SEVEN_ZIP);
             System::print2(System::Color::success, "7zip archive exported at: ", output_path.u8string(), "\n");
             print_next_step_info("[...]");
         }
@@ -584,7 +614,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
                 "may use export in classic mode by running vcpkg outside of a manifest-based project.");
         }
         const StatusParagraphs status_db = database_load_check(paths);
-        const auto opts = handle_export_command_arguments(args, default_triplet, status_db);
+        const auto opts = handle_export_command_arguments(paths, args, default_triplet, status_db);
         for (auto&& spec : opts.specs)
             Input::check_triplet(spec.triplet(), paths);
 

toolsrc/src/vcpkg/tools.cpp

@@ -83,8 +83,12 @@ namespace vcpkg
         const bool has_tool_entry = std::regex_search(XML.cbegin(), XML.cend(), match_tool_entry, tool_regex);
         if (!has_tool_entry)
         {
-            return Strings::format(
-                "Could not find entry for tool %s in %s for os=%s", tool, XML_PATH.u8string(), OS_STRING);
+            return Strings::format("Could not automatically acquire %s because there is no entry in %s for os=%s. You "
+                                   "may be able to install %s via your system package manager.",
+                                   tool,
+                                   XML_PATH.u8string(),
+                                   OS_STRING,
+                                   tool);
         }
 
         const std::string tool_data =