diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 80c547633e..cda29f7e6a 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -23,6 +23,9 @@ stages: displayName: Check the formatting of port manifests pool: $(windows-pool) dependsOn: [] + variables: + - name: VCPKG_DOWNLOADS + value: D:\downloads jobs: - job: workspace: @@ -32,7 +35,7 @@ stages: displayName: 'Check port manifest Formatting' inputs: filePath: 'scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1' - arguments: '-Root . -Downloads D:\Downloads' + arguments: '-Root .' - stage: run_port_ci displayName: 'Run the Port CI' dependsOn: diff --git a/scripts/azure-pipelines/end-to-end-tests.ps1 b/scripts/azure-pipelines/end-to-end-tests.ps1 index b0e4476575..43ec7aa73e 100644 --- a/scripts/azure-pipelines/end-to-end-tests.ps1 +++ b/scripts/azure-pipelines/end-to-end-tests.ps1 @@ -158,8 +158,13 @@ Require-FileNotExists "$installRoot/$Triplet/include/rapidjson/rapidjson.h" Require-FileNotExists "$buildtreesRoot/rapidjson/src" Require-FileExists "$TestingRoot/packages.config" -& $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot" -Throw-IfFailed +if ($IsLinux -or $IsMacOS) { + mono $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot" + Throw-IfFailed +} else { + & $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot" + Throw-IfFailed +} Remove-Item -Recurse -Force $NuGetRoot -ErrorAction SilentlyContinue mkdir $NuGetRoot @@ -177,3 +182,15 @@ Require-FileExists "$buildtreesRoot/tinyxml/src" if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 1) { throw "In '$CurrentTest': did not create exactly 1 NuGet package" } + +# Test export +$args = $commonArgs + @("export","rapidjson","tinyxml","--nuget","--nuget-id=vcpkg-export","--nuget-version=1.0.0","--output=vcpkg-export-output","--raw","--zip","--output-dir=$TestingRoot") +$CurrentTest = "./vcpkg $($args -join ' ')" +Write-Host $CurrentTest +Require-FileNotExists "$TestingRoot/vcpkg-export-output" +Require-FileNotExists "$TestingRoot/vcpkg-export.1.0.0.nupkg" +Require-FileNotExists "$TestingRoot/vcpkg-export-output.zip" +./vcpkg @args +Require-FileExists "$TestingRoot/vcpkg-export-output" +Require-FileExists "$TestingRoot/vcpkg-export.1.0.0.nupkg" +Require-FileExists "$TestingRoot/vcpkg-export-output.zip" diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index 4d4eada627..969348980b 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -9,6 +9,12 @@ jobs: workspace: clean: resources timeoutInMinutes: 1440 # 1 day + variables: + - name: WORKING_ROOT + value: /mnt/vcpkg-ci + - name: VCPKG_DOWNLOADS + value: /mnt/vcpkg-ci/downloads + steps: - bash: df -h displayName: 'Report on Disk Space' @@ -19,17 +25,15 @@ jobs: displayName: 'Create /home/agent' # Note: /mnt is the Azure machines' temporary disk. - bash: | - sudo mkdir /mnt/vcpkg-ci -m=777 - sudo mkdir /mnt/vcpkg-ci/downloads -m=777 + sudo mkdir ${{ variables.WORKING_ROOT }} -m=777 + sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777 exit 0 - displayName: 'Create /mnt/vcpkg-ci/downloads' + displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}' - task: Bash@3 displayName: 'Build vcpkg' inputs: filePath: bootstrap-vcpkg.sh arguments: "-buildTests" - env: - VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads' - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' - task: PowerShell@2 @@ -37,7 +41,7 @@ jobs: inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' - arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot /mnt/vcpkg-ci -ArtifactsDirectory $(System.ArtifactsDirectory)' + arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)' - bash: | df -h displayName: 'Report on Disk Space After Build' diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 3c9510bac5..6e48238a7e 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -10,6 +10,12 @@ jobs: workspace: clean: resources timeoutInMinutes: 1440 # 1 day + variables: + - name: WORKING_ROOT + value: /Users/vagrant/Data + - name: VCPKG_DOWNLOADS + value: /Users/vagrant/Data/downloads + steps: - bash: | df -h @@ -22,28 +28,32 @@ jobs: brew list libtool || brew install libtool brew list bison || brew install bison brew list gfortran || brew cask install gfortran + brew list mono || brew install mono brew list yasm || brew install yasm displayName: 'Install brew dependencies' - bash: | - sudo mkdir /Users/vagrant/Data/downloads || 0 - sudo chmod 777 /Users/vagrant/Data/downloads || 0 + sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0 + sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0 exit 0 - displayName: 'Create /Users/vagrant/Data/downloads' + displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}' - task: Bash@3 displayName: 'Build vcpkg' inputs: filePath: bootstrap-vcpkg.sh arguments: '-buildTests' - env: - VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads' - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' + - task: PowerShell@2 + displayName: 'Run vcpkg end-to-end tests' + inputs: + filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1' + arguments: '-Triplet x64-osx -WorkingRoot ${{ variables.WORKING_ROOT }}' - task: PowerShell@2 displayName: '*** Test Modified Ports and Prepare Test Logs ***' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' - arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot /Users/vagrant/Data/archives -WorkingRoot /Users/vagrant/Data -ArtifactsDirectory $(System.ArtifactsDirectory)' + arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)' - bash: | df -h displayName: 'Report on Disk Space After Build' diff --git a/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1 b/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1 index f4385c2b4e..6e97ba35c7 100644 --- a/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1 +++ b/scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1 @@ -3,8 +3,6 @@ Param( [Parameter(Mandatory=$True)] [string]$Root, [Parameter()] - [string]$DownloadsDirectory, - [Parameter()] [switch]$IgnoreErrors # allows one to just format ) @@ -16,11 +14,6 @@ if (-not (Test-Path "$Root/.vcpkg-root")) throw } -if (-not [string]::IsNullOrEmpty($DownloadsDirectory)) -{ - $env:VCPKG_DOWNLOADS = $DownloadsDirectory -} - if (-not (Test-Path "$Root/vcpkg.exe")) { & "$Root/bootstrap-vcpkg.bat" diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 5ec0b300c2..340138e124 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -9,6 +9,11 @@ jobs: workspace: clean: resources timeoutInMinutes: 1440 # 1 day + variables: + - name: WORKING_ROOT + value: D:\ + - name: VCPKG_DOWNLOADS + value: D:\downloads steps: - task: PowerShell@2 @@ -21,35 +26,35 @@ jobs: inputs: filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' # Note: D: is the Azure machines' temporary disk. - - task: CmdLine@2 + - script: .\bootstrap-vcpkg.bat displayName: 'Build vcpkg' - inputs: - script: | - set VCPKG_DOWNLOADS=D:\downloads - .\bootstrap-vcpkg.bat - task: CmdLine@2 displayName: "Build vcpkg with CMake and Run Tests" condition: eq('${{ parameters.triplet }}', 'x86-windows') inputs: script: | :: TRANSITION, get these tools on the VMs next time we roll them - set VCPKG_DOWNLOADS=D:\downloads .\vcpkg.exe fetch cmake .\vcpkg.exe fetch ninja - set PATH=D:\downloads\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;D:\downloads\tools\ninja-1.10.0-windows;%PATH% + set PATH=${{ variables.VCPKG_DOWNLOADS }}\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;${{ variables.VCPKG_DOWNLOADS }}\tools\ninja-1.10.0-windows;%PATH% call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86 rmdir /s /q build.x86.debug > nul 2> nul cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=ON -B build.x86.debug -S toolsrc ninja.exe -C build.x86.debug build.x86.debug\vcpkg-test.exe - powershell.exe -NoProfile -ExecutionPolicy Bypass "scripts\azure-pipelines\end-to-end-tests.ps1 -WorkingRoot \"%cd%\testing\" -triplet x86-windows" failOnStderr: true + - task: PowerShell@2 + displayName: 'Run vcpkg end-to-end tests' + condition: eq('${{ parameters.triplet }}', 'x86-windows') + inputs: + filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1' + arguments: '-Triplet ${{ parameters.triplet }} -WorkingRoot ${{ variables.WORKING_ROOT }}' - task: PowerShell@2 displayName: '*** Test Modified Ports and Prepare Test Logs ***' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' - arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot D:\ -ArtifactsDirectory $(System.ArtifactsDirectory)' + arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)' - task: PowerShell@2 displayName: 'Report on Disk Space After Build' condition: always() @@ -68,7 +73,6 @@ jobs: inputs: targetType: inline script: | - $env:VCPKG_DOWNLOADS = "D:\downloads" ./vcpkg.exe fetch python3 & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\ - task: PublishBuildArtifacts@1 diff --git a/toolsrc/include/vcpkg/base/system.process.h b/toolsrc/include/vcpkg/base/system.process.h index 91faa59854..b9184c05cd 100644 --- a/toolsrc/include/vcpkg/base/system.process.h +++ b/toolsrc/include/vcpkg/base/system.process.h @@ -27,6 +27,12 @@ namespace vcpkg::System { CmdLineBuilder& path_arg(const fs::path& p) { return string_arg(p.u8string()); } CmdLineBuilder& string_arg(StringView s); + CmdLineBuilder& ampersand() + { + buf.push_back('&'); + buf.push_back('&'); + return *this; + } std::string extract() noexcept { return std::move(buf); } operator ZStringView() const { return buf; } diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index f2684633ce..59931603b7 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -1041,7 +1041,7 @@ namespace vcpkg::Files return res; } - System::printf("Waiting to take filesystem lock on %s...\n", path.u8string()); + Debug::print("Waiting to take filesystem lock on ", path.u8string(), "...\n"); auto wait = std::chrono::milliseconds(100); // waits, at most, a second and a half. while (wait < std::chrono::milliseconds(1000)) diff --git a/toolsrc/src/vcpkg/export.cpp b/toolsrc/src/vcpkg/export.cpp index 10d79195f5..a124345a7f 100644 --- a/toolsrc/src/vcpkg/export.cpp +++ b/toolsrc/src/vcpkg/export.cpp @@ -150,13 +150,19 @@ namespace vcpkg::Export fs.write_contents(nuspec_file_path, nuspec_file_content, VCPKG_LINE_INFO); // -NoDefaultExcludes is needed for ".vcpkg-root" - const auto cmd_line = Strings::format(R"("%s" pack -OutputDirectory "%s" "%s" -NoDefaultExcludes)", - nuget_exe.u8string(), - output_dir.u8string(), - nuspec_file_path.u8string()); + System::CmdLineBuilder cmd; +#ifndef _WIN32 + cmd.path_arg(paths.get_tool_exe(Tools::MONO)); +#endif + cmd.path_arg(nuget_exe) + .string_arg("pack") + .path_arg(nuspec_file_path) + .string_arg("-OutputDirectory") + .path_arg(output_dir) + .string_arg("-NoDefaultExcludes"); const int exit_code = - System::cmd_execute_and_capture_output(cmd_line, System::get_clean_environment()).exit_code; + System::cmd_execute_and_capture_output(cmd.extract(), System::get_clean_environment()).exit_code; Checks::check_exit(VCPKG_LINE_INFO, exit_code == 0, "Error: NuGet package creation failed"); const fs::path output_path = output_dir / (nuget_id + "." + nuget_version + ".nupkg"); @@ -206,14 +212,26 @@ namespace vcpkg::Export Strings::format("%s.%s", exported_dir_filename, format.extension()); const fs::path exported_archive_path = (output_dir / exported_archive_filename); - // -NoDefaultExcludes is needed for ".vcpkg-root" - const auto cmd_line = Strings::format(R"("%s" -E tar "cf" "%s" --format=%s -- "%s")", - cmake_exe.u8string(), - exported_archive_path.u8string(), - format.cmake_option(), - raw_exported_dir.u8string()); + System::CmdLineBuilder cmd; + cmd.string_arg("cd").path_arg(raw_exported_dir.parent_path()); + cmd.ampersand(); + cmd.path_arg(cmake_exe) + .string_arg("-E") + .string_arg("tar") + .string_arg("cf") + .path_arg(exported_archive_path) + .string_arg(Strings::concat("--format=", format.cmake_option())) + .string_arg("--") + .path_arg(raw_exported_dir); - const int exit_code = System::cmd_execute_clean(cmd_line); + auto cmdline = cmd.extract(); +#ifdef WIN32 + // Invoke through `cmd` to support `&&` + cmdline.insert(0, "cmd /c \""); + cmdline.push_back('"'); +#endif + + const int exit_code = System::cmd_execute_clean(cmdline); Checks::check_exit( VCPKG_LINE_INFO, exit_code == 0, "Error: %s creation failed", exported_archive_path.generic_string()); return exported_archive_path; @@ -262,6 +280,7 @@ namespace vcpkg::Export bool all_installed = false; Optional maybe_output; + fs::path output_dir; Optional maybe_nuget_id; Optional maybe_nuget_version; @@ -273,6 +292,7 @@ namespace vcpkg::Export }; static constexpr StringLiteral OPTION_OUTPUT = "output"; + static constexpr StringLiteral OPTION_OUTPUT_DIR = "output-dir"; static constexpr StringLiteral OPTION_DRY_RUN = "dry-run"; static constexpr StringLiteral OPTION_RAW = "raw"; static constexpr StringLiteral OPTION_NUGET = "nuget"; @@ -314,8 +334,9 @@ namespace vcpkg::Export {OPTION_ALL_INSTALLED, "Export all installed packages"}, }}; - static constexpr std::array EXPORT_SETTINGS = {{ + static constexpr std::array EXPORT_SETTINGS = {{ {OPTION_OUTPUT, "Specify the output name (used to construct filename)"}, + {OPTION_OUTPUT_DIR, "Specify the output directory for produced artifacts"}, {OPTION_NUGET_ID, "Specify the id for the exported NuGet package (overrides --output)"}, {OPTION_NUGET_VERSION, "Specify the version for the exported NuGet package"}, {OPTION_IFW_REPOSITORY_URL, "Specify the remote repository URL for the online installer"}, @@ -342,7 +363,8 @@ namespace vcpkg::Export nullptr, }; - static ExportArguments handle_export_command_arguments(const VcpkgCmdArguments& args, + static ExportArguments handle_export_command_arguments(const VcpkgPaths& paths, + const VcpkgCmdArguments& args, Triplet default_triplet, const StatusParagraphs& status_db) { @@ -361,6 +383,15 @@ namespace vcpkg::Export ret.prefab_options.enable_maven = options.switches.find(OPTION_PREFAB_ENABLE_MAVEN) != options.switches.cend(); ret.prefab_options.enable_debug = options.switches.find(OPTION_PREFAB_ENABLE_DEBUG) != options.switches.cend(); ret.maybe_output = maybe_lookup(options.settings, OPTION_OUTPUT); + auto maybe_output_dir = maybe_lookup(options.settings, OPTION_OUTPUT_DIR); + if (auto output_dir = maybe_output_dir.get()) + { + ret.output_dir = Files::combine(paths.original_cwd, fs::u8path(*output_dir)); + } + else + { + ret.output_dir = paths.root; + } ret.all_installed = options.switches.find(OPTION_ALL_INSTALLED) != options.switches.end(); if (ret.all_installed) @@ -479,8 +510,7 @@ namespace vcpkg::Export const VcpkgPaths& paths) { Files::Filesystem& fs = paths.get_filesystem(); - const fs::path export_to_path = paths.root; - const fs::path raw_exported_dir_path = export_to_path / export_id; + const fs::path raw_exported_dir_path = opts.output_dir / export_id; fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO); // TODO: error handling @@ -538,7 +568,7 @@ namespace vcpkg::Export const std::string nuget_id = opts.maybe_nuget_id.value_or(raw_exported_dir_path.filename().string()); const std::string nuget_version = opts.maybe_nuget_version.value_or("1.0.0"); const fs::path output_path = - do_nuget_export(paths, nuget_id, nuget_version, raw_exported_dir_path, export_to_path); + do_nuget_export(paths, nuget_id, nuget_version, raw_exported_dir_path, opts.output_dir); System::print2(System::Color::success, "NuGet package exported at: ", output_path.u8string(), "\n"); System::printf(R"( @@ -554,7 +584,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console { System::print2("Creating zip archive...\n"); const fs::path output_path = - do_archive_export(paths, raw_exported_dir_path, export_to_path, ArchiveFormatC::ZIP); + do_archive_export(paths, raw_exported_dir_path, opts.output_dir, ArchiveFormatC::ZIP); System::print2(System::Color::success, "Zip archive exported at: ", output_path.u8string(), "\n"); print_next_step_info("[...]"); } @@ -563,7 +593,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console { System::print2("Creating 7zip archive...\n"); const fs::path output_path = - do_archive_export(paths, raw_exported_dir_path, export_to_path, ArchiveFormatC::SEVEN_ZIP); + do_archive_export(paths, raw_exported_dir_path, opts.output_dir, ArchiveFormatC::SEVEN_ZIP); System::print2(System::Color::success, "7zip archive exported at: ", output_path.u8string(), "\n"); print_next_step_info("[...]"); } @@ -584,7 +614,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console "may use export in classic mode by running vcpkg outside of a manifest-based project."); } const StatusParagraphs status_db = database_load_check(paths); - const auto opts = handle_export_command_arguments(args, default_triplet, status_db); + const auto opts = handle_export_command_arguments(paths, args, default_triplet, status_db); for (auto&& spec : opts.specs) Input::check_triplet(spec.triplet(), paths); diff --git a/toolsrc/src/vcpkg/tools.cpp b/toolsrc/src/vcpkg/tools.cpp index 9098cb135f..0d3f61757f 100644 --- a/toolsrc/src/vcpkg/tools.cpp +++ b/toolsrc/src/vcpkg/tools.cpp @@ -83,8 +83,12 @@ namespace vcpkg const bool has_tool_entry = std::regex_search(XML.cbegin(), XML.cend(), match_tool_entry, tool_regex); if (!has_tool_entry) { - return Strings::format( - "Could not find entry for tool %s in %s for os=%s", tool, XML_PATH.u8string(), OS_STRING); + return Strings::format("Could not automatically acquire %s because there is no entry in %s for os=%s. You " + "may be able to install %s via your system package manager.", + tool, + XML_PATH.u8string(), + OS_STRING, + tool); } const std::string tool_data =