From b9942d07441934c40869b16184455a7fef2024f5 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 11 Feb 2020 18:41:03 -0800 Subject: [PATCH 1/8] [vcpkg] Extract binary caching functionality into separate interface --- toolsrc/include/vcpkg/binarycaching.h | 43 ++++ toolsrc/include/vcpkg/build.h | 7 +- toolsrc/src/vcpkg/binarycaching.cpp | 206 +++++++++++++++++++ toolsrc/src/vcpkg/build.cpp | 167 ++------------- toolsrc/src/vcpkg/commands.buildexternal.cpp | 2 +- toolsrc/src/vcpkg/install.cpp | 10 +- 6 files changed, 284 insertions(+), 151 deletions(-) create mode 100644 toolsrc/include/vcpkg/binarycaching.h create mode 100644 toolsrc/src/vcpkg/binarycaching.cpp diff --git a/toolsrc/include/vcpkg/binarycaching.h b/toolsrc/include/vcpkg/binarycaching.h new file mode 100644 index 00000000000..ff2be6eacde --- /dev/null +++ b/toolsrc/include/vcpkg/binarycaching.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +namespace vcpkg::Dependencies +{ + struct InstallPlanAction; +} +namespace vcpkg::Build +{ + struct AbiTagAndFile; + struct BuildPackageOptions; +} + +namespace vcpkg +{ + enum RestoreResult + { + MISSING, + SUCCESS, + BUILD_FAILED, + }; + + struct IBinaryProvider + { + virtual ~IBinaryProvider() = default; + virtual void prefetch() = 0; + virtual RestoreResult try_restore(const VcpkgPaths& paths, + const PackageSpec& spec, + const Build::AbiTagAndFile& abi_tag_and_file, + const Build::BuildPackageOptions& build_options) = 0; + virtual void push_success(const VcpkgPaths& paths, + const Build::AbiTagAndFile& abi_tag_and_file, + const Dependencies::InstallPlanAction& action) = 0; + virtual void push_failure(const VcpkgPaths& paths, + const Build::AbiTagAndFile& abi_tag_and_file, + const PackageSpec& spec) = 0; + }; + + std::unique_ptr create_archives_provider(); +} diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 67505d8f225..58a807776d0 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -16,6 +16,11 @@ #include #include +namespace vcpkg +{ + struct IBinaryProvider; +} + namespace vcpkg::Dependencies { struct InstallPlanAction; @@ -28,7 +33,6 @@ namespace vcpkg::Build void perform_and_exit_ex(const FullPackageSpec& full_spec, const SourceControlFileLocation& scfl, const PortFileProvider::PathsPortFileProvider& provider, - const ParsedArguments& options, const VcpkgPaths& paths); void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet); @@ -202,6 +206,7 @@ namespace vcpkg::Build ExtendedBuildResult build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& config, const CMakeVars::CMakeVarProvider& var_provider, + IBinaryProvider* binaries_provider, const StatusParagraphs& status_db); enum class BuildPolicy diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp new file mode 100644 index 00000000000..bbf565c9ce9 --- /dev/null +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -0,0 +1,206 @@ +#include +#include +#include +#include +#include +#include +#include + +using namespace vcpkg; + +namespace +{ + static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths, + const PackageSpec& spec, + const fs::path& archive_path) + { + auto& fs = paths.get_filesystem(); + + auto pkg_path = paths.package_dir(spec); + fs.remove_all(pkg_path, VCPKG_LINE_INFO); + std::error_code ec; + fs.create_directories(pkg_path, ec); + auto files = fs.get_files_non_recursive(pkg_path); + Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string()); + +#if defined(_WIN32) + auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); + auto cmd = Strings::format( + R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string()); +#else + auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string()); +#endif + return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment()); + } + + // Compress the source directory into the destination file. + static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination) + { + auto& fs = paths.get_filesystem(); + + std::error_code ec; + + fs.remove(destination, ec); + Checks::check_exit( + VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string()); +#if defined(_WIN32) + auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); + + System::cmd_execute_and_capture_output( + Strings::format( + R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()), + System::get_clean_environment()); +#else + System::cmd_execute_clean( + Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string())); +#endif + } + + static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination) + { + compress_directory(paths, paths.package_dir(spec), destination); + } + + struct ArchivesBinaryProvider : IBinaryProvider + { + ~ArchivesBinaryProvider() = default; + void prefetch() override {} + RestoreResult try_restore(const VcpkgPaths& paths, + const PackageSpec& spec, + const Build::AbiTagAndFile& abi_tag_and_file, + const Build::BuildPackageOptions& build_options) override + { + auto& fs = paths.get_filesystem(); + std::error_code ec; + const fs::path archives_root_dir = paths.root / "archives"; + const std::string archive_name = abi_tag_and_file.tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); + const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; + if (fs.exists(archive_path)) + { + System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); + + int archive_result = decompress_archive(paths, spec, archive_path).exit_code; + + if (archive_result != 0) + { + System::print2("Failed to decompress archive package\n"); + if (build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO) + { + return RestoreResult::BUILD_FAILED; + } + else + { + System::print2("Purging bad archive\n"); + fs.remove(archive_path, ec); + } + } + else + { + return RestoreResult::SUCCESS; + } + } + + if (fs.exists(archive_tombstone_path)) + { + if (build_options.fail_on_tombstone == Build::FailOnTombstone::YES) + { + System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); + return RestoreResult::BUILD_FAILED; + } + else + { + System::print2( + System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); + } + } + else + { + System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); + } + + return RestoreResult::MISSING; + } + void push_success(const VcpkgPaths& paths, + const Build::AbiTagAndFile& abi_tag_and_file, + const Dependencies::InstallPlanAction& action) override + { + auto& spec = action.spec; + auto& fs = paths.get_filesystem(); + std::error_code ec; + const fs::path archives_root_dir = paths.root / "archives"; + const std::string archive_name = abi_tag_and_file.tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); + const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; + + const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); + + compress_archive(paths, spec, tmp_archive_path); + + fs.create_directories(archive_path.parent_path(), ec); + fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); + if (ec) + { + System::printf(System::Color::warning, + "Failed to store binary cache %s: %s\n", + archive_path.u8string(), + ec.message()); + } + else + System::printf("Stored binary cache: %s\n", archive_path.u8string()); + } + void push_failure(const VcpkgPaths& paths, + const Build::AbiTagAndFile& abi_tag_and_file, + const PackageSpec& spec) override + { + auto& fs = paths.get_filesystem(); + std::error_code ec; + const fs::path archives_root_dir = paths.root / "archives"; + const std::string archive_name = abi_tag_and_file.tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); + const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; + + if (!fs.exists(archive_tombstone_path)) + { + // Build failed, store all failure logs in the tombstone. + const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; + const auto tmp_log_path_destination = tmp_log_path / spec.name(); + const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; + fs.create_directories(tmp_log_path_destination, ec); + + for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) + { + if (log_file.path().extension() == ".log") + { + fs.copy_file(log_file.path(), + tmp_log_path_destination / log_file.path().filename(), + fs::stdfs::copy_options::none, + ec); + } + } + + compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); + + fs.create_directories(archive_tombstone_path.parent_path(), ec); + fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); + + // clean up temporary directory + fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); + } + } + }; +} + +std::unique_ptr vcpkg::create_archives_provider() +{ + return std::make_unique(); +} diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index b4b8a3485e9..7ec6ff60e39 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -38,11 +39,8 @@ namespace vcpkg::Build::Command void perform_and_exit_ex(const FullPackageSpec& full_spec, const SourceControlFileLocation& scfl, const PathsPortFileProvider& provider, - const ParsedArguments& options, const VcpkgPaths& paths) { - vcpkg::Util::unused(options); - CMakeVars::TripletCMakeVarProvider var_provider(paths); var_provider.load_dep_info_vars(std::array{full_spec.package_spec}); var_provider.load_tag_vars(std::array{full_spec}, provider); @@ -94,7 +92,8 @@ namespace vcpkg::Build::Command action->build_options = build_package_options; const auto build_timer = Chrono::ElapsedTimer::create_started(); - const auto result = Build::build_package(paths, *action, var_provider, status_db); + const auto result = + Build::build_package(paths, *action, var_provider, create_archives_provider().get(), status_db); System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n'); if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES) @@ -147,7 +146,7 @@ namespace vcpkg::Build::Command Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name); - perform_and_exit_ex(spec, *scfl, provider, options, paths); + perform_and_exit_ex(spec, *scfl, provider, paths); } } @@ -737,62 +736,14 @@ namespace vcpkg::Build return nullopt; } - static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths, - const PackageSpec& spec, - const fs::path& archive_path) - { - auto& fs = paths.get_filesystem(); - - auto pkg_path = paths.package_dir(spec); - fs.remove_all(pkg_path, VCPKG_LINE_INFO); - std::error_code ec; - fs.create_directories(pkg_path, ec); - auto files = fs.get_files_non_recursive(pkg_path); - Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string()); - -#if defined(_WIN32) - auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); - auto cmd = Strings::format( - R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string()); -#else - auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string()); -#endif - return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment()); - } - - // Compress the source directory into the destination file. - static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination) - { - auto& fs = paths.get_filesystem(); - - std::error_code ec; - - fs.remove(destination, ec); - Checks::check_exit( - VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string()); -#if defined(_WIN32) - auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); - - System::cmd_execute_and_capture_output( - Strings::format( - R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()), - System::get_clean_environment()); -#else - System::cmd_execute_clean( - Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string())); -#endif - } - - static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination) - { - compress_directory(paths, paths.package_dir(spec), destination); - } - ExtendedBuildResult build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action, const CMakeVars::CMakeVarProvider& var_provider, + IBinaryProvider* binaries_provider, const StatusParagraphs& status_db) { + auto binary_caching_enabled = binaries_provider && action.build_options.binary_caching == BinaryCaching::YES; + auto& fs = paths.get_filesystem(); Triplet triplet = action.spec.triplet(); const std::string& name = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO) @@ -843,116 +794,40 @@ namespace vcpkg::Build std::error_code ec; const auto abi_tag_and_file = maybe_abi_tag_and_file.get(); - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag_and_file->tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - - if (action.build_options.binary_caching == BinaryCaching::YES) + if (binary_caching_enabled) { - if (fs.exists(archive_path)) + auto restore = binaries_provider->try_restore(paths, spec, *abi_tag_and_file, action.build_options); + if (restore == RestoreResult::BUILD_FAILED) + return BuildResult::BUILD_FAILED; + else if (restore == RestoreResult::SUCCESS) { - System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); - - int archive_result = decompress_archive(paths, spec, archive_path).exit_code; - - if (archive_result != 0) - { - System::print2("Failed to decompress archive package\n"); - if (action.build_options.purge_decompress_failure == PurgeDecompressFailure::NO) - { - return BuildResult::BUILD_FAILED; - } - else - { - System::print2("Purging bad archive\n"); - fs.remove(archive_path, ec); - } - } - else - { - auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec); - auto bcf = std::make_unique(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); - return {BuildResult::SUCCEEDED, std::move(bcf)}; - } + auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec); + auto bcf = std::make_unique(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); + return {BuildResult::SUCCEEDED, std::move(bcf)}; } - - if (fs.exists(archive_tombstone_path)) + else { - if (action.build_options.fail_on_tombstone == FailOnTombstone::YES) - { - System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); - return BuildResult::BUILD_FAILED; - } - else - { - System::print2( - System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); - } + // missing package, proceed to build. } - - System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); } ExtendedBuildResult result = do_build_package_and_clean_buildtrees( paths, pre_build_info, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), action); fs.create_directories(abi_package_dir, ec); - Checks::check_exit(VCPKG_LINE_INFO, !ec, "Coud not create directory %s", abi_package_dir.u8string()); fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); - if (action.build_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED) + if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED) { - const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); - - compress_archive(paths, spec, tmp_archive_path); - - fs.create_directories(archive_path.parent_path(), ec); - fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); - if (ec) - { - System::printf(System::Color::warning, - "Failed to store binary cache %s: %s\n", - archive_path.u8string(), - ec.message()); - } - else - System::printf("Stored binary cache: %s\n", archive_path.u8string()); + binaries_provider->push_success(paths, *abi_tag_and_file, action); } - else if (action.build_options.binary_caching == BinaryCaching::YES && + else if (binary_caching_enabled && (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) { - if (!fs.exists(archive_tombstone_path)) - { - // Build failed, store all failure logs in the tombstone. - const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; - const auto tmp_log_path_destination = tmp_log_path / spec.name(); - const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; - fs.create_directories(tmp_log_path_destination, ec); - - for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) - { - if (log_file.path().extension() == ".log") - { - fs.copy_file(log_file.path(), - tmp_log_path_destination / log_file.path().filename(), - fs::stdfs::copy_options::none, - ec); - } - } - - compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); - - fs.create_directories(archive_tombstone_path.parent_path(), ec); - fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); - - // clean up temporary directory - fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); - } + binaries_provider->push_failure(paths, *abi_tag_and_file, spec); } return result; diff --git a/toolsrc/src/vcpkg/commands.buildexternal.cpp b/toolsrc/src/vcpkg/commands.buildexternal.cpp index 4a69be87942..07ad7b6b173 100644 --- a/toolsrc/src/vcpkg/commands.buildexternal.cpp +++ b/toolsrc/src/vcpkg/commands.buildexternal.cpp @@ -33,6 +33,6 @@ namespace vcpkg::Commands::BuildExternal Checks::check_exit( VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name()); - Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, options, paths); + Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, paths); } } diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index e2a8ce4c559..1cd29284c2b 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -299,6 +300,7 @@ namespace vcpkg::Install ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths, InstallPlanAction& action, StatusParagraphs& status_db, + IBinaryProvider* binaries_provider, const CMakeVars::CMakeVarProvider& var_provider) { const InstallPlanType& plan_type = action.plan_type; @@ -339,7 +341,7 @@ namespace vcpkg::Install else System::printf("Building package %s...\n", display_name_with_features); - auto result = Build::build_package(paths, action, var_provider, status_db); + auto result = Build::build_package(paths, action, var_provider, binaries_provider, status_db); if (BuildResult::DOWNLOADED == result.code) { @@ -456,13 +458,15 @@ namespace vcpkg::Install for (auto&& action : action_plan.already_installed) { results.emplace_back(action.spec, &action); - results.back().build_result = perform_install_plan_action(paths, action, status_db, var_provider); + results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr, var_provider); } + auto binary_provider = create_archives_provider(); for (auto&& action : action_plan.install_actions) { with_tracking(action.spec, [&]() { - auto result = perform_install_plan_action(paths, action, status_db, var_provider); + auto result = + perform_install_plan_action(paths, action, status_db, binary_provider.get(), var_provider); if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO) { From 3d841c9b6a67edf2126c68698a9f05b5d1b5fbcd Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 11 Feb 2020 23:52:56 -0800 Subject: [PATCH 2/8] [vcpkg] Compute all ABIs upfront instead of during build_package() --- toolsrc/include/vcpkg/binarycaching.h | 16 ++-- toolsrc/include/vcpkg/build.h | 8 +- toolsrc/include/vcpkg/dependencies.h | 6 +- toolsrc/src/vcpkg/binarycaching.cpp | 76 +++++++++++------- toolsrc/src/vcpkg/build.cpp | 111 ++++++++++++++++++-------- toolsrc/src/vcpkg/commands.ci.cpp | 77 ++++-------------- toolsrc/src/vcpkg/dependencies.cpp | 9 +++ toolsrc/src/vcpkg/install.cpp | 12 +-- 8 files changed, 174 insertions(+), 141 deletions(-) diff --git a/toolsrc/include/vcpkg/binarycaching.h b/toolsrc/include/vcpkg/binarycaching.h index ff2be6eacde..60b2eeab76a 100644 --- a/toolsrc/include/vcpkg/binarycaching.h +++ b/toolsrc/include/vcpkg/binarycaching.h @@ -27,16 +27,12 @@ namespace vcpkg { virtual ~IBinaryProvider() = default; virtual void prefetch() = 0; - virtual RestoreResult try_restore(const VcpkgPaths& paths, - const PackageSpec& spec, - const Build::AbiTagAndFile& abi_tag_and_file, - const Build::BuildPackageOptions& build_options) = 0; - virtual void push_success(const VcpkgPaths& paths, - const Build::AbiTagAndFile& abi_tag_and_file, - const Dependencies::InstallPlanAction& action) = 0; - virtual void push_failure(const VcpkgPaths& paths, - const Build::AbiTagAndFile& abi_tag_and_file, - const PackageSpec& spec) = 0; + virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0; + virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0; + virtual void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) = 0; + virtual RestoreResult precheck(const VcpkgPaths& paths, + const Dependencies::InstallPlanAction& action, + bool purge_tombstones) = 0; }; std::unique_ptr create_archives_provider(); diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 58a807776d0..d36b641430c 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -24,6 +24,7 @@ namespace vcpkg namespace vcpkg::Dependencies { struct InstallPlanAction; + struct ActionPlan; } namespace vcpkg::Build @@ -205,7 +206,6 @@ namespace vcpkg::Build ExtendedBuildResult build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& config, - const CMakeVars::CMakeVarProvider& var_provider, IBinaryProvider* binaries_provider, const StatusParagraphs& status_db); @@ -293,8 +293,12 @@ namespace vcpkg::Build fs::path tag_file; }; + void compute_all_abis(const VcpkgPaths& paths, + Dependencies::ActionPlan& action_plan, + const CMakeVars::CMakeVarProvider& var_provider, + const StatusParagraphs& status_db); + Optional compute_abi_tag(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& config, - const PreBuildInfo& pre_build_info, Span dependency_abis); } diff --git a/toolsrc/include/vcpkg/dependencies.h b/toolsrc/include/vcpkg/dependencies.h index 1f92ee63c0a..26ba9a67c73 100644 --- a/toolsrc/include/vcpkg/dependencies.h +++ b/toolsrc/include/vcpkg/dependencies.h @@ -53,6 +53,7 @@ namespace vcpkg::Dependencies std::unordered_map>&& dependencies); std::string displayname() const; + const std::string& public_abi() const; PackageSpec spec; @@ -65,8 +66,11 @@ namespace vcpkg::Dependencies std::unordered_map> feature_dependencies; std::vector package_dependencies; - std::vector feature_list; + + Optional> pre_build_info; + Optional package_abi; + Optional abi_tag_file; }; enum class RemovePlanType diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp index bbf565c9ce9..dd877033037 100644 --- a/toolsrc/src/vcpkg/binarycaching.cpp +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -56,29 +56,21 @@ namespace #endif } - static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination) - { - compress_directory(paths, paths.package_dir(spec), destination); - } - struct ArchivesBinaryProvider : IBinaryProvider { ~ArchivesBinaryProvider() = default; void prefetch() override {} - RestoreResult try_restore(const VcpkgPaths& paths, - const PackageSpec& spec, - const Build::AbiTagAndFile& abi_tag_and_file, - const Build::BuildPackageOptions& build_options) override + RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override { + const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO); + auto& spec = action.spec; auto& fs = paths.get_filesystem(); std::error_code ec; const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag_and_file.tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name; + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; const fs::path archive_path = archives_root_dir / archive_subpath; const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); - const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; if (fs.exists(archive_path)) { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); @@ -88,7 +80,7 @@ namespace if (archive_result != 0) { System::print2("Failed to decompress archive package\n"); - if (build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO) + if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO) { return RestoreResult::BUILD_FAILED; } @@ -106,7 +98,7 @@ namespace if (fs.exists(archive_tombstone_path)) { - if (build_options.fail_on_tombstone == Build::FailOnTombstone::YES) + if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) { System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); return RestoreResult::BUILD_FAILED; @@ -124,24 +116,20 @@ namespace return RestoreResult::MISSING; } - void push_success(const VcpkgPaths& paths, - const Build::AbiTagAndFile& abi_tag_and_file, - const Dependencies::InstallPlanAction& action) override + void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override { + const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO); auto& spec = action.spec; auto& fs = paths.get_filesystem(); std::error_code ec; const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag_and_file.tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name; + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); - const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); - compress_archive(paths, spec, tmp_archive_path); + compress_directory(paths, paths.package_dir(spec), tmp_archive_path); fs.create_directories(archive_path.parent_path(), ec); fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); @@ -155,15 +143,13 @@ namespace else System::printf("Stored binary cache: %s\n", archive_path.u8string()); } - void push_failure(const VcpkgPaths& paths, - const Build::AbiTagAndFile& abi_tag_and_file, - const PackageSpec& spec) override + void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) override { auto& fs = paths.get_filesystem(); std::error_code ec; const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag_and_file.tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name; + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; const fs::path archive_path = archives_root_dir / archive_subpath; const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); @@ -197,6 +183,38 @@ namespace fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); } } + RestoreResult precheck(const VcpkgPaths& paths, + const Dependencies::InstallPlanAction& action, + bool purge_tombstones) override + { + const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO); + auto& fs = paths.get_filesystem(); + std::error_code ec; + const fs::path archives_root_dir = paths.root / "archives"; + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + + if (fs.exists(archive_path)) + { + return RestoreResult::SUCCESS; + } + + if (purge_tombstones) + { + fs.remove(archive_tombstone_path, ec); // Ignore error + } + else if (fs.exists(archive_tombstone_path)) + { + if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) + { + return RestoreResult::BUILD_FAILED; + } + } + + return RestoreResult::MISSING; + } }; } diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 7ec6ff60e39..c6b3588192f 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -92,8 +92,7 @@ namespace vcpkg::Build::Command action->build_options = build_package_options; const auto build_timer = Chrono::ElapsedTimer::create_started(); - const auto result = - Build::build_package(paths, *action, var_provider, create_archives_provider().get(), status_db); + const auto result = Build::build_package(paths, *action, create_archives_provider().get(), status_db); System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n'); if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES) @@ -462,11 +461,10 @@ namespace vcpkg::Build return hash; } - static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, - const PreBuildInfo& pre_build_info, - const std::string& abi_tag, - const Dependencies::InstallPlanAction& action) + static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) { + const auto& pre_build_info = *action.pre_build_info.value_or_exit(VCPKG_LINE_INFO).get(); + auto& fs = paths.get_filesystem(); auto&& scfl = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO); @@ -563,8 +561,11 @@ namespace vcpkg::Build auto find_itr = action.feature_dependencies.find("core"); Checks::check_exit(VCPKG_LINE_INFO, find_itr != action.feature_dependencies.end()); - std::unique_ptr bcf = create_binary_control_file( - *scfl.source_control_file->core_paragraph, triplet, build_info, abi_tag, std::move(find_itr->second)); + std::unique_ptr bcf = create_binary_control_file(*scfl.source_control_file->core_paragraph, + triplet, + build_info, + action.public_abi(), + std::move(find_itr->second)); if (error_count != 0) { @@ -590,11 +591,9 @@ namespace vcpkg::Build } static ExtendedBuildResult do_build_package_and_clean_buildtrees(const VcpkgPaths& paths, - const PreBuildInfo& pre_build_info, - const std::string& abi_tag, const Dependencies::InstallPlanAction& action) { - auto result = do_build_package(paths, pre_build_info, abi_tag, action); + auto result = do_build_package(paths, action); if (action.build_options.clean_buildtrees == CleanBuildtrees::YES) { @@ -617,12 +616,12 @@ namespace vcpkg::Build Optional compute_abi_tag(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action, - const PreBuildInfo& pre_build_info, Span dependency_abis) { auto& fs = paths.get_filesystem(); Triplet triplet = action.spec.triplet(); const std::string& name = action.spec.name(); + const auto& pre_build_info = *action.pre_build_info.value_or_exit(VCPKG_LINE_INFO); std::vector abi_tag_entries(dependency_abis.begin(), dependency_abis.end()); @@ -736,9 +735,62 @@ namespace vcpkg::Build return nullopt; } + void compute_all_abis(const VcpkgPaths& paths, + Dependencies::ActionPlan& action_plan, + const CMakeVars::CMakeVarProvider& var_provider, + const StatusParagraphs& status_db) + { + using Dependencies::InstallPlanAction; + for (auto it = action_plan.install_actions.begin(); it != action_plan.install_actions.end(); ++it) + { + auto& action = *it; + std::vector dependency_abis; + if (!Util::Enum::to_bool(action.build_options.only_downloads)) + { + for (auto&& pspec : action.package_dependencies) + { + if (pspec == action.spec) continue; + + auto pred = [&](const InstallPlanAction& ipa) { return ipa.spec == pspec; }; + auto it2 = std::find_if(action_plan.install_actions.begin(), it, pred); + if (it2 == it) + { + // Finally, look in current installed + auto status_it = status_db.find(pspec); + if (status_it == status_db.end()) + { + Checks::exit_with_message( + VCPKG_LINE_INFO, "Failed to find dependency abi for %s -> %s", action.spec, pspec); + } + else + { + dependency_abis.emplace_back(AbiEntry{pspec.name(), status_it->get()->package.abi}); + } + } + else + { + dependency_abis.emplace_back(AbiEntry{pspec.name(), it2->public_abi()}); + } + } + } + + action.pre_build_info = std::make_unique( + paths, action.spec.triplet(), var_provider.get_tag_vars(action.spec).value_or_exit(VCPKG_LINE_INFO)); + auto maybe_abi_tag_and_file = compute_abi_tag(paths, action, dependency_abis); + if (auto p = maybe_abi_tag_and_file.get()) + { + action.abi_tag_file = std::move(p->tag_file); + action.package_abi = std::move(p->tag); + } + else + { + action.package_abi = ""; + } + } + } + ExtendedBuildResult build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action, - const CMakeVars::CMakeVarProvider& var_provider, IBinaryProvider* binaries_provider, const StatusParagraphs& status_db) { @@ -746,17 +798,18 @@ namespace vcpkg::Build auto& fs = paths.get_filesystem(); Triplet triplet = action.spec.triplet(); + auto& spec = action.spec; const std::string& name = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO) .source_control_file->core_paragraph->name; std::vector missing_fspecs; for (const auto& kv : action.feature_dependencies) { - for (const FeatureSpec& spec : kv.second) + for (const FeatureSpec& fspec : kv.second) { - if (!(status_db.is_installed(spec) || spec.name() == name)) + if (!(status_db.is_installed(fspec) || spec.name() == name)) { - missing_fspecs.emplace_back(spec); + missing_fspecs.emplace_back(fspec); } } } @@ -766,8 +819,6 @@ namespace vcpkg::Build return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(missing_fspecs)}; } - const PackageSpec spec(name, triplet); - std::vector dependency_abis; for (auto&& pspec : action.package_dependencies) { @@ -781,24 +832,19 @@ namespace vcpkg::Build AbiEntry{status_it->get()->package.spec.name(), status_it->get()->package.abi}); } - const std::unordered_map& cmake_vars = - var_provider.get_tag_vars(spec).value_or_exit(VCPKG_LINE_INFO); - const PreBuildInfo pre_build_info(paths, triplet, cmake_vars); - - auto maybe_abi_tag_and_file = compute_abi_tag(paths, action, pre_build_info, dependency_abis); - if (!maybe_abi_tag_and_file) + if (!action.abi_tag_file) { - return do_build_package_and_clean_buildtrees( - paths, pre_build_info, pre_build_info.public_abi_override.value_or(AbiTagAndFile{}.tag), action); + return do_build_package_and_clean_buildtrees(paths, action); } + auto& abi_file = *action.abi_tag_file.get(); + std::error_code ec; - const auto abi_tag_and_file = maybe_abi_tag_and_file.get(); const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; if (binary_caching_enabled) { - auto restore = binaries_provider->try_restore(paths, spec, *abi_tag_and_file, action.build_options); + auto restore = binaries_provider->try_restore(paths, action); if (restore == RestoreResult::BUILD_FAILED) return BuildResult::BUILD_FAILED; else if (restore == RestoreResult::SUCCESS) @@ -813,21 +859,20 @@ namespace vcpkg::Build } } - ExtendedBuildResult result = do_build_package_and_clean_buildtrees( - paths, pre_build_info, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), action); + ExtendedBuildResult result = do_build_package_and_clean_buildtrees(paths, action); fs.create_directories(abi_package_dir, ec); - fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); + fs.copy_file(abi_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED) { - binaries_provider->push_success(paths, *abi_tag_and_file, action); + binaries_provider->push_success(paths, action); } else if (binary_caching_enabled && (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) { - binaries_provider->push_failure(paths, *abi_tag_and_file, spec); + binaries_provider->push_failure(paths, action.package_abi.value_or_exit(VCPKG_LINE_INFO), spec); } return result; diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 00e25e223fe..842bb9b0a04 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -220,7 +221,6 @@ namespace vcpkg::Commands::CI std::map known; std::map> features; std::unordered_map default_feature_provider; - std::map abi_tag_map; }; static bool supported_for_triplet(const CMakeVars::TripletCMakeVarProvider& var_provider, @@ -259,8 +259,6 @@ namespace vcpkg::Commands::CI { auto ret = std::make_unique(); - auto& fs = paths.get_filesystem(); - std::set will_fail; const Build::BuildPackageOptions build_options = { @@ -306,59 +304,24 @@ namespace vcpkg::Commands::CI Checks::check_exit(VCPKG_LINE_INFO, action_plan.already_installed.empty(), "Cannot use CI command with packages already installed."); + + Build::compute_all_abis(paths, action_plan, var_provider, {}); + + auto binaryprovider = create_archives_provider(); + std::string stdout_buffer; for (auto&& action : action_plan.install_actions) { auto p = &action; - // determine abi tag - std::string abi; if (auto scfl = p->source_control_file_location.get()) { auto emp = ret->default_feature_provider.emplace(p->spec.name(), *scfl); emp.first->second.source_control_file->core_paragraph->default_features = p->feature_list; - auto triplet = p->spec.triplet(); - p->build_options = build_options; - - auto dependency_abis = - Util::fmap(p->package_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry { - auto it = ret->abi_tag_map.find(spec); - - if (it == ret->abi_tag_map.end()) - return {spec.name(), ""}; - else - return {spec.name(), it->second}; - }); - - const auto pre_build_info = Build::PreBuildInfo( - paths, triplet, var_provider.get_tag_vars(p->spec).value_or_exit(VCPKG_LINE_INFO)); - - auto maybe_tag_and_file = Build::compute_abi_tag(paths, *p, pre_build_info, dependency_abis); - if (auto tag_and_file = maybe_tag_and_file.get()) - { - abi = tag_and_file->tag; - ret->abi_tag_map.emplace(p->spec, abi); - } - } - else if (auto ipv = p->installed_package.get()) - { - abi = ipv->core->package.abi; - if (!abi.empty()) ret->abi_tag_map.emplace(p->spec, abi); - } - - auto archives_root_dir = paths.root / "archives"; - auto archive_name = abi + ".zip"; - auto archive_subpath = fs::u8path(abi.substr(0, 2)) / archive_name; - auto archive_path = archives_root_dir / archive_subpath; - auto archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - - if (purge_tombstones) - { - std::error_code ec; - fs.remove(archive_tombstone_path, ec); // Ignore error } + auto precheck_result = binaryprovider->precheck(paths, action, purge_tombstones); bool b_will_build = false; std::string state; @@ -385,12 +348,12 @@ namespace vcpkg::Commands::CI ret->known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES); will_fail.emplace(p->spec); } - else if (fs.exists(archive_path)) + else if (precheck_result == RestoreResult::SUCCESS) { state = "pass"; ret->known.emplace(p->spec, BuildResult::SUCCEEDED); } - else if (fs.exists(archive_tombstone_path)) + else if (precheck_result == RestoreResult::BUILD_FAILED) { state = "fail"; ret->known.emplace(p->spec, BuildResult::BUILD_FAILED); @@ -402,8 +365,10 @@ namespace vcpkg::Commands::CI b_will_build = true; } - Strings::append(stdout_buffer, - Strings::format("%40s: %1s %8s: %s\n", p->spec, (b_will_build ? "*" : " "), state, abi)); + Strings::append( + stdout_buffer, + Strings::format( + "%40s: %1s %8s: %s\n", p->spec, (b_will_build ? "*" : " "), state, action.public_abi())); if (stdout_buffer.size() > 2048) { System::print2(stdout_buffer); @@ -465,7 +430,6 @@ namespace vcpkg::Commands::CI }; std::vector> all_known_results; - std::map abi_tag_map; XunitTestResults xunitTestResults; @@ -543,26 +507,19 @@ namespace vcpkg::Commands::CI { auto& port_features = split_specs->features[result.spec]; split_specs->known.erase(result.spec); - xunitTestResults.add_test_results(result.spec.to_string(), - result.build_result.code, - result.timing, - split_specs->abi_tag_map.at(result.spec), - port_features); + xunitTestResults.add_test_results( + result.spec.to_string(), result.build_result.code, result.timing, "", port_features); } // Adding results for ports that were not built because they have known states for (auto&& port : split_specs->known) { auto& port_features = split_specs->features[port.first]; - xunitTestResults.add_test_results(port.first.to_string(), - port.second, - Chrono::ElapsedTime{}, - split_specs->abi_tag_map.at(port.first), - port_features); + xunitTestResults.add_test_results( + port.first.to_string(), port.second, Chrono::ElapsedTime{}, "", port_features); } all_known_results.emplace_back(std::move(split_specs->known)); - abi_tag_map.insert(split_specs->abi_tag_map.begin(), split_specs->abi_tag_map.end()); results.push_back({triplet, std::move(summary)}); diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index beb92f4b097..07aad4d2209 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -403,6 +403,15 @@ namespace vcpkg::Dependencies const std::string features = Strings::join(",", feature_list); return Strings::format("%s[%s]:%s", this->spec.name(), features, this->spec.triplet()); } + const std::string& InstallPlanAction::public_abi() const + { + if (auto p = pre_build_info.get()) + { + if (auto q = p->get()->public_abi_override.get()) return *q; + } + if (auto p = installed_package.get()) return p->core->package.abi; + return package_abi.value_or_exit(VCPKG_LINE_INFO); + } bool InstallPlanAction::compare_by_name(const InstallPlanAction* left, const InstallPlanAction* right) { diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 1cd29284c2b..fc398bd45a1 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -300,8 +300,7 @@ namespace vcpkg::Install ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths, InstallPlanAction& action, StatusParagraphs& status_db, - IBinaryProvider* binaries_provider, - const CMakeVars::CMakeVarProvider& var_provider) + IBinaryProvider* binaries_provider) { const InstallPlanType& plan_type = action.plan_type; const std::string display_name = action.spec.to_string(); @@ -341,7 +340,7 @@ namespace vcpkg::Install else System::printf("Building package %s...\n", display_name_with_features); - auto result = Build::build_package(paths, action, var_provider, binaries_provider, status_db); + auto result = Build::build_package(paths, action, binaries_provider, status_db); if (BuildResult::DOWNLOADED == result.code) { @@ -458,15 +457,16 @@ namespace vcpkg::Install for (auto&& action : action_plan.already_installed) { results.emplace_back(action.spec, &action); - results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr, var_provider); + results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr); } + Build::compute_all_abis(paths, action_plan, var_provider, status_db); + auto binary_provider = create_archives_provider(); for (auto&& action : action_plan.install_actions) { with_tracking(action.spec, [&]() { - auto result = - perform_install_plan_action(paths, action, status_db, binary_provider.get(), var_provider); + auto result = perform_install_plan_action(paths, action, status_db, binary_provider.get()); if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO) { From b9273daa2049b1cf51976806a5f6848145ae0aa6 Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Wed, 12 Feb 2020 14:25:47 -0800 Subject: [PATCH 3/8] Add binarycaching.(h|cpp) to project files --- toolsrc/src/vcpkg/binarycaching.cpp | 2 ++ toolsrc/vcpkglib/vcpkglib.vcxproj | 2 ++ toolsrc/vcpkglib/vcpkglib.vcxproj.filters | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp index dd877033037..659681c9ea0 100644 --- a/toolsrc/src/vcpkg/binarycaching.cpp +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -1,3 +1,5 @@ +#include "pch.h" + #include #include #include diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj b/toolsrc/vcpkglib/vcpkglib.vcxproj index a36cb9bd04e..2fb7766401d 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj @@ -169,6 +169,7 @@ + @@ -228,6 +229,7 @@ + diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj.filters b/toolsrc/vcpkglib/vcpkglib.vcxproj.filters index bc0a3702920..4e2bbb45a9d 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj.filters +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj.filters @@ -231,6 +231,9 @@ Source Files\vcpkg\base + + Source Files\vcpkg + @@ -434,5 +437,8 @@ Header Files\vcpkg + + Header Files\vcpkg + \ No newline at end of file From e21400c7cae2c2df3a9dce6ef8e3c4bf7109aaac Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 12 Feb 2020 15:50:39 -0800 Subject: [PATCH 4/8] [vcpkg] Fix VS2015 compile --- toolsrc/include/vcpkg/textrowcol.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/toolsrc/include/vcpkg/textrowcol.h b/toolsrc/include/vcpkg/textrowcol.h index 5bbf6a899e0..cd2b223e299 100644 --- a/toolsrc/include/vcpkg/textrowcol.h +++ b/toolsrc/include/vcpkg/textrowcol.h @@ -4,6 +4,8 @@ namespace vcpkg::Parse { struct TextRowCol { + TextRowCol() = default; + TextRowCol(int row, int column) : row(row), column(column) {} /// '0' indicates uninitialized; '1' is the first row. int row = 0; /// '0' indicates uninitialized; '1' is the first column. From ebf8213945060c646e8074bcdf2b9463bcd44216 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 18 Mar 2020 11:19:25 -0700 Subject: [PATCH 5/8] [vcpkg] Use enum class for RestoreResult --- toolsrc/include/vcpkg/binarycaching.h | 8 ++++---- toolsrc/src/vcpkg/binarycaching.cpp | 14 +++++++------- toolsrc/src/vcpkg/build.cpp | 10 +++++++--- toolsrc/src/vcpkg/commands.ci.cpp | 4 ++-- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/toolsrc/include/vcpkg/binarycaching.h b/toolsrc/include/vcpkg/binarycaching.h index 60b2eeab76a..199b01accc0 100644 --- a/toolsrc/include/vcpkg/binarycaching.h +++ b/toolsrc/include/vcpkg/binarycaching.h @@ -16,11 +16,11 @@ namespace vcpkg::Build namespace vcpkg { - enum RestoreResult + enum class RestoreResult { - MISSING, - SUCCESS, - BUILD_FAILED, + missing, + success, + build_failed, }; struct IBinaryProvider diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp index 659681c9ea0..af4a879ca6a 100644 --- a/toolsrc/src/vcpkg/binarycaching.cpp +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -84,7 +84,7 @@ namespace System::print2("Failed to decompress archive package\n"); if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO) { - return RestoreResult::BUILD_FAILED; + return RestoreResult::build_failed; } else { @@ -94,7 +94,7 @@ namespace } else { - return RestoreResult::SUCCESS; + return RestoreResult::success; } } @@ -103,7 +103,7 @@ namespace if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) { System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n"); - return RestoreResult::BUILD_FAILED; + return RestoreResult::build_failed; } else { @@ -116,7 +116,7 @@ namespace System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); } - return RestoreResult::MISSING; + return RestoreResult::missing; } void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override { @@ -200,7 +200,7 @@ namespace if (fs.exists(archive_path)) { - return RestoreResult::SUCCESS; + return RestoreResult::success; } if (purge_tombstones) @@ -211,11 +211,11 @@ namespace { if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES) { - return RestoreResult::BUILD_FAILED; + return RestoreResult::build_failed; } } - return RestoreResult::MISSING; + return RestoreResult::missing; } }; } diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index e6a81607d06..81997e79c99 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -527,7 +527,11 @@ namespace vcpkg::Build { std::error_code err; fs.create_directory(buildpath, err); - Checks::check_exit(VCPKG_LINE_INFO, !err.value(), "Failed to create directory '%s', code: %d", buildpath.u8string(), err.value()); + Checks::check_exit(VCPKG_LINE_INFO, + !err.value(), + "Failed to create directory '%s', code: %d", + buildpath.u8string(), + err.value()); } auto stdoutlog = buildpath / ("stdout-" + action.spec.triplet().canonical_name() + ".log"); std::ofstream out_file(stdoutlog.native().c_str(), std::ios::out | std::ios::binary | std::ios::trunc); @@ -865,9 +869,9 @@ namespace vcpkg::Build if (binary_caching_enabled) { auto restore = binaries_provider->try_restore(paths, action); - if (restore == RestoreResult::BUILD_FAILED) + if (restore == RestoreResult::build_failed) return BuildResult::BUILD_FAILED; - else if (restore == RestoreResult::SUCCESS) + else if (restore == RestoreResult::success) { auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec); auto bcf = std::make_unique(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 9cf3026cb44..09bee8ddcaa 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -348,12 +348,12 @@ namespace vcpkg::Commands::CI ret->known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES); will_fail.emplace(p->spec); } - else if (precheck_result == RestoreResult::SUCCESS) + else if (precheck_result == RestoreResult::success) { state = "pass"; ret->known.emplace(p->spec, BuildResult::SUCCEEDED); } - else if (precheck_result == RestoreResult::BUILD_FAILED) + else if (precheck_result == RestoreResult::build_failed) { state = "fail"; ret->known.emplace(p->spec, BuildResult::BUILD_FAILED); From 090e7f8e76bffc232293dac98e1dbf38d4256e69 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 18 Mar 2020 11:20:13 -0700 Subject: [PATCH 6/8] [vcpkg] Write abi information to generated test results --- toolsrc/src/vcpkg/commands.ci.cpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 09bee8ddcaa..b5ab9e79d01 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -221,6 +221,7 @@ namespace vcpkg::Commands::CI std::map known; std::map> features; std::unordered_map default_feature_provider; + std::map abi_map; }; static bool supported_for_triplet(const CMakeVars::CMakeVarProvider& var_provider, @@ -295,6 +296,7 @@ namespace vcpkg::Commands::CI install_specs.emplace_back(FullPackageSpec{ install_action.spec, std::vector{install_action.feature_list.begin(), install_action.feature_list.end()}}); + ret->abi_map.emplace(install_action.spec, install_action.package_abi.value_or_exit(VCPKG_LINE_INFO)); } var_provider.load_tag_vars(install_specs, provider); @@ -365,10 +367,12 @@ namespace vcpkg::Commands::CI b_will_build = true; } - Strings::append( - stdout_buffer, - Strings::format( - "%40s: %1s %8s: %s\n", p->spec, (b_will_build ? "*" : " "), state, action.public_abi())); + Strings::append(stdout_buffer, + Strings::format("%40s: %1s %8s: %s\n", + p->spec, + (b_will_build ? "*" : " "), + state, + action.package_abi.value_or_exit(VCPKG_LINE_INFO))); if (stdout_buffer.size() > 2048) { System::print2(stdout_buffer); @@ -506,18 +510,24 @@ namespace vcpkg::Commands::CI // Adding results for ports that were built or pulled from an archive for (auto&& result : summary.results) { - auto& port_features = split_specs->features[result.spec]; + auto& port_features = split_specs->features.at(result.spec); split_specs->known.erase(result.spec); - xunitTestResults.add_test_results( - result.spec.to_string(), result.build_result.code, result.timing, "", port_features); + xunitTestResults.add_test_results(result.spec.to_string(), + result.build_result.code, + result.timing, + split_specs->abi_map.at(result.spec), + port_features); } // Adding results for ports that were not built because they have known states for (auto&& port : split_specs->known) { - auto& port_features = split_specs->features[port.first]; - xunitTestResults.add_test_results( - port.first.to_string(), port.second, Chrono::ElapsedTime{}, "", port_features); + auto& port_features = split_specs->features.at(port.first); + xunitTestResults.add_test_results(port.first.to_string(), + port.second, + Chrono::ElapsedTime{}, + split_specs->abi_map.at(port.first), + port_features); } all_known_results.emplace_back(std::move(split_specs->known)); From 44b9703a810c38256bb1e221385a63faf59255bb Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 18 Mar 2020 11:48:33 -0700 Subject: [PATCH 7/8] [vcpkg] Move package abi access after they have been computed --- toolsrc/src/vcpkg/commands.ci.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index b5ab9e79d01..7b65eac99b5 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -296,7 +296,6 @@ namespace vcpkg::Commands::CI install_specs.emplace_back(FullPackageSpec{ install_action.spec, std::vector{install_action.feature_list.begin(), install_action.feature_list.end()}}); - ret->abi_map.emplace(install_action.spec, install_action.package_abi.value_or_exit(VCPKG_LINE_INFO)); } var_provider.load_tag_vars(install_specs, provider); @@ -315,6 +314,7 @@ namespace vcpkg::Commands::CI for (auto&& action : action_plan.install_actions) { auto p = &action; + ret->abi_map.emplace(action.spec, action.package_abi.value_or_exit(VCPKG_LINE_INFO)); if (auto scfl = p->source_control_file_location.get()) { auto emp = ret->default_feature_provider.emplace(p->spec.name(), *scfl); From 4bf1cf35ba18b2549b0db87f4f8465fdabed2f47 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 18 Mar 2020 12:18:24 -0700 Subject: [PATCH 8/8] [vcpkg] Fill in feature lists during CI reporting --- toolsrc/src/vcpkg/commands.ci.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 7b65eac99b5..f21366a36e8 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -293,9 +293,7 @@ namespace vcpkg::Commands::CI std::vector install_specs; for (auto&& install_action : action_plan.install_actions) { - install_specs.emplace_back(FullPackageSpec{ - install_action.spec, - std::vector{install_action.feature_list.begin(), install_action.feature_list.end()}}); + install_specs.emplace_back(FullPackageSpec{install_action.spec, install_action.feature_list}); } var_provider.load_tag_vars(install_specs, provider); @@ -315,6 +313,7 @@ namespace vcpkg::Commands::CI { auto p = &action; ret->abi_map.emplace(action.spec, action.package_abi.value_or_exit(VCPKG_LINE_INFO)); + ret->features.emplace(action.spec, action.feature_list); if (auto scfl = p->source_control_file_location.get()) { auto emp = ret->default_feature_provider.emplace(p->spec.name(), *scfl);