Merge pull request #10032 from ras0219-msft/dev/roschuma/compute-all-abis

[vcpkg] Compute ABIs upfront instead of just-in-time

toolsrc/include/vcpkg/binarycaching.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/packagespec.h>
+#include <vcpkg/vcpkgpaths.h>
+
+namespace vcpkg::Dependencies
+{
+    struct InstallPlanAction;
+}
+namespace vcpkg::Build
+{
+    struct AbiTagAndFile;
+    struct BuildPackageOptions;
+}
+
+namespace vcpkg
+{
+    enum class RestoreResult
+    {
+        missing,
+        success,
+        build_failed,
+    };
+
+    struct IBinaryProvider
+    {
+        virtual ~IBinaryProvider() = default;
+        virtual void prefetch() = 0;
+        virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+        virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+        virtual void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) = 0;
+        virtual RestoreResult precheck(const VcpkgPaths& paths,
+                                       const Dependencies::InstallPlanAction& action,
+                                       bool purge_tombstones) = 0;
+    };
+
+    std::unique_ptr<IBinaryProvider> create_archives_provider();
+}

toolsrc/include/vcpkg/build.h

@@ -16,9 +16,15 @@
 #include <set>
 #include <vector>
 
+namespace vcpkg
+{
+    struct IBinaryProvider;
+}
+
 namespace vcpkg::Dependencies
 {
     struct InstallPlanAction;
+    struct ActionPlan;
 }
 
 namespace vcpkg::Build
@@ -28,7 +34,6 @@ namespace vcpkg::Build
         void perform_and_exit_ex(const FullPackageSpec& full_spec,
                                  const SourceControlFileLocation& scfl,
                                  const PortFileProvider::PathsPortFileProvider& provider,
-                                 const ParsedArguments& options,
                                  const VcpkgPaths& paths);
 
         void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet);
@@ -201,7 +206,7 @@ namespace vcpkg::Build
 
     ExtendedBuildResult build_package(const VcpkgPaths& paths,
                                       const Dependencies::InstallPlanAction& config,
-                                      const CMakeVars::CMakeVarProvider& var_provider,
+                                      IBinaryProvider* binaries_provider,
                                       const StatusParagraphs& status_db);
 
     enum class BuildPolicy
@@ -288,8 +293,12 @@ namespace vcpkg::Build
         fs::path tag_file;
     };
 
+    void compute_all_abis(const VcpkgPaths& paths,
+                          Dependencies::ActionPlan& action_plan,
+                          const CMakeVars::CMakeVarProvider& var_provider,
+                          const StatusParagraphs& status_db);
+
     Optional<AbiTagAndFile> compute_abi_tag(const VcpkgPaths& paths,
                                             const Dependencies::InstallPlanAction& config,
-                                            const PreBuildInfo& pre_build_info,
                                             Span<const AbiEntry> dependency_abis);
 }

toolsrc/include/vcpkg/dependencies.h

@@ -53,6 +53,7 @@ namespace vcpkg::Dependencies
                           std::unordered_map<std::string, std::vector<FeatureSpec>>&& dependencies);
 
         std::string displayname() const;
+        const std::string& public_abi() const;
 
         PackageSpec spec;
 
@@ -65,8 +66,11 @@ namespace vcpkg::Dependencies
 
         std::unordered_map<std::string, std::vector<FeatureSpec>> feature_dependencies;
         std::vector<PackageSpec> package_dependencies;
-
         std::vector<std::string> feature_list;
+
+        Optional<std::unique_ptr<Build::PreBuildInfo>> pre_build_info;
+        Optional<std::string> package_abi;
+        Optional<fs::path> abi_tag_file;
     };
 
     enum class RemovePlanType

toolsrc/src/vcpkg/binarycaching.cpp

@@ -0,0 +1,226 @@
+#include "pch.h"
+
+#include <vcpkg/base/checks.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/system.print.h>
+#include <vcpkg/base/system.process.h>
+#include <vcpkg/binarycaching.h>
+#include <vcpkg/build.h>
+#include <vcpkg/dependencies.h>
+
+using namespace vcpkg;
+
+namespace
+{
+    static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,
+                                                        const PackageSpec& spec,
+                                                        const fs::path& archive_path)
+    {
+        auto& fs = paths.get_filesystem();
+
+        auto pkg_path = paths.package_dir(spec);
+        fs.remove_all(pkg_path, VCPKG_LINE_INFO);
+        std::error_code ec;
+        fs.create_directories(pkg_path, ec);
+        auto files = fs.get_files_non_recursive(pkg_path);
+        Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());
+
+#if defined(_WIN32)
+        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
+        auto cmd = Strings::format(
+            R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());
+#else
+        auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());
+#endif
+        return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());
+    }
+
+    // Compress the source directory into the destination file.
+    static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)
+    {
+        auto& fs = paths.get_filesystem();
+
+        std::error_code ec;
+
+        fs.remove(destination, ec);
+        Checks::check_exit(
+            VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());
+#if defined(_WIN32)
+        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
+
+        System::cmd_execute_and_capture_output(
+            Strings::format(
+                R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),
+            System::get_clean_environment());
+#else
+        System::cmd_execute_clean(
+            Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));
+#endif
+    }
+
+    struct ArchivesBinaryProvider : IBinaryProvider
+    {
+        ~ArchivesBinaryProvider() = default;
+        void prefetch() override {}
+        RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
+        {
+            const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);
+            auto& spec = action.spec;
+            auto& fs = paths.get_filesystem();
+            std::error_code ec;
+            const fs::path archives_root_dir = paths.root / "archives";
+            const std::string archive_name = abi_tag + ".zip";
+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
+            const fs::path archive_path = archives_root_dir / archive_subpath;
+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
+            if (fs.exists(archive_path))
+            {
+                System::print2("Using cached binary package: ", archive_path.u8string(), "\n");
+
+                int archive_result = decompress_archive(paths, spec, archive_path).exit_code;
+
+                if (archive_result != 0)
+                {
+                    System::print2("Failed to decompress archive package\n");
+                    if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO)
+                    {
+                        return RestoreResult::build_failed;
+                    }
+                    else
+                    {
+                        System::print2("Purging bad archive\n");
+                        fs.remove(archive_path, ec);
+                    }
+                }
+                else
+                {
+                    return RestoreResult::success;
+                }
+            }
+
+            if (fs.exists(archive_tombstone_path))
+            {
+                if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)
+                {
+                    System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
+                    return RestoreResult::build_failed;
+                }
+                else
+                {
+                    System::print2(
+                        System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
+                }
+            }
+            else
+            {
+                System::printf("Could not locate cached archive: %s\n", archive_path.u8string());
+            }
+
+            return RestoreResult::missing;
+        }
+        void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
+        {
+            const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);
+            auto& spec = action.spec;
+            auto& fs = paths.get_filesystem();
+            std::error_code ec;
+            const fs::path archives_root_dir = paths.root / "archives";
+            const std::string archive_name = abi_tag + ".zip";
+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
+            const fs::path archive_path = archives_root_dir / archive_subpath;
+
+            const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");
+
+            compress_directory(paths, paths.package_dir(spec), tmp_archive_path);
+
+            fs.create_directories(archive_path.parent_path(), ec);
+            fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);
+            if (ec)
+            {
+                System::printf(System::Color::warning,
+                               "Failed to store binary cache %s: %s\n",
+                               archive_path.u8string(),
+                               ec.message());
+            }
+            else
+                System::printf("Stored binary cache: %s\n", archive_path.u8string());
+        }
+        void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) override
+        {
+            auto& fs = paths.get_filesystem();
+            std::error_code ec;
+            const fs::path archives_root_dir = paths.root / "archives";
+            const std::string archive_name = abi_tag + ".zip";
+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
+            const fs::path archive_path = archives_root_dir / archive_subpath;
+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
+            const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();
+            const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";
+
+            if (!fs.exists(archive_tombstone_path))
+            {
+                // Build failed, store all failure logs in the tombstone.
+                const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";
+                const auto tmp_log_path_destination = tmp_log_path / spec.name();
+                const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";
+                fs.create_directories(tmp_log_path_destination, ec);
+
+                for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))
+                {
+                    if (log_file.path().extension() == ".log")
+                    {
+                        fs.copy_file(log_file.path(),
+                                     tmp_log_path_destination / log_file.path().filename(),
+                                     fs::stdfs::copy_options::none,
+                                     ec);
+                    }
+                }
+
+                compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");
+
+                fs.create_directories(archive_tombstone_path.parent_path(), ec);
+                fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);
+
+                // clean up temporary directory
+                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);
+            }
+        }
+        RestoreResult precheck(const VcpkgPaths& paths,
+                               const Dependencies::InstallPlanAction& action,
+                               bool purge_tombstones) override
+        {
+            const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);
+            auto& fs = paths.get_filesystem();
+            std::error_code ec;
+            const fs::path archives_root_dir = paths.root / "archives";
+            const std::string archive_name = abi_tag + ".zip";
+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
+            const fs::path archive_path = archives_root_dir / archive_subpath;
+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
+
+            if (fs.exists(archive_path))
+            {
+                return RestoreResult::success;
+            }
+
+            if (purge_tombstones)
+            {
+                fs.remove(archive_tombstone_path, ec); // Ignore error
+            }
+            else if (fs.exists(archive_tombstone_path))
+            {
+                if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)
+                {
+                    return RestoreResult::build_failed;
+                }
+            }
+
+            return RestoreResult::missing;
+        }
+    };
+}
+
+std::unique_ptr<vcpkg::IBinaryProvider> vcpkg::create_archives_provider()
+{
+    return std::make_unique<ArchivesBinaryProvider>();
+}

toolsrc/src/vcpkg/build.cpp

@@ -12,6 +12,7 @@
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/base/util.h>
 
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/build.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/dependencies.h>
@@ -38,11 +39,8 @@ namespace vcpkg::Build::Command
     void perform_and_exit_ex(const FullPackageSpec& full_spec,
                              const SourceControlFileLocation& scfl,
                              const PathsPortFileProvider& provider,
-                             const ParsedArguments& options,
                              const VcpkgPaths& paths)
     {
-        vcpkg::Util::unused(options);
-
         auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths);
         auto& var_provider = *var_provider_storage;
         var_provider.load_dep_info_vars(std::array<PackageSpec, 1>{full_spec.package_spec});
@@ -95,7 +93,7 @@ namespace vcpkg::Build::Command
         action->build_options = build_package_options;
 
         const auto build_timer = Chrono::ElapsedTimer::create_started();
-        const auto result = Build::build_package(paths, *action, var_provider, status_db);
+        const auto result = Build::build_package(paths, *action, create_archives_provider().get(), status_db);
         System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n');
 
         if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES)
@@ -148,7 +146,7 @@ namespace vcpkg::Build::Command
 
         Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name);
 
-        perform_and_exit_ex(spec, *scfl, provider, options, paths);
+        perform_and_exit_ex(spec, *scfl, provider, paths);
     }
 }
 
@@ -464,11 +462,10 @@ namespace vcpkg::Build
         return hash;
     }
 
-    static ExtendedBuildResult do_build_package(const VcpkgPaths& paths,
+    static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action)
-                                                const PreBuildInfo& pre_build_info,
-                                                const std::string& abi_tag,
-                                                const Dependencies::InstallPlanAction& action)
     {
+        const auto& pre_build_info = *action.pre_build_info.value_or_exit(VCPKG_LINE_INFO).get();
+
         auto& fs = paths.get_filesystem();
         auto&& scfl = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO);
 
@@ -530,7 +527,11 @@ namespace vcpkg::Build
         {
             std::error_code err;
             fs.create_directory(buildpath, err);
-            Checks::check_exit(VCPKG_LINE_INFO, !err.value(), "Failed to create directory '%s', code: %d", buildpath.u8string(), err.value());
+            Checks::check_exit(VCPKG_LINE_INFO,
+                               !err.value(),
+                               "Failed to create directory '%s', code: %d",
+                               buildpath.u8string(),
+                               err.value());
         }
         auto stdoutlog = buildpath / ("stdout-" + action.spec.triplet().canonical_name() + ".log");
         std::ofstream out_file(stdoutlog.native().c_str(), std::ios::out | std::ios::binary | std::ios::trunc);
@@ -584,8 +585,11 @@ namespace vcpkg::Build
         auto find_itr = action.feature_dependencies.find("core");
         Checks::check_exit(VCPKG_LINE_INFO, find_itr != action.feature_dependencies.end());
 
-        std::unique_ptr<BinaryControlFile> bcf = create_binary_control_file(
+        std::unique_ptr<BinaryControlFile> bcf = create_binary_control_file(*scfl.source_control_file->core_paragraph,
-            *scfl.source_control_file->core_paragraph, triplet, build_info, abi_tag, std::move(find_itr->second));
+                                                                            triplet,
+                                                                            build_info,
+                                                                            action.public_abi(),
+                                                                            std::move(find_itr->second));
 
         if (error_count != 0)
         {
@@ -611,11 +615,9 @@ namespace vcpkg::Build
     }
 
     static ExtendedBuildResult do_build_package_and_clean_buildtrees(const VcpkgPaths& paths,
-                                                                     const PreBuildInfo& pre_build_info,
-                                                                     const std::string& abi_tag,
                                                                      const Dependencies::InstallPlanAction& action)
     {
-        auto result = do_build_package(paths, pre_build_info, abi_tag, action);
+        auto result = do_build_package(paths, action);
 
         if (action.build_options.clean_buildtrees == CleanBuildtrees::YES)
         {
@@ -638,12 +640,12 @@ namespace vcpkg::Build
 
     Optional<AbiTagAndFile> compute_abi_tag(const VcpkgPaths& paths,
                                             const Dependencies::InstallPlanAction& action,
-                                            const PreBuildInfo& pre_build_info,
                                             Span<const AbiEntry> dependency_abis)
     {
         auto& fs = paths.get_filesystem();
         Triplet triplet = action.spec.triplet();
         const std::string& name = action.spec.name();
+        const auto& pre_build_info = *action.pre_build_info.value_or_exit(VCPKG_LINE_INFO);
 
         std::vector<AbiEntry> abi_tag_entries(dependency_abis.begin(), dependency_abis.end());
 
@@ -757,75 +759,81 @@ namespace vcpkg::Build
         return nullopt;
     }
 
-    static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,
+    void compute_all_abis(const VcpkgPaths& paths,
-                                                        const PackageSpec& spec,
+                          Dependencies::ActionPlan& action_plan,
-                                                        const fs::path& archive_path)
+                          const CMakeVars::CMakeVarProvider& var_provider,
+                          const StatusParagraphs& status_db)
     {
-        auto& fs = paths.get_filesystem();
+        using Dependencies::InstallPlanAction;
+        for (auto it = action_plan.install_actions.begin(); it != action_plan.install_actions.end(); ++it)
+        {
+            auto& action = *it;
+            std::vector<AbiEntry> dependency_abis;
+            if (!Util::Enum::to_bool(action.build_options.only_downloads))
+            {
+                for (auto&& pspec : action.package_dependencies)
+                {
+                    if (pspec == action.spec) continue;
 
-        auto pkg_path = paths.package_dir(spec);
+                    auto pred = [&](const InstallPlanAction& ipa) { return ipa.spec == pspec; };
-        fs.remove_all(pkg_path, VCPKG_LINE_INFO);
+                    auto it2 = std::find_if(action_plan.install_actions.begin(), it, pred);
-        std::error_code ec;
+                    if (it2 == it)
-        fs.create_directories(pkg_path, ec);
+                    {
-        auto files = fs.get_files_non_recursive(pkg_path);
+                        // Finally, look in current installed
-        Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());
+                        auto status_it = status_db.find(pspec);
+                        if (status_it == status_db.end())
+                        {
+                            Checks::exit_with_message(
+                                VCPKG_LINE_INFO, "Failed to find dependency abi for %s -> %s", action.spec, pspec);
+                        }
+                        else
+                        {
+                            dependency_abis.emplace_back(AbiEntry{pspec.name(), status_it->get()->package.abi});
+                        }
+                    }
+                    else
+                    {
+                        dependency_abis.emplace_back(AbiEntry{pspec.name(), it2->public_abi()});
+                    }
+                }
+            }
 
-#if defined(_WIN32)
+            action.pre_build_info = std::make_unique<PreBuildInfo>(
-        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
+                paths, action.spec.triplet(), var_provider.get_tag_vars(action.spec).value_or_exit(VCPKG_LINE_INFO));
-        auto cmd = Strings::format(
+            auto maybe_abi_tag_and_file = compute_abi_tag(paths, action, dependency_abis);
-            R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());
+            if (auto p = maybe_abi_tag_and_file.get())
-#else
+            {
-        auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());
+                action.abi_tag_file = std::move(p->tag_file);
-#endif
+                action.package_abi = std::move(p->tag);
-        return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());
+            }
-    }
+            else
-
+            {
-    // Compress the source directory into the destination file.
+                action.package_abi = "";
-    static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)
+            }
-    {
+        }
-        auto& fs = paths.get_filesystem();
-
-        std::error_code ec;
-
-        fs.remove(destination, ec);
-        Checks::check_exit(
-            VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());
-#if defined(_WIN32)
-        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
-
-        System::cmd_execute_and_capture_output(
-            Strings::format(
-                R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),
-            System::get_clean_environment());
-#else
-        System::cmd_execute_clean(
-            Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));
-#endif
-    }
-
-    static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination)
-    {
-        compress_directory(paths, paths.package_dir(spec), destination);
     }
 
     ExtendedBuildResult build_package(const VcpkgPaths& paths,
                                       const Dependencies::InstallPlanAction& action,
-                                      const CMakeVars::CMakeVarProvider& var_provider,
+                                      IBinaryProvider* binaries_provider,
                                       const StatusParagraphs& status_db)
     {
+        auto binary_caching_enabled = binaries_provider && action.build_options.binary_caching == BinaryCaching::YES;
+
         auto& fs = paths.get_filesystem();
         Triplet triplet = action.spec.triplet();
+        auto& spec = action.spec;
         const std::string& name = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO)
                                       .source_control_file->core_paragraph->name;
 
         std::vector<FeatureSpec> missing_fspecs;
         for (const auto& kv : action.feature_dependencies)
         {
-            for (const FeatureSpec& spec : kv.second)
+            for (const FeatureSpec& fspec : kv.second)
             {
-                if (!(status_db.is_installed(spec) || spec.name() == name))
+                if (!(status_db.is_installed(fspec) || spec.name() == name))
                 {
-                    missing_fspecs.emplace_back(spec);
+                    missing_fspecs.emplace_back(fspec);
                 }
             }
         }
@@ -835,8 +843,6 @@ namespace vcpkg::Build
             return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(missing_fspecs)};
         }
 
-        const PackageSpec spec(name, triplet);
-
         std::vector<AbiEntry> dependency_abis;
         for (auto&& pspec : action.package_dependencies)
         {
@@ -850,129 +856,47 @@ namespace vcpkg::Build
                 AbiEntry{status_it->get()->package.spec.name(), status_it->get()->package.abi});
         }
 
-        const std::unordered_map<std::string, std::string>& cmake_vars =
+        if (!action.abi_tag_file)
-            var_provider.get_tag_vars(spec).value_or_exit(VCPKG_LINE_INFO);
-        const PreBuildInfo pre_build_info(paths, triplet, cmake_vars);
-
-        auto maybe_abi_tag_and_file = compute_abi_tag(paths, action, pre_build_info, dependency_abis);
-        if (!maybe_abi_tag_and_file)
         {
-            return do_build_package_and_clean_buildtrees(
+            return do_build_package_and_clean_buildtrees(paths, action);
-                paths, pre_build_info, pre_build_info.public_abi_override.value_or(AbiTagAndFile{}.tag), action);
         }
 
+        auto& abi_file = *action.abi_tag_file.get();
+
         std::error_code ec;
-        const auto abi_tag_and_file = maybe_abi_tag_and_file.get();
-        const fs::path archives_root_dir = paths.root / "archives";
-        const std::string archive_name = abi_tag_and_file->tag + ".zip";
-        const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name;
-        const fs::path archive_path = archives_root_dir / archive_subpath;
-        const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
         const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();
         const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";
-
+        if (binary_caching_enabled)
-        if (action.build_options.binary_caching == BinaryCaching::YES)
         {
-            if (fs.exists(archive_path))
+            auto restore = binaries_provider->try_restore(paths, action);
+            if (restore == RestoreResult::build_failed)
+                return BuildResult::BUILD_FAILED;
+            else if (restore == RestoreResult::success)
             {
-                System::print2("Using cached binary package: ", archive_path.u8string(), "\n");
+                auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec);
-
+                auto bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO));
-                int archive_result = decompress_archive(paths, spec, archive_path).exit_code;
+                return {BuildResult::SUCCEEDED, std::move(bcf)};
-
-                if (archive_result != 0)
-                {
-                    System::print2("Failed to decompress archive package\n");
-                    if (action.build_options.purge_decompress_failure == PurgeDecompressFailure::NO)
-                    {
-                        return BuildResult::BUILD_FAILED;
-                    }
-                    else
-                    {
-                        System::print2("Purging bad archive\n");
-                        fs.remove(archive_path, ec);
-                    }
-                }
-                else
-                {
-                    auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec);
-                    auto bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO));
-                    return {BuildResult::SUCCEEDED, std::move(bcf)};
-                }
-            }
-
-            if (fs.exists(archive_tombstone_path))
-            {
-                if (action.build_options.fail_on_tombstone == FailOnTombstone::YES)
-                {
-                    System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
-                    return BuildResult::BUILD_FAILED;
-                }
-                else
-                {
-                    System::print2(
-                        System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
-                }
-            }
-
-            System::printf("Could not locate cached archive: %s\n", archive_path.u8string());
-        }
-
-        ExtendedBuildResult result = do_build_package_and_clean_buildtrees(
-            paths, pre_build_info, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), action);
-
-        fs.create_directories(abi_package_dir, ec);
-        Checks::check_exit(VCPKG_LINE_INFO, !ec, "Coud not create directory %s", abi_package_dir.u8string());
-        fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec);
-        Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string());
-
-        if (action.build_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED)
-        {
-            const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");
-
-            compress_archive(paths, spec, tmp_archive_path);
-
-            fs.create_directories(archive_path.parent_path(), ec);
-            fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);
-            if (ec)
-            {
-                System::printf(System::Color::warning,
-                               "Failed to store binary cache %s: %s\n",
-                               archive_path.u8string(),
-                               ec.message());
             }
             else
-                System::printf("Stored binary cache: %s\n", archive_path.u8string());
+            {
+                // missing package, proceed to build.
+            }
         }
-        else if (action.build_options.binary_caching == BinaryCaching::YES &&
+
+        ExtendedBuildResult result = do_build_package_and_clean_buildtrees(paths, action);
+
+        fs.create_directories(abi_package_dir, ec);
+        fs.copy_file(abi_file, abi_file_in_package, fs::stdfs::copy_options::none, ec);
+        Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string());
+
+        if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED)
+        {
+            binaries_provider->push_success(paths, action);
+        }
+        else if (binary_caching_enabled &&
                  (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED))
         {
-            if (!fs.exists(archive_tombstone_path))
+            binaries_provider->push_failure(paths, action.package_abi.value_or_exit(VCPKG_LINE_INFO), spec);
-            {
-                // Build failed, store all failure logs in the tombstone.
-                const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";
-                const auto tmp_log_path_destination = tmp_log_path / spec.name();
-                const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";
-                fs.create_directories(tmp_log_path_destination, ec);
-
-                for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))
-                {
-                    if (log_file.path().extension() == ".log")
-                    {
-                        fs.copy_file(log_file.path(),
-                                     tmp_log_path_destination / log_file.path().filename(),
-                                     fs::stdfs::copy_options::none,
-                                     ec);
-                    }
-                }
-
-                compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");
-
-                fs.create_directories(archive_tombstone_path.parent_path(), ec);
-                fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);
-
-                // clean up temporary directory
-                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);
-            }
         }
 
         return result;

toolsrc/src/vcpkg/commands.buildexternal.cpp

@@ -33,6 +33,6 @@ namespace vcpkg::Commands::BuildExternal
         Checks::check_exit(
             VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name());
 
-        Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, options, paths);
+        Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, paths);
     }
 }

toolsrc/src/vcpkg/commands.ci.cpp

@@ -6,6 +6,7 @@
 #include <vcpkg/base/stringliteral.h>
 #include <vcpkg/base/system.h>
 #include <vcpkg/base/util.h>
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/build.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/dependencies.h>
@@ -220,7 +221,7 @@ namespace vcpkg::Commands::CI
         std::map<PackageSpec, Build::BuildResult> known;
         std::map<PackageSpec, std::vector<std::string>> features;
         std::unordered_map<std::string, SourceControlFileLocation> default_feature_provider;
-        std::map<PackageSpec, std::string> abi_tag_map;
+        std::map<PackageSpec, std::string> abi_map;
     };
 
     static bool supported_for_triplet(const CMakeVars::CMakeVarProvider& var_provider,
@@ -259,8 +260,6 @@ namespace vcpkg::Commands::CI
     {
         auto ret = std::make_unique<UnknownCIPortsResults>();
 
-        auto& fs = paths.get_filesystem();
-
         std::set<PackageSpec> will_fail;
 
         const Build::BuildPackageOptions build_options = {
@@ -294,9 +293,7 @@ namespace vcpkg::Commands::CI
         std::vector<FullPackageSpec> install_specs;
         for (auto&& install_action : action_plan.install_actions)
         {
-            install_specs.emplace_back(FullPackageSpec{
+            install_specs.emplace_back(FullPackageSpec{install_action.spec, install_action.feature_list});
-                install_action.spec,
-                std::vector<std::string>{install_action.feature_list.begin(), install_action.feature_list.end()}});
         }
 
         var_provider.load_tag_vars(install_specs, provider);
@@ -306,59 +303,26 @@ namespace vcpkg::Commands::CI
         Checks::check_exit(VCPKG_LINE_INFO,
                            action_plan.already_installed.empty(),
                            "Cannot use CI command with packages already installed.");
+
+        Build::compute_all_abis(paths, action_plan, var_provider, {});
+
+        auto binaryprovider = create_archives_provider();
+
         std::string stdout_buffer;
         for (auto&& action : action_plan.install_actions)
         {
             auto p = &action;
-            // determine abi tag
+            ret->abi_map.emplace(action.spec, action.package_abi.value_or_exit(VCPKG_LINE_INFO));
-            std::string abi;
+            ret->features.emplace(action.spec, action.feature_list);
             if (auto scfl = p->source_control_file_location.get())
             {
                 auto emp = ret->default_feature_provider.emplace(p->spec.name(), *scfl);
                 emp.first->second.source_control_file->core_paragraph->default_features = p->feature_list;
 
-                auto triplet = p->spec.triplet();
-
                 p->build_options = build_options;
-
-                auto dependency_abis =
-                    Util::fmap(p->package_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry {
-                        auto it = ret->abi_tag_map.find(spec);
-
-                        if (it == ret->abi_tag_map.end())
-                            return {spec.name(), ""};
-                        else
-                            return {spec.name(), it->second};
-                    });
-
-                const auto pre_build_info = Build::PreBuildInfo(
-                    paths, triplet, var_provider.get_tag_vars(p->spec).value_or_exit(VCPKG_LINE_INFO));
-
-                auto maybe_tag_and_file = Build::compute_abi_tag(paths, *p, pre_build_info, dependency_abis);
-                if (auto tag_and_file = maybe_tag_and_file.get())
-                {
-                    abi = tag_and_file->tag;
-                    ret->abi_tag_map.emplace(p->spec, abi);
-                }
-            }
-            else if (auto ipv = p->installed_package.get())
-            {
-                abi = ipv->core->package.abi;
-                if (!abi.empty()) ret->abi_tag_map.emplace(p->spec, abi);
-            }
-
-            auto archives_root_dir = paths.root / "archives";
-            auto archive_name = abi + ".zip";
-            auto archive_subpath = fs::u8path(abi.substr(0, 2)) / archive_name;
-            auto archive_path = archives_root_dir / archive_subpath;
-            auto archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
-
-            if (purge_tombstones)
-            {
-                std::error_code ec;
-                fs.remove(archive_tombstone_path, ec); // Ignore error
             }
 
+            auto precheck_result = binaryprovider->precheck(paths, action, purge_tombstones);
             bool b_will_build = false;
 
             std::string state;
@@ -385,12 +349,12 @@ namespace vcpkg::Commands::CI
                 ret->known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES);
                 will_fail.emplace(p->spec);
             }
-            else if (fs.exists(archive_path))
+            else if (precheck_result == RestoreResult::success)
             {
                 state = "pass";
                 ret->known.emplace(p->spec, BuildResult::SUCCEEDED);
             }
-            else if (fs.exists(archive_tombstone_path))
+            else if (precheck_result == RestoreResult::build_failed)
             {
                 state = "fail";
                 ret->known.emplace(p->spec, BuildResult::BUILD_FAILED);
@@ -403,7 +367,11 @@ namespace vcpkg::Commands::CI
             }
 
             Strings::append(stdout_buffer,
-                            Strings::format("%40s: %1s %8s: %s\n", p->spec, (b_will_build ? "*" : " "), state, abi));
+                            Strings::format("%40s: %1s %8s: %s\n",
+                                            p->spec,
+                                            (b_will_build ? "*" : " "),
+                                            state,
+                                            action.package_abi.value_or_exit(VCPKG_LINE_INFO)));
             if (stdout_buffer.size() > 2048)
             {
                 System::print2(stdout_buffer);
@@ -466,7 +434,6 @@ namespace vcpkg::Commands::CI
         };
 
         std::vector<std::map<PackageSpec, BuildResult>> all_known_results;
-        std::map<PackageSpec, std::string> abi_tag_map;
 
         XunitTestResults xunitTestResults;
 
@@ -542,28 +509,27 @@ namespace vcpkg::Commands::CI
                 // Adding results for ports that were built or pulled from an archive
                 for (auto&& result : summary.results)
                 {
-                    auto& port_features = split_specs->features[result.spec];
+                    auto& port_features = split_specs->features.at(result.spec);
                     split_specs->known.erase(result.spec);
                     xunitTestResults.add_test_results(result.spec.to_string(),
                                                       result.build_result.code,
                                                       result.timing,
-                                                      split_specs->abi_tag_map.at(result.spec),
+                                                      split_specs->abi_map.at(result.spec),
                                                       port_features);
                 }
 
                 // Adding results for ports that were not built because they have known states
                 for (auto&& port : split_specs->known)
                 {
-                    auto& port_features = split_specs->features[port.first];
+                    auto& port_features = split_specs->features.at(port.first);
                     xunitTestResults.add_test_results(port.first.to_string(),
                                                       port.second,
                                                       Chrono::ElapsedTime{},
-                                                      split_specs->abi_tag_map.at(port.first),
+                                                      split_specs->abi_map.at(port.first),
                                                       port_features);
                 }
 
                 all_known_results.emplace_back(std::move(split_specs->known));
-                abi_tag_map.insert(split_specs->abi_tag_map.begin(), split_specs->abi_tag_map.end());
 
                 results.push_back({triplet, std::move(summary)});
 

toolsrc/src/vcpkg/dependencies.cpp

@@ -403,6 +403,15 @@ namespace vcpkg::Dependencies
         const std::string features = Strings::join(",", feature_list);
         return Strings::format("%s[%s]:%s", this->spec.name(), features, this->spec.triplet());
     }
+    const std::string& InstallPlanAction::public_abi() const
+    {
+        if (auto p = pre_build_info.get())
+        {
+            if (auto q = p->get()->public_abi_override.get()) return *q;
+        }
+        if (auto p = installed_package.get()) return p->core->package.abi;
+        return package_abi.value_or_exit(VCPKG_LINE_INFO);
+    }
 
     bool InstallPlanAction::compare_by_name(const InstallPlanAction* left, const InstallPlanAction* right)
     {

toolsrc/src/vcpkg/install.cpp

@@ -4,6 +4,7 @@
 #include <vcpkg/base/hash.h>
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/util.h>
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/build.h>
 #include <vcpkg/cmakevars.h>
 #include <vcpkg/commands.h>
@@ -299,7 +300,7 @@ namespace vcpkg::Install
     ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths,
                                                     InstallPlanAction& action,
                                                     StatusParagraphs& status_db,
-                                                    const CMakeVars::CMakeVarProvider& var_provider)
+                                                    IBinaryProvider* binaries_provider)
     {
         const InstallPlanType& plan_type = action.plan_type;
         const std::string display_name = action.spec.to_string();
@@ -339,7 +340,7 @@ namespace vcpkg::Install
             else
                 System::printf("Building package %s...\n", display_name_with_features);
 
-            auto result = Build::build_package(paths, action, var_provider, status_db);
+            auto result = Build::build_package(paths, action, binaries_provider, status_db);
 
             if (BuildResult::DOWNLOADED == result.code)
             {
@@ -456,13 +457,16 @@ namespace vcpkg::Install
         for (auto&& action : action_plan.already_installed)
         {
             results.emplace_back(action.spec, &action);
-            results.back().build_result = perform_install_plan_action(paths, action, status_db, var_provider);
+            results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr);
         }
 
+        Build::compute_all_abis(paths, action_plan, var_provider, status_db);
+
+        auto binary_provider = create_archives_provider();
         for (auto&& action : action_plan.install_actions)
         {
             with_tracking(action.spec, [&]() {
-                auto result = perform_install_plan_action(paths, action, status_db, var_provider);
+                auto result = perform_install_plan_action(paths, action, status_db, binary_provider.get());
 
                 if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO)
                 {

toolsrc/vcpkglib/vcpkglib.vcxproj

@@ -169,6 +169,7 @@
     <ClInclude Include="..\include\vcpkg\base\util.h" />
     <ClInclude Include="..\include\vcpkg\base\view.h" />
     <ClInclude Include="..\include\vcpkg\base\zstringview.h" />
+    <ClInclude Include="..\include\vcpkg\binarycaching.h" />
     <ClInclude Include="..\include\vcpkg\binaryparagraph.h" />
     <ClInclude Include="..\include\vcpkg\build.h" />
     <ClInclude Include="..\include\vcpkg\cmakevars.h" />
@@ -228,6 +229,7 @@
     <ClCompile Include="..\src\vcpkg\base\system.cpp" />
     <ClCompile Include="..\src\vcpkg\base\system.print.cpp" />
     <ClCompile Include="..\src\vcpkg\base\system.process.cpp" />
+    <ClCompile Include="..\src\vcpkg\binarycaching.cpp" />
     <ClCompile Include="..\src\vcpkg\binaryparagraph.cpp" />
     <ClCompile Include="..\src\vcpkg\build.cpp" />
     <ClCompile Include="..\src\vcpkg\cmakevars.cpp" />

toolsrc/vcpkglib/vcpkglib.vcxproj.filters

@@ -231,6 +231,9 @@
     <ClCompile Include="..\src\vcpkg\base\system.process.cpp">
       <Filter>Source Files\vcpkg\base</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\vcpkg\binarycaching.cpp">
+      <Filter>Source Files\vcpkg</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\include\pch.h">
@@ -434,5 +437,8 @@
     <ClInclude Include="..\include\vcpkg\textrowcol.h">
       <Filter>Header Files\vcpkg</Filter>
     </ClInclude>
+    <ClInclude Include="..\include\vcpkg\binarycaching.h">
+      <Filter>Header Files\vcpkg</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>