[vcpkg registries] Add git registries (#15054)

* [vcpkg registries] Add git registries support

* Add git registries support to the registries module of vcpkg.
* add e2e tests for git registries
* fix vcpkg.cmake for registries

* fix CRs, remove a thing

* better error messages

* Billy CRs

* fix Robert's CR comment

* I learned about `-c` today

* format

* fix baseline.json

* failing to find baseline is technically not a bug

scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1

@@ -1,13 +1,187 @@
 . "$PSScriptRoot/../end-to-end-tests-prelude.ps1"
 
 
-$commonArgs += @("--x-builtin-port-versions-dir=$PSScriptRoot/../../e2e_ports/port_versions")
+$builtinRegistryArgs = $commonArgs + @("--x-builtin-port-versions-dir=$PSScriptRoot/../../e2e_ports/port_versions")
 
-Run-Vcpkg install @commonArgs 'vcpkg-internal-e2e-test-port'
+Run-Vcpkg install @builtinRegistryArgs 'vcpkg-internal-e2e-test-port'
 Throw-IfNotFailed
 
-Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port'
+# We should not look into the port_versions directory unless we have a baseline,
+# even if we pass the registries feature flag
+Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port'
+Throw-IfNotFailed
+
+Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'zlib'
 Throw-IfFailed
 
-Run-Vcpkg install @commonArgs --feature-flags=registries 'zlib'
-Throw-IfFailed
+# Test git and filesystem registries
+Refresh-TestRoot
+$filesystemRegistry = "$TestingRoot/filesystem-registry"
+$gitRegistryUpstream = "$TestingRoot/git-registry-upstream"
+
+# build a filesystem registry
+New-Item -Path $filesystemRegistry -ItemType Directory
+$filesystemRegistry = (Get-Item $filesystemRegistry).FullName
+
+Copy-Item -Recurse `
+    -LiteralPath "$PSScriptRoot/../../e2e_ports/vcpkg-internal-e2e-test-port" `
+    -Destination "$filesystemRegistry"
+New-Item `
+    -Path "$filesystemRegistry/port_versions" `
+    -ItemType Directory
+Copy-Item `
+    -LiteralPath "$PSScriptRoot/../../e2e_ports/port_versions/baseline.json" `
+    -Destination "$filesystemRegistry/port_versions/baseline.json"
+New-Item `
+    -Path "$filesystemRegistry/port_versions/v-" `
+    -ItemType Directory
+
+$vcpkgInternalE2eTestPortJson = @{
+    "versions" = @(
+        @{
+            "version-string" = "1.0.0";
+            "path" = "$/vcpkg-internal-e2e-test-port"
+        }
+    )
+}
+New-Item `
+    -Path "$filesystemRegistry/port_versions/v-/vcpkg-internal-e2e-test-port.json" `
+    -ItemType File `
+    -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgInternalE2eTestPortJson)
+
+
+# build a git registry
+New-Item -Path $gitRegistryUpstream -ItemType Directory
+$gitRegistryUpstream = (Get-Item $gitRegistryUpstream).FullName
+
+Push-Location $gitRegistryUpstream
+try
+{
+    $gitConfigOptions = @(
+        '-c', 'user.name=Nobody',
+        '-c', 'user.email=nobody@example.com',
+        '-c', 'core.autocrlf=false'
+    )
+
+    $CurrentTest = 'git init .'
+    git @gitConfigOptions init .
+    Throw-IfFailed
+    Copy-Item -Recurse -LiteralPath "$PSScriptRoot/../../e2e_ports/port_versions" -Destination .
+    Copy-Item -Recurse -LiteralPath "$PSScriptRoot/../../e2e_ports/vcpkg-internal-e2e-test-port" -Destination .
+
+    $CurrentTest = 'git add -A'
+    git @gitConfigOptions add -A
+    Throw-IfFailed
+    $CurrentTest = 'git commit'
+    git @gitConfigOptions commit -m 'initial commit'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}
+
+# actually test the registries
+$vcpkgJson = @{
+    "name" = "manifest-test";
+    "version-string" = "1.0.0";
+    "dependencies" = @(
+        "vcpkg-internal-e2e-test-port"
+    )
+}
+
+$manifestDir = "$TestingRoot/builtin-registry-test-manifest-dir"
+
+New-Item -Path $manifestDir -ItemType Directory
+$manifestDir = (Get-Item $manifestDir).FullName
+
+Push-Location $manifestDir
+
+try
+{
+    $vcpkgJsonWithBaseline = $vcpkgJson.Clone()
+    $vcpkgJsonWithBaseline['$x-default-baseline'] = 'default'
+
+    New-Item -Path 'vcpkg.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfNotFailed
+
+    New-Item -Path 'vcpkg.json' -ItemType File -Force `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJsonWithBaseline)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}
+
+
+# test the filesystem registry
+$manifestDir = "$TestingRoot/filesystem-registry-test-manifest-dir"
+
+New-Item -Path $manifestDir -ItemType Directory
+$manifestDir = (Get-Item $manifestDir).FullName
+
+Push-Location $manifestDir
+try
+{
+    New-Item -Path 'vcpkg.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson)
+
+    $vcpkgConfigurationJson = @{
+        "default-registry" = $null;
+        "registries" = @(
+            @{
+                "kind" = "filesystem";
+                "path" = $filesystemRegistry;
+                "packages" = @( "vcpkg-internal-e2e-test-port" )
+            }
+        )
+    }
+    New-Item -Path 'vcpkg-configuration.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}
+
+# test the git registry
+$manifestDir = "$TestingRoot/git-registry-test-manifest-dir"
+
+New-Item -Path $manifestDir -ItemType Directory
+$manifestDir = (Get-Item $manifestDir).FullName
+
+Push-Location $manifestDir
+try
+{
+    New-Item -Path 'vcpkg.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson)
+
+    $vcpkgConfigurationJson = @{
+        "default-registry" = $null;
+        "registries" = @(
+            @{
+                "kind" = "git";
+                "repository" = $gitRegistryUpstream;
+                "packages" = @( "vcpkg-internal-e2e-test-port" )
+            }
+        )
+    }
+    New-Item -Path 'vcpkg-configuration.json' -ItemType File `
+        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson)
+
+    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Throw-IfFailed
+}
+finally
+{
+    Pop-Location
+}

scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1

@@ -69,7 +69,7 @@ Throw-IfNotFailed
 
 git fetch https://github.com/vicroms/test-registries
 $CurrentTest = "default baseline"
-./vcpkg $commonArgs "--feature-flags=versions" install `
+./vcpkg $commonArgs --debug "--feature-flags=versions" install `
     "--x-manifest-root=scripts/testing/version-files/default-baseline-2" `
     "--x-builtin-port-versions-dir=scripts/testing/version-files/default-baseline-2/port_versions"
 Throw-IfFailed

scripts/azure-pipelines/end-to-end-tests-prelude.ps1

@@ -12,10 +12,19 @@ $commonArgs = @(
     "--x-buildtrees-root=$buildtreesRoot",
     "--x-install-root=$installRoot",
     "--x-packages-root=$packagesRoot",
-    "--overlay-ports=scripts/e2e_ports/overlays"
+    "--overlay-ports=$PSScriptRoot/../e2e_ports/overlays"
 )
 $Script:CurrentTest = 'unassigned'
 
+if ($IsWindows)
+{
+    $VcpkgExe = Get-Item './vcpkg.exe'
+}
+else
+{
+    $VcpkgExe = Get-Item './vcpkg'
+}
+
 function Refresh-TestRoot {
     Remove-Item -Recurse -Force $TestingRoot -ErrorAction SilentlyContinue
     mkdir $TestingRoot | Out-Null
@@ -59,17 +68,9 @@ function Run-Vcpkg {
         [Parameter(ValueFromRemainingArguments)]
         [string[]]$TestArgs
     )
-
-    if ($IsWindows) {
-        $vcpkgName = 'vcpkg.exe'
-    } else {
-        $vcpkgName = 'vcpkg'
-    }
-
-    $Script:CurrentTest = "./$vcpkgName $($testArgs -join ' ')"
+    $Script:CurrentTest = "vcpkg $($testArgs -join ' ')"
     Write-Host $Script:CurrentTest
-    & "./$vcpkgName" @testArgs | Tee-Object -Variable 'ConsoleOutput'
-    return $ConsoleOutput
+    & $VcpkgExe @testArgs
 }
 
 Refresh-TestRoot

scripts/azure-pipelines/end-to-end-tests.ps1

@@ -32,6 +32,8 @@ Param(
 
 $ErrorActionPreference = "Stop"
 
+$WorkingRoot = (Get-Item $WorkingRoot).FullName
+
 $AllTests = Get-ChildItem $PSScriptRoot/end-to-end-tests-dir/*.ps1
 if ($Filter -ne $Null) {
     $AllTests = $AllTests | ? { $_.Name -match $Filter }

scripts/buildsystems/vcpkg.cmake

@@ -384,6 +384,11 @@ if(VCPKG_MANIFEST_MODE AND VCPKG_MANIFEST_INSTALL AND NOT _CMAKE_IN_TRY_COMPILE
             endforeach()
         endif()
 
+        if(DEFINED VCPKG_FEATURE_FLAGS OR DEFINED CACHE{VCPKG_FEATURE_FLAGS})
+            list(JOIN VCPKG_FEATURE_FLAGS "," _VCPKG_FEATURE_FLAGS)
+            set(_VCPKG_FEATURE_FLAGS "--feature-flags=${_VCPKG_FEATURE_FLAGS}")
+        endif()
+
         foreach(feature IN LISTS VCPKG_MANIFEST_FEATURES)
             list(APPEND _VCPKG_ADDITIONAL_MANIFEST_PARAMS "--x-feature=${feature}")
         endforeach()
@@ -405,6 +410,7 @@ if(VCPKG_MANIFEST_MODE AND VCPKG_MANIFEST_INSTALL AND NOT _CMAKE_IN_TRY_COMPILE
                 "--x-wait-for-lock"
                 "--x-manifest-root=${_VCPKG_MANIFEST_DIR}"
                 "--x-install-root=${_VCPKG_INSTALLED_DIR}"
+                "${_VCPKG_FEATURE_FLAGS}"
                 ${_VCPKG_ADDITIONAL_MANIFEST_PARAMS}
                 ${VCPKG_INSTALL_OPTIONS}
             OUTPUT_VARIABLE _VCPKG_MANIFEST_INSTALL_LOGTEXT
@@ -422,6 +428,10 @@ if(VCPKG_MANIFEST_MODE AND VCPKG_MANIFEST_INSTALL AND NOT _CMAKE_IN_TRY_COMPILE
             set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
                 "${_VCPKG_MANIFEST_DIR}/vcpkg.json"
                 "${_VCPKG_INSTALLED_DIR}/vcpkg/status")
+            if(EXISTS "${_VCPKG_MANIFEST_DIR}/vcpkg-configuration.json")
+                set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
+                    "${_VCPKG_MANIFEST_DIR}/vcpkg-configuration.json")
+            endif()
         else()
             message(STATUS "Running vcpkg install - failed")
             _vcpkg_add_fatal_error("vcpkg install failed. See logs for more information: ${_VCPKG_MANIFEST_INSTALL_LOGFILE}")

scripts/e2e_ports/port_versions/baseline.json

@@ -1,3 +1,5 @@
 {
-	"vcpkg-internal-e2e-test-port": { "baseline": "1.0.0" }
+    "default": {
+        "vcpkg-internal-e2e-test-port": { "baseline": "1.0.0" }
+    }
 }

toolsrc/CMakeLists.txt

@@ -185,7 +185,7 @@ endif()
 
 # === Target: tls12-download ===
 
-set(TLS12_DOWNLOAD_SOURCES src/tls12-download.c)
+set(TLS12_DOWNLOAD_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/tls12-download.c)
 if(WIN32)
     add_executable(tls12-download ${TLS12_DOWNLOAD_SOURCES})
     set_property(TARGET tls12-download PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded")

toolsrc/include/vcpkg/base/files.h

@@ -125,6 +125,12 @@ namespace fs
 
 #endif
 
+    inline bool operator==(SystemHandle lhs, SystemHandle rhs) noexcept
+    {
+        return lhs.system_handle == rhs.system_handle;
+    }
+    inline bool operator!=(SystemHandle lhs, SystemHandle rhs) noexcept { return !(lhs == rhs); }
+
     inline bool is_symlink(file_status s) noexcept
     {
 #if defined(_WIN32)
@@ -266,4 +272,55 @@ namespace vcpkg::Files
 #if defined(_WIN32)
     fs::path win32_fix_path_case(const fs::path& source);
 #endif // _WIN32
+
+    struct ExclusiveFileLock
+    {
+        enum class Wait
+        {
+            Yes,
+            No,
+        };
+
+        ExclusiveFileLock() = default;
+        ExclusiveFileLock(ExclusiveFileLock&& other)
+            : fs_(other.fs_), handle_(std::exchange(handle_, fs::SystemHandle{}))
+        {
+        }
+        ExclusiveFileLock& operator=(ExclusiveFileLock&& other)
+        {
+            if (this == &other) return *this;
+
+            clear();
+            fs_ = other.fs_;
+            handle_ = std::exchange(other.handle_, fs::SystemHandle{});
+            return *this;
+        }
+
+        ExclusiveFileLock(Wait wait, Filesystem& fs, const fs::path& path_, std::error_code& ec) : fs_(&fs)
+        {
+            switch (wait)
+            {
+                case Wait::Yes: handle_ = fs_->take_exclusive_file_lock(path_, ec); break;
+                case Wait::No: handle_ = fs_->try_take_exclusive_file_lock(path_, ec); break;
+            }
+        }
+        ~ExclusiveFileLock() { clear(); }
+
+        explicit operator bool() const { return handle_.is_valid(); }
+        bool has_lock() const { return handle_.is_valid(); }
+
+        void clear()
+        {
+            if (fs_ && handle_.is_valid())
+            {
+                std::error_code ignore;
+                fs_->unlock_file_lock(std::exchange(handle_, fs::SystemHandle{}), ignore);
+            }
+        }
+
+    private:
+        fs::SystemHandle handle_;
+        Filesystem* fs_;
+    };
+
 }

toolsrc/include/vcpkg/base/system.h

@@ -20,6 +20,8 @@ namespace vcpkg::System
 
     Optional<std::string> get_registry_string(void* base_hkey, StringView subkey, StringView valuename);
 
+    long get_process_id();
+
     enum class CPUArchitecture
     {
         X86,

toolsrc/include/vcpkg/vcpkgpaths.h

@@ -124,6 +124,16 @@ namespace vcpkg
 
         ExpectedS<std::map<std::string, std::string, std::less<>>> git_get_local_port_treeish_map() const;
 
+        // Git manipulation for remote registries
+        // runs `git fetch {uri} {treeish}`, and returns the hash of FETCH_HEAD.
+        // If `treeish` is empty, then just runs `git fetch {uri}`
+        ExpectedS<std::string> git_fetch_from_remote_registry(StringView uri, StringView treeish = {}) const;
+        ExpectedS<std::string> git_show_from_remote_registry(StringView hash,
+                                                             const fs::path& relative_path_to_file) const;
+        ExpectedS<std::string> git_find_object_id_for_remote_registry_path(StringView hash,
+                                                                           const fs::path& relative_path_to_file) const;
+        ExpectedS<fs::path> git_checkout_object_from_remote_registry(StringView tree) const;
+
         Optional<const Json::Object&> get_manifest() const;
         Optional<const fs::path&> get_manifest_path() const;
         const Configuration& get_configuration() const;

toolsrc/src/vcpkg/base/system.cpp

@@ -10,6 +10,15 @@ using namespace vcpkg::System;
 
 namespace vcpkg
 {
+    long System::get_process_id()
+    {
+#ifdef _WIN32
+        return ::_getpid();
+#else
+        return ::getpid();
+#endif
+    }
+
     Optional<CPUArchitecture> System::to_cpu_architecture(StringView arch)
     {
         if (Strings::case_insensitive_ascii_equals(arch, "x86")) return CPUArchitecture::X86;

toolsrc/src/vcpkg/install.cpp

@@ -1,5 +1,6 @@
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/hash.h>
+#include <vcpkg/base/system.debug.h>
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/util.h>
 
@@ -844,15 +845,18 @@ namespace vcpkg::Install
                 features.erase(core_it);
             }
 
-            if (args.versions_enabled())
+            if (args.versions_enabled() || args.registries_enabled())
             {
-                auto verprovider = PortFileProvider::make_versioned_portfile_provider(paths);
-                auto baseprovider = PortFileProvider::make_baseline_provider(paths);
                 if (auto p_baseline = manifest_scf.core_paragraph->extra_info.get("$x-default-baseline"))
                 {
                     paths.get_configuration().registry_set.experimental_set_builtin_registry_baseline(
                         p_baseline->string());
                 }
+            }
+            if (args.versions_enabled())
+            {
+                auto verprovider = PortFileProvider::make_versioned_portfile_provider(paths);
+                auto baseprovider = PortFileProvider::make_baseline_provider(paths);
                 auto oprovider = PortFileProvider::make_overlay_provider(paths, args.overlay_ports);
 
                 auto install_plan =

toolsrc/src/vcpkg/registries.cpp

@@ -25,10 +25,13 @@ namespace
     // when `BuiltinRegistryEntry` is using a port versions file for a port,
     // it uses this as it's underlying type;
     // when `BuiltinRegistryEntry` is using a port tree, it uses the scfl
-    struct GitRegistryEntry
+    struct GitRegistryEntry final : RegistryEntry
     {
         explicit GitRegistryEntry(std::string&& port_name) : port_name(port_name) { }
 
+        View<VersionT> get_port_versions() const override { return port_versions; }
+        ExpectedS<fs::path> get_path_to_version(const VcpkgPaths&, const VersionT& version) const override;
+
         std::string port_name;
 
         // these two map port versions to git trees
@@ -37,6 +40,67 @@ namespace
         std::vector<std::string> git_trees;
     };
 
+    struct GitRegistry final : RegistryImplementation
+    {
+        GitRegistry(std::string&& repo, std::string&& baseline)
+            : m_repo(std::move(repo)), m_baseline_identifier(std::move(baseline))
+        {
+        }
+
+        std::unique_ptr<RegistryEntry> get_port_entry(const VcpkgPaths&, StringView) const override;
+
+        void get_all_port_names(std::vector<std::string>&, const VcpkgPaths&) const override;
+
+        Optional<VersionT> get_baseline_version(const VcpkgPaths&, StringView) const override;
+
+        StringView get_commit_of_repo(const VcpkgPaths& paths) const
+        {
+            return m_commit.get([this, &paths]() -> std::string {
+                auto maybe_hash = paths.git_fetch_from_remote_registry(m_repo);
+                if (!maybe_hash.has_value())
+                {
+                    Checks::exit_with_message(VCPKG_LINE_INFO,
+                                              "Error: Failed to fetch from remote registry `%s`: %s",
+                                              m_repo,
+                                              maybe_hash.error());
+                }
+                return std::move(*maybe_hash.get());
+            });
+        }
+
+        fs::path get_versions_tree_path(const VcpkgPaths& paths) const
+        {
+            return m_versions_tree.get([this, &paths]() -> fs::path {
+                auto maybe_tree = paths.git_find_object_id_for_remote_registry_path(get_commit_of_repo(paths),
+                                                                                    fs::u8path("port_versions"));
+                if (!maybe_tree)
+                {
+                    Checks::exit_with_message(
+                        VCPKG_LINE_INFO,
+                        "Error: could not find the git tree for `port_versions` in repo `%s` at commit `%s`: %s",
+                        m_repo,
+                        get_commit_of_repo(paths),
+                        maybe_tree.error());
+                }
+                auto maybe_path = paths.git_checkout_object_from_remote_registry(*maybe_tree.get());
+                if (!maybe_path)
+                {
+                    Checks::exit_with_message(VCPKG_LINE_INFO,
+                                              "Error: failed to check out `port_versions` from repo %s: %s",
+                                              m_repo,
+                                              maybe_path.error());
+                }
+                return std::move(*maybe_path.get());
+            });
+        }
+
+        std::string m_repo;
+        DelayedInit<std::string> m_commit; // TODO: eventually this should end up in the vcpkg-lock.json file
+        DelayedInit<fs::path> m_versions_tree;
+        std::string m_baseline_identifier;
+        DelayedInit<Baseline> m_baseline;
+    };
+
     struct BuiltinRegistryEntry final : RegistryEntry
     {
         explicit BuiltinRegistryEntry(std::unique_ptr<GitRegistryEntry>&& entry)
@@ -195,8 +259,7 @@ namespace
     std::unique_ptr<RegistryEntry> BuiltinRegistry::get_port_entry(const VcpkgPaths& paths, StringView port_name) const
     {
         auto versions_path = paths.builtin_port_versions / relative_path_to_versions(port_name);
-        if ((paths.get_feature_flags().registries || paths.get_feature_flags().versions) &&
-            paths.get_filesystem().exists(versions_path))
+        if (!m_baseline_identifier.empty() && paths.get_filesystem().exists(versions_path))
         {
             auto maybe_version_entries =
                 load_versions_file(paths.get_filesystem(), VersionDbType::Git, paths.builtin_port_versions, port_name);
@@ -247,6 +310,7 @@ namespace
 
     ExpectedS<Baseline> try_parse_builtin_baseline(const VcpkgPaths& paths, StringView baseline_identifier)
     {
+        if (baseline_identifier.size() == 0) return Baseline{};
         auto path_to_baseline = paths.builtin_port_versions / fs::u8path("baseline.json");
         auto res_baseline = load_baseline_versions(paths, path_to_baseline, baseline_identifier);
 
@@ -261,15 +325,10 @@ namespace
             return std::move(*p);
         }
 
-        if (baseline_identifier.size() == 0)
-        {
-            return {{}, expected_left_tag};
-        }
-
         if (baseline_identifier == "default")
         {
-            return Strings::format("Couldn't find explicitly specified baseline `\"default\"` in the baseline file.",
-                                   baseline_identifier);
+            return Strings::format("Couldn't find explicitly specified baseline `\"default\"` in baseline file: %s",
+                                   fs::u8string(path_to_baseline));
         }
 
         // attempt to check out the baseline:
@@ -305,6 +364,7 @@ namespace
     }
     Optional<VersionT> BuiltinRegistry::get_baseline_version(const VcpkgPaths& paths, StringView port_name) const
     {
+        Debug::print("Baseline version: \"", m_baseline_identifier, "\"\n");
         if (!m_baseline_identifier.empty())
         {
             const auto& baseline = m_baseline.get(
@@ -318,7 +378,7 @@ namespace
         }
         else
         {
-            // fall back to using the ports directory version
+            // if a baseline is not specified, use the ports directory version
             auto maybe_scf = Paragraphs::try_load_port(paths.get_filesystem(),
                                                        paths.builtin_ports_directory() / fs::u8path(port_name));
             if (auto pscf = maybe_scf.get())
@@ -333,8 +393,7 @@ namespace
 
     void BuiltinRegistry::get_all_port_names(std::vector<std::string>& out, const VcpkgPaths& paths) const
     {
-        if ((paths.get_feature_flags().registries || paths.get_feature_flags().versions) &&
-            paths.get_filesystem().exists(paths.builtin_port_versions))
+        if (!m_baseline_identifier.empty() && paths.get_filesystem().exists(paths.builtin_port_versions))
         {
             load_all_port_names_from_port_versions(out, paths, paths.builtin_port_versions);
         }
@@ -415,6 +474,118 @@ namespace
     }
     // } FilesystemRegistry::RegistryImplementation
 
+    // { GitRegistry::RegistryImplementation
+    std::unique_ptr<RegistryEntry> GitRegistry::get_port_entry(const VcpkgPaths& paths, StringView port_name) const
+    {
+        auto port_versions = get_versions_tree_path(paths);
+        auto maybe_version_entries =
+            load_versions_file(paths.get_filesystem(), VersionDbType::Git, port_versions, port_name);
+        Checks::check_exit(
+            VCPKG_LINE_INFO, maybe_version_entries.has_value(), "Error: %s", maybe_version_entries.error());
+        auto version_entries = std::move(maybe_version_entries).value_or_exit(VCPKG_LINE_INFO);
+
+        auto res = std::make_unique<GitRegistryEntry>(port_name.to_string());
+        for (auto&& version_entry : version_entries)
+        {
+            res->port_versions.push_back(version_entry.version);
+            res->git_trees.push_back(version_entry.git_tree);
+        }
+        return res;
+    }
+
+    Optional<VersionT> GitRegistry::get_baseline_version(const VcpkgPaths& paths, StringView port_name) const
+    {
+        const auto& baseline = m_baseline.get([this, &paths]() -> Baseline {
+            auto baseline_file = get_versions_tree_path(paths) / fs::u8path("baseline.json");
+
+            auto res_baseline = load_baseline_versions(paths, baseline_file, m_baseline_identifier);
+
+            if (!res_baseline.has_value())
+            {
+                Checks::exit_with_message(VCPKG_LINE_INFO, res_baseline.error());
+            }
+            auto opt_baseline = res_baseline.get();
+            if (auto p = opt_baseline->get())
+            {
+                return std::move(*p);
+            }
+
+            if (m_baseline_identifier.empty())
+            {
+                return {};
+            }
+
+            if (m_baseline_identifier == "default")
+            {
+                Checks::exit_with_message(
+                    VCPKG_LINE_INFO,
+                    "Couldn't find explicitly specified baseline `\"default\"` in the baseline file.",
+                    m_baseline_identifier);
+            }
+
+            // attempt to check out the baseline:
+            auto explicit_hash = paths.git_fetch_from_remote_registry(m_repo, m_baseline_identifier);
+            if (!explicit_hash.has_value())
+            {
+                Checks::exit_with_message(
+                    VCPKG_LINE_INFO,
+                    "Error: Couldn't find explicitly specified baseline `\"%s\"` in the baseline file for repo %s, "
+                    "and that commit doesn't exist.\n%s",
+                    m_baseline_identifier,
+                    m_repo,
+                    explicit_hash.error());
+            }
+            auto path_to_baseline = fs::u8path("port_versions") / fs::u8path("baseline.json");
+            auto maybe_contents = paths.git_show_from_remote_registry(*explicit_hash.get(), path_to_baseline);
+            if (!maybe_contents.has_value())
+            {
+                Checks::exit_with_message(
+                    VCPKG_LINE_INFO,
+                    "Error: Couldn't find explicitly specified baseline `\"%s\"` in the baseline file for repo %s, "
+                    "and the baseline file doesn't exist at that commit.\n%s\n",
+                    m_baseline_identifier,
+                    m_repo,
+                    maybe_contents.error());
+            }
+
+            auto contents = maybe_contents.get();
+            res_baseline = parse_baseline_versions(*contents, {});
+            if (!res_baseline.has_value())
+            {
+                Checks::exit_with_message(VCPKG_LINE_INFO, res_baseline.error());
+            }
+            opt_baseline = res_baseline.get();
+            if (auto p = opt_baseline->get())
+            {
+                return std::move(*p);
+            }
+            else
+            {
+                Checks::exit_with_message(
+                    VCPKG_LINE_INFO,
+                    "Couldn't find explicitly specified baseline `\"%s\"` in the baseline file for repo %s, "
+                    "and the `\"default\"` baseline does not exist at that commit.",
+                    m_baseline_identifier,
+                    m_repo);
+            }
+        });
+
+        auto it = baseline.find(port_name);
+        if (it != baseline.end())
+        {
+            return it->second;
+        }
+
+        return nullopt;
+    }
+
+    void GitRegistry::get_all_port_names(std::vector<std::string>& out, const VcpkgPaths& paths) const
+    {
+        auto versions_path = get_versions_tree_path(paths);
+        load_all_port_names_from_port_versions(out, paths, versions_path);
+    }
+    // } GitRegistry::RegistryImplementation
+
     // } RegistryImplementation
 
     // { RegistryEntry
@@ -464,6 +635,20 @@ namespace
     }
     // } FilesystemRegistryEntry::RegistryEntry
 
+    // { GitRegistryEntry::RegistryEntry
+    ExpectedS<fs::path> GitRegistryEntry::get_path_to_version(const VcpkgPaths& paths, const VersionT& version) const
+    {
+        auto it = std::find(port_versions.begin(), port_versions.end(), version);
+        if (it == port_versions.end())
+        {
+            return Strings::concat("Error: No version entry for ", port_name, " at version ", version, ".");
+        }
+
+        const auto& git_tree = git_trees[it - port_versions.begin()];
+        return paths.git_checkout_object_from_remote_registry(git_tree);
+    }
+    // } GitRegistryEntry::RegistryEntry
+
     // } RegistryEntry
 }
 
@@ -601,9 +786,11 @@ namespace
         constexpr static StringLiteral KIND = "kind";
         constexpr static StringLiteral BASELINE = "baseline";
         constexpr static StringLiteral PATH = "path";
+        constexpr static StringLiteral REPO = "repository";
 
         constexpr static StringLiteral KIND_BUILTIN = "builtin";
         constexpr static StringLiteral KIND_FILESYSTEM = "filesystem";
+        constexpr static StringLiteral KIND_GIT = "git";
 
         virtual StringView type_name() const override { return "a registry"; }
         virtual View<StringView> valid_fields() const override;
@@ -620,8 +807,10 @@ namespace
     constexpr StringLiteral RegistryImplDeserializer::KIND;
     constexpr StringLiteral RegistryImplDeserializer::BASELINE;
     constexpr StringLiteral RegistryImplDeserializer::PATH;
+    constexpr StringLiteral RegistryImplDeserializer::REPO;
     constexpr StringLiteral RegistryImplDeserializer::KIND_BUILTIN;
     constexpr StringLiteral RegistryImplDeserializer::KIND_FILESYSTEM;
+    constexpr StringLiteral RegistryImplDeserializer::KIND_GIT;
 
     struct RegistryDeserializer final : Json::IDeserializer<Registry>
     {
@@ -640,7 +829,7 @@ namespace
 
     View<StringView> RegistryImplDeserializer::valid_fields() const
     {
-        static const StringView t[] = {KIND, BASELINE, PATH};
+        static const StringView t[] = {KIND, BASELINE, PATH, REPO};
         return t;
     }
     View<StringView> valid_builtin_fields()
@@ -662,6 +851,16 @@ namespace
         };
         return t;
     }
+    View<StringView> valid_git_fields()
+    {
+        static const StringView t[] = {
+            RegistryImplDeserializer::KIND,
+            RegistryImplDeserializer::BASELINE,
+            RegistryImplDeserializer::REPO,
+            RegistryDeserializer::PACKAGES,
+        };
+        return t;
+    }
 
     Optional<std::unique_ptr<RegistryImplementation>> RegistryImplDeserializer::visit_null(Json::Reader&)
     {
@@ -695,9 +894,19 @@ namespace
 
             res = std::make_unique<FilesystemRegistry>(config_directory / path, std::move(baseline));
         }
+        else if (kind == KIND_GIT)
+        {
+            r.check_for_unexpected_fields(obj, valid_git_fields(), "a git registry");
+
+            std::string repo;
+            Json::StringDeserializer repo_des{"a git repository URL"};
+            r.required_object_field("a git registry", obj, REPO, repo, repo_des);
+
+            res = std::make_unique<GitRegistry>(std::move(repo), std::move(baseline));
+        }
         else
         {
-            StringLiteral valid_kinds[] = {KIND_BUILTIN, KIND_FILESYSTEM};
+            StringLiteral valid_kinds[] = {KIND_BUILTIN, KIND_FILESYSTEM, KIND_GIT};
             r.add_generic_error(type_name(),
                                 "Field \"kind\" did not have an expected value (expected one of: \"",
                                 Strings::join("\", \"", valid_kinds),
@@ -716,6 +925,7 @@ namespace
             RegistryImplDeserializer::KIND,
             RegistryImplDeserializer::BASELINE,
             RegistryImplDeserializer::PATH,
+            RegistryImplDeserializer::REPO,
             PACKAGES,
         };
         return t;
@@ -853,6 +1063,7 @@ namespace
         }
         else if (maybe_contents.error() == std::errc::no_such_file_or_directory)
         {
+            Debug::print("Failed to find baseline.json\n");
             return {nullopt, expected_left_tag};
         }
         else

toolsrc/src/vcpkg/vcpkgpaths.cpp

@@ -202,6 +202,11 @@ namespace vcpkg
                 , m_env_cache(ff_settings.compiler_tracking)
                 , m_ff_settings(ff_settings)
             {
+                const auto& cache_root =
+                    System::get_platform_cache_home().value_or_exit(VCPKG_LINE_INFO) / fs::u8path("vcpkg");
+                registries_work_tree_dir = cache_root / fs::u8path("registries") / fs::u8path("git");
+                registries_dot_git_dir = registries_work_tree_dir / fs::u8path(".git");
+                registries_git_trees = cache_root / fs::u8path("registries") / fs::u8path("git-trees");
             }
 
             Lazy<std::vector<VcpkgPaths::TripletFile>> available_triplets;
@@ -224,6 +229,10 @@ namespace vcpkg
             Configuration m_config;
 
             FeatureFlagSettings m_ff_settings;
+
+            fs::path registries_work_tree_dir;
+            fs::path registries_dot_git_dir;
+            fs::path registries_git_trees;
         };
     }
 
@@ -726,6 +735,156 @@ If you wish to silence this error and use classic mode, you can:
         return destination;
     }
 
+    ExpectedS<std::string> VcpkgPaths::git_fetch_from_remote_registry(StringView repo, StringView treeish) const
+    {
+        auto& fs = get_filesystem();
+
+        auto work_tree = m_pimpl->registries_work_tree_dir;
+        fs.create_directories(work_tree, VCPKG_LINE_INFO);
+        auto dot_git_dir = m_pimpl->registries_dot_git_dir;
+
+        System::CmdLineBuilder init_registries_git_dir =
+            git_cmd_builder(*this, dot_git_dir, work_tree).string_arg("init");
+        auto init_output = System::cmd_execute_and_capture_output(init_registries_git_dir);
+        if (init_output.exit_code != 0)
+        {
+            return {Strings::format("Error: Failed to initialize local repository %s.\n%s\n",
+                                    fs::u8string(work_tree),
+                                    init_output.output),
+                    expected_right_tag};
+        }
+
+        auto lock_file = work_tree / fs::u8path(".vcpkg-lock");
+
+        std::error_code ec;
+        auto guard = Files::ExclusiveFileLock(Files::ExclusiveFileLock::Wait::Yes, fs, lock_file, ec);
+
+        System::CmdLineBuilder fetch_git_ref =
+            git_cmd_builder(*this, dot_git_dir, work_tree).string_arg("fetch").string_arg("--").string_arg(repo);
+        if (treeish.size() != 0)
+        {
+            fetch_git_ref.string_arg(treeish);
+        }
+
+        auto fetch_output = System::cmd_execute_and_capture_output(fetch_git_ref);
+        if (fetch_output.exit_code != 0)
+        {
+            return {Strings::format("Error: Failed to fetch %s%s from repository %s.\n%s\n",
+                                    treeish.size() != 0 ? "ref " : "",
+                                    treeish,
+                                    repo,
+                                    fetch_output.output),
+                    expected_right_tag};
+        }
+
+        System::CmdLineBuilder get_fetch_head =
+            git_cmd_builder(*this, dot_git_dir, work_tree).string_arg("rev-parse").string_arg("FETCH_HEAD");
+        auto fetch_head_output = System::cmd_execute_and_capture_output(get_fetch_head);
+        if (fetch_head_output.exit_code != 0)
+        {
+            return {Strings::format("Error: Failed to rev-parse FETCH_HEAD.\n%s\n", fetch_head_output.output),
+                    expected_right_tag};
+        }
+        return {Strings::trim(fetch_head_output.output).to_string(), expected_left_tag};
+    }
+    // returns an error if there was an unexpected error; returns nullopt if the file doesn't exist at the specified
+    // hash
+    ExpectedS<std::string> VcpkgPaths::git_show_from_remote_registry(StringView hash,
+                                                                     const fs::path& relative_path) const
+    {
+        auto revision = Strings::format("%s:%s", hash, fs::generic_u8string(relative_path));
+        System::CmdLineBuilder git_show =
+            git_cmd_builder(*this, m_pimpl->registries_dot_git_dir, m_pimpl->registries_work_tree_dir)
+                .string_arg("show")
+                .string_arg(revision);
+
+        auto git_show_output = System::cmd_execute_and_capture_output(git_show);
+        if (git_show_output.exit_code != 0)
+        {
+            return {git_show_output.output, expected_right_tag};
+        }
+        return {git_show_output.output, expected_left_tag};
+    }
+    ExpectedS<std::string> VcpkgPaths::git_find_object_id_for_remote_registry_path(StringView hash,
+                                                                                   const fs::path& relative_path) const
+    {
+        auto revision = Strings::format("%s:%s", hash, fs::generic_u8string(relative_path));
+        System::CmdLineBuilder git_rev_parse =
+            git_cmd_builder(*this, m_pimpl->registries_dot_git_dir, m_pimpl->registries_work_tree_dir)
+                .string_arg("rev-parse")
+                .string_arg(revision);
+
+        auto git_rev_parse_output = System::cmd_execute_and_capture_output(git_rev_parse);
+        if (git_rev_parse_output.exit_code != 0)
+        {
+            return {git_rev_parse_output.output, expected_right_tag};
+        }
+        return {Strings::trim(git_rev_parse_output.output).to_string(), expected_left_tag};
+    }
+    ExpectedS<fs::path> VcpkgPaths::git_checkout_object_from_remote_registry(StringView object) const
+    {
+        auto& fs = get_filesystem();
+        fs.create_directories(m_pimpl->registries_git_trees, VCPKG_LINE_INFO);
+
+        auto git_tree_final = m_pimpl->registries_git_trees / fs::u8path(object);
+        if (fs.exists(git_tree_final))
+        {
+            return std::move(git_tree_final);
+        }
+
+        auto pid = System::get_process_id();
+
+        fs::path git_tree_temp = fs::u8path(Strings::format("%s.tmp%ld", fs::u8string(git_tree_final), pid));
+        fs::path git_tree_temp_tar = fs::u8path(Strings::format("%s.tmp%ld.tar", fs::u8string(git_tree_final), pid));
+        fs.remove_all(git_tree_temp, VCPKG_LINE_INFO);
+        fs.create_directory(git_tree_temp, VCPKG_LINE_INFO);
+
+        auto dot_git_dir = m_pimpl->registries_dot_git_dir;
+        System::CmdLineBuilder git_archive = git_cmd_builder(*this, dot_git_dir, m_pimpl->registries_work_tree_dir)
+                                                 .string_arg("archive")
+                                                 .string_arg("--format")
+                                                 .string_arg("tar")
+                                                 .string_arg(object)
+                                                 .string_arg("--output")
+                                                 .path_arg(git_tree_temp_tar);
+        auto git_archive_output = System::cmd_execute_and_capture_output(git_archive);
+        if (git_archive_output.exit_code != 0)
+        {
+            return {Strings::format("git archive failed with message:\n%s", git_archive_output.output),
+                    expected_right_tag};
+        }
+
+        auto untar = System::CmdLineBuilder{get_tool_exe(Tools::CMAKE)}
+                         .string_arg("-E")
+                         .string_arg("tar")
+                         .string_arg("xf")
+                         .path_arg(git_tree_temp_tar);
+
+        auto untar_output = System::cmd_execute_and_capture_output(untar, System::InWorkingDirectory{git_tree_temp});
+        if (untar_output.exit_code != 0)
+        {
+            return {Strings::format("cmake's untar failed with message:\n%s", untar_output.output), expected_right_tag};
+        }
+
+        std::error_code ec;
+        fs.rename(git_tree_temp, git_tree_final, ec);
+
+        if (fs.exists(git_tree_final))
+        {
+            return git_tree_final;
+        }
+        if (ec)
+        {
+            return {
+                Strings::format("rename to %s failed with message:\n%s", fs::u8string(git_tree_final), ec.message()),
+                expected_right_tag};
+        }
+        else
+        {
+            return {"Unknown error", expected_right_tag};
+        }
+    }
+
     Optional<const Json::Object&> VcpkgPaths::get_manifest() const
     {
         if (auto p = m_pimpl->m_manifest_doc.get())