[vcpkg] Implement VersionedPortfileProvider and BaselineProvider (#14123)

* WIP: Get versions from database files

* Fix formatting

* Provider inherits ResourceBase

* Correct versions JSON file location

* Fix formatting

* Fix formatting

* Fix include in versions.h

* Fetch port versions using git tree object

* Undo changes to x-history

* Remove unnecesary moves

Co-authored-by: nicole mazzuca <mazzucan@outlook.com>

* Extract Git manipulation code

* [WIP] Review comments

* [WIP] Review comments pt. 2

* [WIP] Review comments / fix formatting

* Generate baseline.json

* Extract deserializers from registries source file

* BaselineProvider initial implementation

* Modify gitignore

* Update .gitignore again

* Use JSON deserializer for versions db

* Lazy load baseline file

* Fetch baseline.json from baseline commit

* More git abstractions

* Clean up code

* Path helpers

* Formatting

* Move data into impl object

* Use implementation object for VersionedPortfileProvider

* Reuse cloned instance for checkouts

* Code cleanup and formatting

* Fix returning dangling reference

* Prepare to remove files in port_versions/

* Remove files in port_versions/

* Update .gitignore

* Some PR review comments

* Use StringView

* More StringView conversions

* More refactoring

* Make some implementation members private

* Functions for parsing baseline and version files

* Hide deserializers implementation

* Check for `versions` feature flag in registries.

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: nicole mazzuca <mazzucan@outlook.com>
This commit is contained in:
Victor Romero 2020-11-27 05:44:21 -08:00 committed by GitHub
parent 62fe6ffbbb
commit 6c9cda1635
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 832 additions and 87 deletions

2
.gitignore vendored
View File

@ -295,6 +295,8 @@ __pycache__/
/toolsrc/windows-bootstrap/msbuild.x86.release/
/toolsrc/windows-bootstrap/msbuild.x64.debug/
/toolsrc/windows-bootstrap/msbuild.x64.release/
#ignore db
/port_versions/
#ignore custom triplets
/triplets/*
#add vcpkg-designed triplets back in

View File

@ -4,6 +4,7 @@ import sys
import subprocess
import json
import time
import shutil
from subprocess import CalledProcessError
from json.decoder import JSONDecodeError
@ -14,9 +15,9 @@ SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
def get_current_git_ref():
output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'],
capture_output=True,
encoding='utf-8')
output = subprocess.run(['git', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'],
capture_output=True,
encoding='utf-8')
if output.returncode == 0:
return output.stdout.strip()
print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr)
@ -25,47 +26,89 @@ def get_current_git_ref():
def generate_port_versions_db(ports_path, db_path, revision):
start_time = time.time()
port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))]
# Assume each directory in ${VCPKG_ROOT}/ports is a different port
port_names = [item for item in os.listdir(
ports_path) if os.path.isdir(os.path.join(ports_path, item))]
port_names.sort()
total_count = len(port_names)
# Dictionary to collect the latest version of each port as baseline
baseline_objects = {}
baseline_objects['default'] = {}
for counter, port_name in enumerate(port_names):
containing_dir = os.path.join(db_path, f'{port_name[0]}-')
os.makedirs(containing_dir, exist_ok=True)
output_filepath = os.path.join(containing_dir, f'{port_name}.json')
if not os.path.exists(output_filepath):
if not os.path.exists(output_filepath):
output = subprocess.run(
[os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'],
[os.path.join(SCRIPT_DIRECTORY, '../vcpkg'),
'x-history', port_name, '--x-json'],
capture_output=True, encoding='utf-8')
if output.returncode == 0:
try:
versions_object = json.loads(output.stdout)
# Put latest version in baseline dictionary
latest_version = versions_object["versions"][0]
baseline_objects['default'][port_name] = {
"version-string": latest_version["version-string"],
"port-version": latest_version["port-version"]
}
with open(output_filepath, 'w') as output_file:
json.dump(versions_object, output_file)
except JSONDecodeError:
print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
print(
f'Malformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
else:
print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr)
print(f'x-history {port_name} failed: ',
output.stdout.strip(), file=sys.stderr)
# This should be replaced by a progress bar
if counter > 0 and counter % 100 == 0:
elapsed_time = time.time() - start_time
print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
print(
f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
# Generate baseline.json
baseline_file_path = os.path.join(db_path, 'baseline.json')
with open(baseline_file_path, 'w') as baseline_output_file:
json.dump(baseline_objects, baseline_output_file)
# Generate timestamp
rev_file = os.path.join(db_path, revision)
Path(rev_file).touch()
elapsed_time = time.time() - start_time
print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
print(
f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
def main(ports_path, db_path):
revision = get_current_git_ref()
if not revision:
print('Couldn\'t fetch current Git revision', file=sys.stderr)
sys.exit(1)
rev_file = os.path.join(db_path, revision)
if os.path.exists(rev_file):
print(f'Database files already exist for commit {revision}')
sys.exit(0)
generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision)
if (os.path.exists(db_path)):
try:
shutil.rmtree(db_path)
except OSError as e:
print(f'Could not delete folder: {db_path}.\nError: {e.strerror}')
generate_port_versions_db(ports_path=ports_path,
db_path=db_path,
revision=revision)
if __name__ == "__main__":
main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'),
db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions'))
main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'),
db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions'))

View File

@ -23,7 +23,7 @@ namespace vcpkg::Json
Optional<Type> visit(Reader&, const Value&);
Optional<Type> visit(Reader&, const Object&);
protected:
public:
virtual Optional<Type> visit_null(Reader&);
virtual Optional<Type> visit_boolean(Reader&, bool);
virtual Optional<Type> visit_integer(Reader& r, int64_t i);
@ -33,6 +33,7 @@ namespace vcpkg::Json
virtual Optional<Type> visit_object(Reader&, const Object&);
virtual View<StringView> valid_fields() const;
protected:
IDeserializer() = default;
IDeserializer(const IDeserializer&) = default;
IDeserializer& operator=(const IDeserializer&) = default;

View File

@ -6,6 +6,7 @@
#include <vcpkg/base/util.h>
#include <vcpkg/sourceparagraph.h>
#include <vcpkg/versions.h>
namespace vcpkg::PortFileProvider
{
@ -36,4 +37,48 @@ namespace vcpkg::PortFileProvider
std::vector<fs::path> overlay_ports;
mutable std::unordered_map<std::string, SourceControlFileLocation> cache;
};
struct IVersionedPortfileProvider
{
virtual const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const = 0;
virtual ExpectedS<const SourceControlFileLocation&> get_control_file(
const vcpkg::Versions::VersionSpec& version_spec) const = 0;
};
struct IBaselineProvider
{
virtual Optional<VersionT> get_baseline_version(StringView port_name) const = 0;
};
namespace details
{
struct BaselineProviderImpl;
struct VersionedPortfileProviderImpl;
}
struct VersionedPortfileProvider : IVersionedPortfileProvider, Util::ResourceBase
{
explicit VersionedPortfileProvider(const vcpkg::VcpkgPaths& paths);
~VersionedPortfileProvider();
const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const override;
ExpectedS<const SourceControlFileLocation&> get_control_file(
const vcpkg::Versions::VersionSpec& version_spec) const override;
private:
std::unique_ptr<details::VersionedPortfileProviderImpl> m_impl;
};
struct BaselineProvider : IBaselineProvider, Util::ResourceBase
{
explicit BaselineProvider(const vcpkg::VcpkgPaths& paths, const std::string& baseline);
~BaselineProvider();
Optional<VersionT> get_baseline_version(StringView port_name) const override;
private:
std::unique_ptr<details::BaselineProviderImpl> m_impl;
};
}

View File

@ -102,11 +102,23 @@ namespace vcpkg
fs::path vcpkg_dir_info;
fs::path vcpkg_dir_updates;
fs::path baselines_dot_git_dir;
fs::path baselines_work_tree;
fs::path baselines_output;
fs::path versions_dot_git_dir;
fs::path versions_work_tree;
fs::path versions_output;
fs::path ports_cmake;
const fs::path& get_tool_exe(const std::string& tool) const;
const std::string& get_tool_version(const std::string& tool) const;
// Git manipulation
fs::path git_checkout_baseline(Files::Filesystem& filesystem, StringView commit_sha) const;
fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const;
Optional<const Json::Object&> get_manifest() const;
Optional<const fs::path&> get_manifest_path() const;
const Configuration& get_configuration() const;
@ -133,5 +145,20 @@ namespace vcpkg
private:
std::unique_ptr<details::VcpkgPathsImpl> m_pimpl;
static void git_checkout_subpath(const VcpkgPaths& paths,
StringView commit_sha,
const fs::path& subpath,
const fs::path& local_repo,
const fs::path& destination,
const fs::path& dot_git_dir,
const fs::path& work_tree);
static void git_checkout_object(const VcpkgPaths& paths,
StringView git_object,
const fs::path& local_repo,
const fs::path& destination,
const fs::path& dot_git_dir,
const fs::path& work_tree);
};
}

View File

@ -0,0 +1,37 @@
#pragma once
#include <vcpkg/base/fwd/stringview.h>
#include <vcpkg/base/jsonreader.h>
#include <vcpkg/base/stringliteral.h>
#include <vcpkg/versions.h>
#include <vcpkg/versiont.h>
namespace vcpkg
{
struct VersionDbEntry
{
VersionT version;
Versions::Scheme scheme;
std::string git_tree;
VersionDbEntry(const std::string& version_string,
int port_version,
Versions::Scheme scheme,
const std::string& git_tree)
: version(VersionT(version_string, port_version)), scheme(scheme), git_tree(git_tree)
{
}
};
Json::IDeserializer<VersionT>& get_versiont_deserializer_instance();
ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
StringView baseline_name,
const fs::path& baseline_file_path);
ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
StringView port_name,
const fs::path& versions_file_path);
}

View File

@ -1,5 +1,7 @@
#pragma once
#include <vcpkg/versiont.h>
namespace vcpkg::Versions
{
enum class Scheme
@ -10,6 +12,25 @@ namespace vcpkg::Versions
String
};
struct VersionSpec
{
std::string port_name;
VersionT version;
Scheme scheme;
VersionSpec(const std::string& port_name, const VersionT& version, Scheme scheme);
VersionSpec(const std::string& port_name, const std::string& version_string, int port_version, Scheme scheme);
friend bool operator==(const VersionSpec& lhs, const VersionSpec& rhs);
friend bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs);
};
struct VersionSpecHasher
{
std::size_t operator()(const VersionSpec& key) const;
};
struct Constraint
{
enum class Type

View File

@ -1,3 +1,6 @@
#include <vcpkg/base/json.h>
#include <vcpkg/base/system.debug.h>
#include <vcpkg/configuration.h>
@ -7,6 +10,33 @@
#include <vcpkg/sourceparagraph.h>
#include <vcpkg/vcpkgcmdarguments.h>
#include <vcpkg/vcpkgpaths.h>
#include <vcpkg/versiondeserializers.h>
#include <regex>
using namespace vcpkg;
using namespace Versions;
namespace
{
Optional<fs::path> get_versions_json_path(const VcpkgPaths& paths, StringView port_name)
{
const auto port_versions_dir_path = paths.root / fs::u8path("port_versions");
const auto subpath = Strings::concat(port_name.substr(0, 1), "-/", port_name, ".json");
const auto json_path = port_versions_dir_path / subpath;
if (paths.get_filesystem().exists(json_path))
{
return json_path;
}
return nullopt;
}
Optional<fs::path> get_baseline_json_path(const VcpkgPaths& paths, StringView baseline_commit_sha)
{
const auto baseline_json = paths.git_checkout_baseline(paths.get_filesystem(), baseline_commit_sha);
return paths.get_filesystem().exists(baseline_json) ? make_optional(baseline_json) : nullopt;
}
}
namespace vcpkg::PortFileProvider
{
@ -27,7 +57,7 @@ namespace vcpkg::PortFileProvider
return Util::fmap(ports, [](auto&& kvpair) -> const SourceControlFileLocation* { return &kvpair.second; });
}
PathsPortFileProvider::PathsPortFileProvider(const vcpkg::VcpkgPaths& paths_,
PathsPortFileProvider::PathsPortFileProvider(const VcpkgPaths& paths_,
const std::vector<std::string>& overlay_ports_)
: paths(paths_)
{
@ -80,7 +110,7 @@ namespace vcpkg::PortFileProvider
}
else
{
vcpkg::print_error_message(maybe_scf.error());
print_error_message(maybe_scf.error());
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir));
}
@ -106,7 +136,7 @@ namespace vcpkg::PortFileProvider
}
else
{
vcpkg::print_error_message(found_scf.error());
print_error_message(found_scf.error());
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir));
}
@ -148,7 +178,7 @@ namespace vcpkg::PortFileProvider
}
else
{
vcpkg::print_error_message(found_scf.error());
print_error_message(found_scf.error());
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(port_directory));
}
@ -224,7 +254,7 @@ namespace vcpkg::PortFileProvider
}
else
{
vcpkg::print_error_message(maybe_scf.error());
print_error_message(maybe_scf.error());
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: Failed to load port from %s", fs::u8string(ports_dir));
}
@ -257,4 +287,154 @@ namespace vcpkg::PortFileProvider
return ret;
}
namespace details
{
struct BaselineProviderImpl
{
BaselineProviderImpl(const VcpkgPaths& paths, const std::string& baseline)
: paths(paths), baseline(baseline)
{
}
~BaselineProviderImpl() { }
const std::map<std::string, VersionT, std::less<>>& get_baseline_cache() const
{
return baseline_cache.get_lazy([&]() -> auto {
auto maybe_baseline_file = get_baseline_json_path(paths, baseline);
Checks::check_exit(VCPKG_LINE_INFO, maybe_baseline_file.has_value(), "Couldn't find baseline.json");
auto baseline_file = maybe_baseline_file.value_or_exit(VCPKG_LINE_INFO);
auto maybe_baselines_map = parse_baseline_file(paths.get_filesystem(), "default", baseline_file);
Checks::check_exit(VCPKG_LINE_INFO,
maybe_baselines_map.has_value(),
"Error: Couldn't parse baseline `%s` from `%s`",
"default",
fs::u8string(baseline_file));
auto baselines_map = *maybe_baselines_map.get();
return std::move(baselines_map);
});
}
private:
const VcpkgPaths& paths;
const std::string baseline;
Lazy<std::map<std::string, VersionT, std::less<>>> baseline_cache;
};
struct VersionedPortfileProviderImpl
{
std::map<std::string, std::vector<VersionSpec>> versions_cache;
std::unordered_map<VersionSpec, std::string, VersionSpecHasher> git_tree_cache;
std::unordered_map<VersionSpec, SourceControlFileLocation, VersionSpecHasher> control_cache;
VersionedPortfileProviderImpl(const VcpkgPaths& paths) : paths(paths) { }
~VersionedPortfileProviderImpl() { }
const VcpkgPaths& get_paths() const { return paths; }
Files::Filesystem& get_filesystem() const { return paths.get_filesystem(); }
private:
const VcpkgPaths& paths;
};
}
VersionedPortfileProvider::VersionedPortfileProvider(const VcpkgPaths& paths)
: m_impl(std::make_unique<details::VersionedPortfileProviderImpl>(paths))
{
}
VersionedPortfileProvider::~VersionedPortfileProvider() { }
const std::vector<VersionSpec>& VersionedPortfileProvider::get_port_versions(StringView port_name) const
{
auto cache_it = m_impl->versions_cache.find(port_name.to_string());
if (cache_it != m_impl->versions_cache.end())
{
return cache_it->second;
}
auto maybe_versions_file_path = get_versions_json_path(m_impl->get_paths(), port_name);
Checks::check_exit(VCPKG_LINE_INFO,
maybe_versions_file_path.has_value(),
"Error: Couldn't find a versions database file: %s.json.",
port_name);
auto versions_file_path = maybe_versions_file_path.value_or_exit(VCPKG_LINE_INFO);
auto maybe_version_entries = parse_versions_file(m_impl->get_filesystem(), port_name, versions_file_path);
Checks::check_exit(VCPKG_LINE_INFO,
maybe_version_entries.has_value(),
"Error: Couldn't parse versions from file: %s",
fs::u8string(versions_file_path));
auto version_entries = maybe_version_entries.value_or_exit(VCPKG_LINE_INFO);
auto port = port_name.to_string();
for (auto&& version_entry : version_entries)
{
VersionSpec spec(port, version_entry.version, version_entry.scheme);
m_impl->versions_cache[port].push_back(spec);
m_impl->git_tree_cache.emplace(std::move(spec), std::move(version_entry.git_tree));
}
return m_impl->versions_cache.at(port);
}
ExpectedS<const SourceControlFileLocation&> VersionedPortfileProvider::get_control_file(
const VersionSpec& version_spec) const
{
auto cache_it = m_impl->control_cache.find(version_spec);
if (cache_it != m_impl->control_cache.end())
{
return cache_it->second;
}
// Pre-populate versions cache.
get_port_versions(version_spec.port_name);
auto git_tree_cache_it = m_impl->git_tree_cache.find(version_spec);
if (git_tree_cache_it == m_impl->git_tree_cache.end())
{
return Strings::concat("No git object SHA for entry %s at version %s.",
version_spec.port_name,
version_spec.version.to_string());
}
const std::string git_tree = git_tree_cache_it->second;
auto port_directory =
m_impl->get_paths().git_checkout_port(m_impl->get_filesystem(), version_spec.port_name, git_tree);
auto maybe_control_file = Paragraphs::try_load_port(m_impl->get_filesystem(), port_directory);
if (auto scf = maybe_control_file.get())
{
if (scf->get()->core_paragraph->name == version_spec.port_name)
{
return m_impl->control_cache
.emplace(version_spec, SourceControlFileLocation{std::move(*scf), std::move(port_directory)})
.first->second;
}
return Strings::format("Error: Failed to load port from %s: names did not match: '%s' != '%s'",
fs::u8string(port_directory),
version_spec.port_name,
scf->get()->core_paragraph->name);
}
print_error_message(maybe_control_file.error());
return Strings::format(
"Error: Failed to load port %s from %s", version_spec.port_name, fs::u8string(port_directory));
}
BaselineProvider::BaselineProvider(const VcpkgPaths& paths, const std::string& baseline)
: m_impl(std::make_unique<details::BaselineProviderImpl>(paths, baseline))
{
}
BaselineProvider::~BaselineProvider() { }
Optional<VersionT> BaselineProvider::get_baseline_version(StringView port_name) const
{
const auto& cache = m_impl->get_baseline_cache();
auto it = cache.find(port_name.to_string());
if (it != cache.end())
{
return it->second;
}
return nullopt;
}
}

View File

@ -5,7 +5,9 @@
#include <vcpkg/configurationdeserializer.h>
#include <vcpkg/registries.h>
#include <vcpkg/vcpkgcmdarguments.h>
#include <vcpkg/vcpkgpaths.h>
#include <vcpkg/versiondeserializers.h>
#include <vcpkg/versiont.h>
#include <map>
@ -70,32 +72,6 @@ namespace
}
};
struct VersionTDeserializer final : Json::IDeserializer<VersionT>
{
StringView type_name() const override { return "a version object"; }
View<StringView> valid_fields() const override
{
static const StringView t[] = {"version-string", "port-version"};
return t;
}
Optional<VersionT> visit_object(Json::Reader& r, const Json::Object& obj) override
{
std::string version;
int port_version = 0;
r.required_object_field(type_name(), obj, "version-string", version, version_deserializer);
r.optional_object_field(obj, "port-version", port_version, Json::NaturalNumberDeserializer::instance);
return VersionT{std::move(version), port_version};
}
static Json::StringDeserializer version_deserializer;
static VersionTDeserializer instance;
};
Json::StringDeserializer VersionTDeserializer::version_deserializer{"version"};
VersionTDeserializer VersionTDeserializer::instance;
struct FilesystemVersionEntryDeserializer final : Json::IDeserializer<std::pair<VersionT, fs::path>>
{
StringView type_name() const override { return "a version entry object"; }
@ -109,7 +85,7 @@ namespace
{
fs::path registry_path;
auto version = VersionTDeserializer::instance.visit_object(r, obj);
auto version = get_versiont_deserializer_instance().visit_object(r, obj);
r.required_object_field(
"version entry", obj, "registry-path", registry_path, Json::PathDeserializer::instance);
@ -162,30 +138,6 @@ namespace
const fs::path& registry_root;
};
struct BaselineDeserializer final : Json::IDeserializer<std::map<std::string, VersionT, std::less<>>>
{
StringView type_name() const override { return "a baseline object"; }
Optional<type> visit_object(Json::Reader& r, const Json::Object& obj) override
{
std::map<std::string, VersionT, std::less<>> result;
for (auto pr : obj)
{
const auto& version_value = pr.second;
VersionT version;
r.visit_in_key(version_value, pr.first, version, VersionTDeserializer::instance);
result.emplace(pr.first.to_string(), std::move(version));
}
return std::move(result);
}
static BaselineDeserializer instance;
};
BaselineDeserializer BaselineDeserializer::instance;
struct FilesystemRegistry final : RegistryImpl
{
std::unique_ptr<RegistryEntry> get_port_entry(const VcpkgPaths& paths, StringView port_name) const override
@ -267,6 +219,12 @@ namespace
Optional<VersionT> get_baseline_version(const VcpkgPaths& paths, StringView port_name) const override
{
if (!paths.get_feature_flags().versions)
{
Checks::check_exit(VCPKG_LINE_INFO,
"This invocation failed because the `versions` feature flag is not enabled.");
}
const auto& baseline_cache = baseline.get([this, &paths] { return load_baseline_versions(paths); });
auto it = baseline_cache.find(port_name);
if (it != baseline_cache.end())
@ -310,26 +268,17 @@ namespace
Checks::exit_with_message(VCPKG_LINE_INFO, "Error: `baseline.json` does not have a top-level object.");
}
const auto& obj = value.first.object();
auto baseline_value = obj.get("default");
if (!baseline_value)
auto maybe_baseline_versions = parse_baseline_file(paths.get_filesystem(), "default", baseline_file);
if (auto baseline_versions = maybe_baseline_versions.get())
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: `baseline.json` does not contain the baseline \"%s\"", "default");
}
Json::Reader r;
std::map<std::string, VersionT, std::less<>> result;
r.visit_in_key(*baseline_value, "default", result, BaselineDeserializer::instance);
if (r.errors().empty())
{
return result;
return std::move(*baseline_versions);
}
else
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: failed to parse `baseline.json`:\n%s", Strings::join("\n", r.errors()));
Checks::exit_with_message(VCPKG_LINE_INFO,
"Error: failed to parse `%s`:\n%s",
fs::u8string(baseline_file),
maybe_baseline_versions.error());
}
}

View File

@ -71,6 +71,15 @@ namespace
return result;
}
System::CmdLineBuilder git_cmd_builder(const VcpkgPaths& paths,
const fs::path& dot_git_dir,
const fs::path& work_tree)
{
return System::CmdLineBuilder()
.path_arg(paths.get_tool_exe(Tools::GIT))
.string_arg(Strings::concat("--git-dir=", fs::u8string(dot_git_dir)))
.string_arg(Strings::concat("--work-tree=", fs::u8string(work_tree)));
}
} // unnamed namespace
namespace vcpkg
@ -353,6 +362,18 @@ If you wish to silence this error and use classic mode, you can:
vcpkg_dir_info = vcpkg_dir / fs::u8path("info");
vcpkg_dir_updates = vcpkg_dir / fs::u8path("updates");
// Versioning paths
const auto versioning_tmp = buildtrees / fs::u8path("versioning_tmp");
const auto versioning_output = buildtrees / fs::u8path("versioning");
baselines_dot_git_dir = versioning_tmp / fs::u8path(".baselines.git");
baselines_work_tree = versioning_tmp / fs::u8path("baselines-worktree");
baselines_output = versioning_output / fs::u8path("baselines");
versions_dot_git_dir = versioning_tmp / fs::u8path(".versions.git");
versions_work_tree = versioning_tmp / fs::u8path("versions-worktree");
versions_output = versioning_output / fs::u8path("versions");
ports_cmake = filesystem.canonical(VCPKG_LINE_INFO, scripts / fs::u8path("ports.cmake"));
for (auto&& overlay_triplets_dir : args.overlay_triplets)
@ -456,6 +477,178 @@ If you wish to silence this error and use classic mode, you can:
return m_pimpl->m_tool_cache->get_tool_version(*this, tool);
}
void VcpkgPaths::git_checkout_subpath(const VcpkgPaths& paths,
StringView commit_sha,
const fs::path& subpath,
const fs::path& local_repo,
const fs::path& destination,
const fs::path& dot_git_dir,
const fs::path& work_tree)
{
Files::Filesystem& fs = paths.get_filesystem();
fs.remove_all(work_tree, VCPKG_LINE_INFO);
fs.remove_all(destination, VCPKG_LINE_INFO);
fs.remove_all(dot_git_dir, VCPKG_LINE_INFO);
// All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp}
// git clone --no-checkout --local {vcpkg_root} {dot_git_dir}
System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
.string_arg("clone")
.string_arg("--no-checkout")
.string_arg("--local")
.path_arg(local_repo)
.path_arg(dot_git_dir);
const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract());
Checks::check_exit(VCPKG_LINE_INFO,
clone_output.exit_code == 0,
"Failed to clone temporary vcpkg instance.\n%s\n",
clone_output.output);
// git checkout {commit-sha} -- {subpath}
System::CmdLineBuilder checkout_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
.string_arg("checkout")
.string_arg(commit_sha)
.string_arg("--")
.path_arg(subpath);
const auto checkout_output = System::cmd_execute_and_capture_output(checkout_cmd_builder.extract());
Checks::check_exit(VCPKG_LINE_INFO,
checkout_output.exit_code == 0,
"Error: Failed to checkout %s:%s\n%s\n",
commit_sha,
fs::u8string(subpath),
checkout_output.output);
const fs::path checked_out_path = work_tree / subpath;
const auto& containing_folder = destination.parent_path();
if (!fs.exists(containing_folder))
{
fs.create_directories(containing_folder, VCPKG_LINE_INFO);
}
std::error_code ec;
fs.rename_or_copy(checked_out_path, destination, ".tmp", ec);
fs.remove_all(work_tree, VCPKG_LINE_INFO);
fs.remove_all(dot_git_dir, VCPKG_LINE_INFO);
if (ec)
{
System::printf(System::Color::error,
"Error: Couldn't move checked out files from %s to destination %s",
fs::u8string(checked_out_path),
fs::u8string(destination));
Checks::exit_fail(VCPKG_LINE_INFO);
}
}
void VcpkgPaths::git_checkout_object(const VcpkgPaths& paths,
StringView git_object,
const fs::path& local_repo,
const fs::path& destination,
const fs::path& dot_git_dir,
const fs::path& work_tree)
{
Files::Filesystem& fs = paths.get_filesystem();
fs.remove_all(work_tree, VCPKG_LINE_INFO);
fs.remove_all(destination, VCPKG_LINE_INFO);
if (!fs.exists(dot_git_dir))
{
// All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp}
// git clone --no-checkout --local {vcpkg_root} {dot_git_dir}
System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
.string_arg("clone")
.string_arg("--no-checkout")
.string_arg("--local")
.path_arg(local_repo)
.path_arg(dot_git_dir);
const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract());
Checks::check_exit(VCPKG_LINE_INFO,
clone_output.exit_code == 0,
"Failed to clone temporary vcpkg instance.\n%s\n",
clone_output.output);
}
else
{
System::CmdLineBuilder fetch_cmd_builder =
git_cmd_builder(paths, dot_git_dir, work_tree).string_arg("fetch");
const auto fetch_output = System::cmd_execute_and_capture_output(fetch_cmd_builder.extract());
Checks::check_exit(VCPKG_LINE_INFO,
fetch_output.exit_code == 0,
"Failed to update refs on temporary vcpkg repository.\n%s\n",
fetch_output.output);
}
if (!fs.exists(work_tree))
{
fs.create_directories(work_tree, VCPKG_LINE_INFO);
}
// git checkout {tree_object} .
System::CmdLineBuilder checkout_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
.string_arg("checkout")
.string_arg(git_object)
.string_arg(".");
const auto checkout_output = System::cmd_execute_and_capture_output(checkout_cmd_builder.extract());
Checks::check_exit(VCPKG_LINE_INFO, checkout_output.exit_code == 0, "Failed to checkout %s", git_object);
const auto& containing_folder = destination.parent_path();
if (!fs.exists(containing_folder))
{
fs.create_directories(containing_folder, VCPKG_LINE_INFO);
}
std::error_code ec;
fs.rename_or_copy(work_tree, destination, ".tmp", ec);
fs.remove_all(work_tree, VCPKG_LINE_INFO);
if (ec)
{
System::printf(System::Color::error,
"Error: Couldn't move checked out files from %s to destination %s",
fs::u8string(work_tree),
fs::u8string(destination));
Checks::exit_fail(VCPKG_LINE_INFO);
}
}
fs::path VcpkgPaths::git_checkout_baseline(Files::Filesystem& fs, StringView commit_sha) const
{
const fs::path local_repo = this->root;
const fs::path destination = this->baselines_output / fs::u8path(commit_sha) / fs::u8path("baseline.json");
const fs::path baseline_subpath = fs::u8path("port_versions") / fs::u8path("baseline.json");
if (!fs.exists(destination))
{
git_checkout_subpath(*this,
commit_sha,
baseline_subpath,
local_repo,
destination,
this->baselines_dot_git_dir,
this->baselines_work_tree);
}
return destination;
}
fs::path VcpkgPaths::git_checkout_port(Files::Filesystem& fs, StringView port_name, StringView git_tree) const
{
/* Clone a new vcpkg repository instance using the local instance as base.
*
* The `--git-dir` directory will store all the Git metadata files,
* and the `--work-tree` is the directory where files will be checked out.
*
* Since we are checking a git tree object, all files will be checked out to the root of `work-tree`.
* Because of that, it makes sense to use the git hash as the name for the directory.
*/
const fs::path local_repo = this->root;
const fs::path destination = this->versions_output / fs::u8path(git_tree) / fs::u8path(port_name);
if (!fs.exists(destination / "CONTROL") && !fs.exists(destination / "vcpkg.json"))
{
git_checkout_object(
*this, git_tree, local_repo, destination, this->versions_dot_git_dir, this->versions_work_tree);
}
return destination;
}
Optional<const Json::Object&> VcpkgPaths::get_manifest() const
{
if (auto p = m_pimpl->m_manifest_doc.get())

View File

@ -0,0 +1,210 @@
#include <vcpkg/versiondeserializers.h>
using namespace vcpkg;
using namespace vcpkg::Versions;
namespace
{
struct VersionDbEntryDeserializer final : Json::IDeserializer<VersionDbEntry>
{
static constexpr StringLiteral VERSION_RELAXED = "version";
static constexpr StringLiteral VERSION_SEMVER = "version-semver";
static constexpr StringLiteral VERSION_STRING = "version-string";
static constexpr StringLiteral VERSION_DATE = "version-date";
static constexpr StringLiteral PORT_VERSION = "port-version";
static constexpr StringLiteral GIT_TREE = "git-tree";
StringView type_name() const override { return "a version database entry"; }
View<StringView> valid_fields() const override
{
static const StringView t[] = {
VERSION_RELAXED, VERSION_SEMVER, VERSION_STRING, VERSION_DATE, PORT_VERSION, GIT_TREE};
return t;
}
Optional<VersionDbEntry> visit_object(Json::Reader& r, const Json::Object& obj) override
{
std::string version;
int port_version = 0;
std::string git_tree;
Versions::Scheme version_scheme = Versions::Scheme::String;
// Code copy-and-paste'd from sourceparagraph.cpp
static Json::StringDeserializer version_exact_deserializer{"an exact version string"};
static Json::StringDeserializer version_relaxed_deserializer{"a relaxed version string"};
static Json::StringDeserializer version_semver_deserializer{"a semantic version string"};
static Json::StringDeserializer version_date_deserializer{"a date version string"};
static Json::StringDeserializer git_tree_deserializer("a git object SHA");
bool has_exact = r.optional_object_field(obj, VERSION_STRING, version, version_exact_deserializer);
bool has_relax = r.optional_object_field(obj, VERSION_RELAXED, version, version_relaxed_deserializer);
bool has_semver = r.optional_object_field(obj, VERSION_SEMVER, version, version_semver_deserializer);
bool has_date = r.optional_object_field(obj, VERSION_DATE, version, version_date_deserializer);
int num_versions = (int)has_exact + (int)has_relax + (int)has_semver + (int)has_date;
if (num_versions == 0)
{
r.add_generic_error(type_name(), "expected a versioning field (example: ", VERSION_STRING, ")");
}
else if (num_versions > 1)
{
r.add_generic_error(type_name(), "expected only one versioning field");
}
else
{
if (has_exact)
version_scheme = Versions::Scheme::String;
else if (has_relax)
version_scheme = Versions::Scheme::Relaxed;
else if (has_semver)
version_scheme = Versions::Scheme::Semver;
else if (has_date)
version_scheme = Versions::Scheme::Date;
else
Checks::unreachable(VCPKG_LINE_INFO);
}
r.optional_object_field(obj, PORT_VERSION, port_version, Json::NaturalNumberDeserializer::instance);
r.required_object_field(type_name(), obj, GIT_TREE, git_tree, git_tree_deserializer);
return VersionDbEntry(version, port_version, version_scheme, git_tree);
}
static VersionDbEntryDeserializer instance;
};
struct VersionDbEntryArrayDeserializer final : Json::IDeserializer<std::vector<VersionDbEntry>>
{
virtual StringView type_name() const override { return "an array of versions"; }
virtual Optional<std::vector<VersionDbEntry>> visit_array(Json::Reader& r, const Json::Array& arr) override
{
return r.array_elements(arr, VersionDbEntryDeserializer::instance);
}
static VersionDbEntryArrayDeserializer instance;
};
VersionDbEntryDeserializer VersionDbEntryDeserializer::instance;
VersionDbEntryArrayDeserializer VersionDbEntryArrayDeserializer::instance;
struct BaselineDeserializer final : Json::IDeserializer<std::map<std::string, VersionT, std::less<>>>
{
StringView type_name() const override { return "a baseline object"; }
Optional<type> visit_object(Json::Reader& r, const Json::Object& obj) override
{
std::map<std::string, VersionT, std::less<>> result;
for (auto&& pr : obj)
{
const auto& version_value = pr.second;
VersionT version;
r.visit_in_key(version_value, pr.first, version, get_versiont_deserializer_instance());
result.emplace(pr.first.to_string(), std::move(version));
}
return std::move(result);
}
static BaselineDeserializer instance;
};
BaselineDeserializer BaselineDeserializer::instance;
struct VersionTDeserializer final : Json::IDeserializer<VersionT>
{
StringView type_name() const override { return "a version object"; }
View<StringView> valid_fields() const override
{
static const StringView t[] = {"version-string", "port-version"};
return t;
}
Optional<VersionT> visit_object(Json::Reader& r, const Json::Object& obj) override
{
std::string version;
int port_version = 0;
r.required_object_field(type_name(), obj, "version-string", version, version_deserializer);
r.optional_object_field(obj, "port-version", port_version, Json::NaturalNumberDeserializer::instance);
return VersionT{std::move(version), port_version};
}
static Json::StringDeserializer version_deserializer;
static VersionTDeserializer instance;
};
Json::StringDeserializer VersionTDeserializer::version_deserializer{"version"};
VersionTDeserializer VersionTDeserializer::instance;
}
namespace vcpkg
{
Json::IDeserializer<VersionT>& get_versiont_deserializer_instance() { return VersionTDeserializer::instance; }
ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
StringView baseline_name,
const fs::path& baseline_file_path)
{
if (!fs.exists(baseline_file_path))
{
return Strings::format("Couldn't find `%s`", fs::u8string(baseline_file_path));
}
auto value = Json::parse_file(VCPKG_LINE_INFO, fs, baseline_file_path);
if (!value.first.is_object())
{
return Strings::format("Error: `%s` does not have a top-level object.", fs::u8string(baseline_file_path));
}
const auto& obj = value.first.object();
auto baseline_value = obj.get(baseline_name);
if (!baseline_value)
{
return Strings::format(
"Error: `%s` does not contain the baseline \"%s\"", fs::u8string(baseline_file_path), baseline_name);
}
Json::Reader r;
std::map<std::string, VersionT, std::less<>> result;
r.visit_in_key(*baseline_value, baseline_name, result, BaselineDeserializer::instance);
if (!r.errors().empty())
{
return Strings::format(
"Error: failed to parse `%s`:\n%s", fs::u8string(baseline_file_path), Strings::join("\n", r.errors()));
}
return result;
}
ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
StringView port_name,
const fs::path& versions_file_path)
{
(void)port_name;
if (!fs.exists(versions_file_path))
{
return Strings::format("Couldn't find the versions database file: %s", fs::u8string(versions_file_path));
}
auto versions_json = Json::parse_file(VCPKG_LINE_INFO, fs, versions_file_path);
if (!versions_json.first.is_object())
{
return Strings::format("Error: `%s` does not have a top level object.", fs::u8string(versions_file_path));
}
const auto& versions_object = versions_json.first.object();
auto maybe_versions_array = versions_object.get("versions");
if (!maybe_versions_array || !maybe_versions_array->is_array())
{
return Strings::format("Error: `%s` does not contain a versions array.", fs::u8string(versions_file_path));
}
std::vector<VersionDbEntry> db_entries;
// Avoid warning treated as error.
if (maybe_versions_array != nullptr)
{
Json::Reader r;
r.visit_in_key(*maybe_versions_array, "versions", db_entries, VersionDbEntryArrayDeserializer::instance);
}
return db_entries;
}
}

View File

@ -0,0 +1,34 @@
#include <vcpkg/versions.h>
namespace vcpkg::Versions
{
VersionSpec::VersionSpec(const std::string& port_name, const VersionT& version, Scheme scheme)
: port_name(port_name), version(version), scheme(scheme)
{
}
VersionSpec::VersionSpec(const std::string& port_name,
const std::string& version_string,
int port_version,
Scheme scheme)
: port_name(port_name), version(version_string, port_version), scheme(scheme)
{
}
bool operator==(const VersionSpec& lhs, const VersionSpec& rhs)
{
return std::tie(lhs.port_name, lhs.version, lhs.scheme) == std::tie(rhs.port_name, rhs.version, rhs.scheme);
}
bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs) { return !(lhs == rhs); }
std::size_t VersionSpecHasher::operator()(const VersionSpec& key) const
{
using std::hash;
using std::size_t;
using std::string;
return ((hash<string>()(key.port_name) ^ (hash<string>()(key.version.to_string()) << 1)) >> 1) ^
(hash<int>()(static_cast<int>(key.scheme)) << 1);
}
}

View File

@ -267,6 +267,7 @@
<ClInclude Include="..\include\vcpkg\vcpkgcmdarguments.h" />
<ClInclude Include="..\include\vcpkg\vcpkglib.h" />
<ClInclude Include="..\include\vcpkg\vcpkgpaths.h" />
<ClInclude Include="..\include\vcpkg\versiondeserializers.h" />
<ClInclude Include="..\include\vcpkg\versions.h" />
<ClInclude Include="..\include\vcpkg\versiont.h" />
<ClInclude Include="..\include\vcpkg\visualstudio.h" />
@ -355,6 +356,8 @@
<ClCompile Include="..\src\vcpkg\vcpkgcmdarguments.cpp" />
<ClCompile Include="..\src\vcpkg\vcpkglib.cpp" />
<ClCompile Include="..\src\vcpkg\vcpkgpaths.cpp" />
<ClCompile Include="..\src\vcpkg\versiondeserializers.cpp" />
<ClCompile Include="..\src\vcpkg\versions.cpp" />
<ClCompile Include="..\src\vcpkg\versiont.cpp" />
<ClCompile Include="..\src\vcpkg\visualstudio.cpp" />
<ClCompile Include="..\src\vcpkg.cpp" />