From 6c9cda1635859571de5c964bbacdece824045305 Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Fri, 27 Nov 2020 05:44:21 -0800 Subject: [PATCH] [vcpkg] Implement VersionedPortfileProvider and BaselineProvider (#14123) * WIP: Get versions from database files * Fix formatting * Provider inherits ResourceBase * Correct versions JSON file location * Fix formatting * Fix formatting * Fix include in versions.h * Fetch port versions using git tree object * Undo changes to x-history * Remove unnecesary moves Co-authored-by: nicole mazzuca * Extract Git manipulation code * [WIP] Review comments * [WIP] Review comments pt. 2 * [WIP] Review comments / fix formatting * Generate baseline.json * Extract deserializers from registries source file * BaselineProvider initial implementation * Modify gitignore * Update .gitignore again * Use JSON deserializer for versions db * Lazy load baseline file * Fetch baseline.json from baseline commit * More git abstractions * Clean up code * Path helpers * Formatting * Move data into impl object * Use implementation object for VersionedPortfileProvider * Reuse cloned instance for checkouts * Code cleanup and formatting * Fix returning dangling reference * Prepare to remove files in port_versions/ * Remove files in port_versions/ * Update .gitignore * Some PR review comments * Use StringView * More StringView conversions * More refactoring * Make some implementation members private * Functions for parsing baseline and version files * Hide deserializers implementation * Check for `versions` feature flag in registries. Co-authored-by: Robert Schumacher Co-authored-by: nicole mazzuca --- .gitignore | 2 + scripts/generatePortVersionsDb.py | 71 +++++-- toolsrc/include/vcpkg/base/jsonreader.h | 3 +- toolsrc/include/vcpkg/portfileprovider.h | 45 ++++ toolsrc/include/vcpkg/vcpkgpaths.h | 27 +++ toolsrc/include/vcpkg/versiondeserializers.h | 37 ++++ toolsrc/include/vcpkg/versions.h | 21 ++ toolsrc/src/vcpkg/portfileprovider.cpp | 190 ++++++++++++++++- toolsrc/src/vcpkg/registries.cpp | 83 ++------ toolsrc/src/vcpkg/vcpkgpaths.cpp | 193 +++++++++++++++++ toolsrc/src/vcpkg/versiondeserializers.cpp | 210 +++++++++++++++++++ toolsrc/src/vcpkg/versions.cpp | 34 +++ toolsrc/windows-bootstrap/vcpkg.vcxproj | 3 + 13 files changed, 832 insertions(+), 87 deletions(-) create mode 100644 toolsrc/include/vcpkg/versiondeserializers.h create mode 100644 toolsrc/src/vcpkg/versiondeserializers.cpp create mode 100644 toolsrc/src/vcpkg/versions.cpp diff --git a/.gitignore b/.gitignore index ed388f4a1c..d12fd1363f 100644 --- a/.gitignore +++ b/.gitignore @@ -295,6 +295,8 @@ __pycache__/ /toolsrc/windows-bootstrap/msbuild.x86.release/ /toolsrc/windows-bootstrap/msbuild.x64.debug/ /toolsrc/windows-bootstrap/msbuild.x64.release/ +#ignore db +/port_versions/ #ignore custom triplets /triplets/* #add vcpkg-designed triplets back in diff --git a/scripts/generatePortVersionsDb.py b/scripts/generatePortVersionsDb.py index cefd61e1c9..e3c338c64e 100644 --- a/scripts/generatePortVersionsDb.py +++ b/scripts/generatePortVersionsDb.py @@ -4,6 +4,7 @@ import sys import subprocess import json import time +import shutil from subprocess import CalledProcessError from json.decoder import JSONDecodeError @@ -14,9 +15,9 @@ SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) def get_current_git_ref(): - output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], - capture_output=True, - encoding='utf-8') + output = subprocess.run(['git', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], + capture_output=True, + encoding='utf-8') if output.returncode == 0: return output.stdout.strip() print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr) @@ -25,47 +26,89 @@ def get_current_git_ref(): def generate_port_versions_db(ports_path, db_path, revision): start_time = time.time() - port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))] + + # Assume each directory in ${VCPKG_ROOT}/ports is a different port + port_names = [item for item in os.listdir( + ports_path) if os.path.isdir(os.path.join(ports_path, item))] + port_names.sort() total_count = len(port_names) + + # Dictionary to collect the latest version of each port as baseline + baseline_objects = {} + baseline_objects['default'] = {} + for counter, port_name in enumerate(port_names): containing_dir = os.path.join(db_path, f'{port_name[0]}-') os.makedirs(containing_dir, exist_ok=True) + output_filepath = os.path.join(containing_dir, f'{port_name}.json') - if not os.path.exists(output_filepath): + if not os.path.exists(output_filepath): output = subprocess.run( - [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'], + [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), + 'x-history', port_name, '--x-json'], capture_output=True, encoding='utf-8') + if output.returncode == 0: try: versions_object = json.loads(output.stdout) + + # Put latest version in baseline dictionary + latest_version = versions_object["versions"][0] + baseline_objects['default'][port_name] = { + "version-string": latest_version["version-string"], + "port-version": latest_version["port-version"] + } with open(output_filepath, 'w') as output_file: json.dump(versions_object, output_file) except JSONDecodeError: - print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr) + print( + f'Malformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr) else: - print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr) + print(f'x-history {port_name} failed: ', + output.stdout.strip(), file=sys.stderr) + # This should be replaced by a progress bar if counter > 0 and counter % 100 == 0: elapsed_time = time.time() - start_time - print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds') + print( + f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds') + + # Generate baseline.json + baseline_file_path = os.path.join(db_path, 'baseline.json') + with open(baseline_file_path, 'w') as baseline_output_file: + json.dump(baseline_objects, baseline_output_file) + + # Generate timestamp rev_file = os.path.join(db_path, revision) Path(rev_file).touch() + elapsed_time = time.time() - start_time - print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds') - + print( + f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds') + def main(ports_path, db_path): revision = get_current_git_ref() if not revision: print('Couldn\'t fetch current Git revision', file=sys.stderr) sys.exit(1) + rev_file = os.path.join(db_path, revision) if os.path.exists(rev_file): print(f'Database files already exist for commit {revision}') sys.exit(0) - generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision) + + if (os.path.exists(db_path)): + try: + shutil.rmtree(db_path) + except OSError as e: + print(f'Could not delete folder: {db_path}.\nError: {e.strerror}') + + generate_port_versions_db(ports_path=ports_path, + db_path=db_path, + revision=revision) if __name__ == "__main__": - main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), - db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions')) + main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), + db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions')) diff --git a/toolsrc/include/vcpkg/base/jsonreader.h b/toolsrc/include/vcpkg/base/jsonreader.h index cdd0299d2f..02c1936e25 100644 --- a/toolsrc/include/vcpkg/base/jsonreader.h +++ b/toolsrc/include/vcpkg/base/jsonreader.h @@ -23,7 +23,7 @@ namespace vcpkg::Json Optional visit(Reader&, const Value&); Optional visit(Reader&, const Object&); - protected: + public: virtual Optional visit_null(Reader&); virtual Optional visit_boolean(Reader&, bool); virtual Optional visit_integer(Reader& r, int64_t i); @@ -33,6 +33,7 @@ namespace vcpkg::Json virtual Optional visit_object(Reader&, const Object&); virtual View valid_fields() const; + protected: IDeserializer() = default; IDeserializer(const IDeserializer&) = default; IDeserializer& operator=(const IDeserializer&) = default; diff --git a/toolsrc/include/vcpkg/portfileprovider.h b/toolsrc/include/vcpkg/portfileprovider.h index c127aed40a..610ecb7350 100644 --- a/toolsrc/include/vcpkg/portfileprovider.h +++ b/toolsrc/include/vcpkg/portfileprovider.h @@ -6,6 +6,7 @@ #include #include +#include namespace vcpkg::PortFileProvider { @@ -36,4 +37,48 @@ namespace vcpkg::PortFileProvider std::vector overlay_ports; mutable std::unordered_map cache; }; + + struct IVersionedPortfileProvider + { + virtual const std::vector& get_port_versions(StringView port_name) const = 0; + + virtual ExpectedS get_control_file( + const vcpkg::Versions::VersionSpec& version_spec) const = 0; + }; + + struct IBaselineProvider + { + virtual Optional get_baseline_version(StringView port_name) const = 0; + }; + + namespace details + { + struct BaselineProviderImpl; + struct VersionedPortfileProviderImpl; + } + + struct VersionedPortfileProvider : IVersionedPortfileProvider, Util::ResourceBase + { + explicit VersionedPortfileProvider(const vcpkg::VcpkgPaths& paths); + ~VersionedPortfileProvider(); + + const std::vector& get_port_versions(StringView port_name) const override; + + ExpectedS get_control_file( + const vcpkg::Versions::VersionSpec& version_spec) const override; + + private: + std::unique_ptr m_impl; + }; + + struct BaselineProvider : IBaselineProvider, Util::ResourceBase + { + explicit BaselineProvider(const vcpkg::VcpkgPaths& paths, const std::string& baseline); + ~BaselineProvider(); + + Optional get_baseline_version(StringView port_name) const override; + + private: + std::unique_ptr m_impl; + }; } diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h index def874e7c3..c85eff0ca3 100644 --- a/toolsrc/include/vcpkg/vcpkgpaths.h +++ b/toolsrc/include/vcpkg/vcpkgpaths.h @@ -102,11 +102,23 @@ namespace vcpkg fs::path vcpkg_dir_info; fs::path vcpkg_dir_updates; + fs::path baselines_dot_git_dir; + fs::path baselines_work_tree; + fs::path baselines_output; + + fs::path versions_dot_git_dir; + fs::path versions_work_tree; + fs::path versions_output; + fs::path ports_cmake; const fs::path& get_tool_exe(const std::string& tool) const; const std::string& get_tool_version(const std::string& tool) const; + // Git manipulation + fs::path git_checkout_baseline(Files::Filesystem& filesystem, StringView commit_sha) const; + fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const; + Optional get_manifest() const; Optional get_manifest_path() const; const Configuration& get_configuration() const; @@ -133,5 +145,20 @@ namespace vcpkg private: std::unique_ptr m_pimpl; + + static void git_checkout_subpath(const VcpkgPaths& paths, + StringView commit_sha, + const fs::path& subpath, + const fs::path& local_repo, + const fs::path& destination, + const fs::path& dot_git_dir, + const fs::path& work_tree); + + static void git_checkout_object(const VcpkgPaths& paths, + StringView git_object, + const fs::path& local_repo, + const fs::path& destination, + const fs::path& dot_git_dir, + const fs::path& work_tree); }; } diff --git a/toolsrc/include/vcpkg/versiondeserializers.h b/toolsrc/include/vcpkg/versiondeserializers.h new file mode 100644 index 0000000000..f5ffda101e --- /dev/null +++ b/toolsrc/include/vcpkg/versiondeserializers.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include + +#include +#include + +namespace vcpkg +{ + struct VersionDbEntry + { + VersionT version; + Versions::Scheme scheme; + std::string git_tree; + + VersionDbEntry(const std::string& version_string, + int port_version, + Versions::Scheme scheme, + const std::string& git_tree) + : version(VersionT(version_string, port_version)), scheme(scheme), git_tree(git_tree) + { + } + }; + + Json::IDeserializer& get_versiont_deserializer_instance(); + + ExpectedS>> parse_baseline_file(Files::Filesystem& fs, + StringView baseline_name, + const fs::path& baseline_file_path); + + ExpectedS> parse_versions_file(Files::Filesystem& fs, + StringView port_name, + const fs::path& versions_file_path); +} \ No newline at end of file diff --git a/toolsrc/include/vcpkg/versions.h b/toolsrc/include/vcpkg/versions.h index 55586ddcc5..7d5b573c23 100644 --- a/toolsrc/include/vcpkg/versions.h +++ b/toolsrc/include/vcpkg/versions.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace vcpkg::Versions { enum class Scheme @@ -10,6 +12,25 @@ namespace vcpkg::Versions String }; + struct VersionSpec + { + std::string port_name; + VersionT version; + Scheme scheme; + + VersionSpec(const std::string& port_name, const VersionT& version, Scheme scheme); + + VersionSpec(const std::string& port_name, const std::string& version_string, int port_version, Scheme scheme); + + friend bool operator==(const VersionSpec& lhs, const VersionSpec& rhs); + friend bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs); + }; + + struct VersionSpecHasher + { + std::size_t operator()(const VersionSpec& key) const; + }; + struct Constraint { enum class Type diff --git a/toolsrc/src/vcpkg/portfileprovider.cpp b/toolsrc/src/vcpkg/portfileprovider.cpp index 9c09a23e4f..e3053bfa62 100644 --- a/toolsrc/src/vcpkg/portfileprovider.cpp +++ b/toolsrc/src/vcpkg/portfileprovider.cpp @@ -1,3 +1,6 @@ + + +#include #include #include @@ -7,6 +10,33 @@ #include #include #include +#include + +#include + +using namespace vcpkg; +using namespace Versions; + +namespace +{ + Optional get_versions_json_path(const VcpkgPaths& paths, StringView port_name) + { + const auto port_versions_dir_path = paths.root / fs::u8path("port_versions"); + const auto subpath = Strings::concat(port_name.substr(0, 1), "-/", port_name, ".json"); + const auto json_path = port_versions_dir_path / subpath; + if (paths.get_filesystem().exists(json_path)) + { + return json_path; + } + return nullopt; + } + + Optional get_baseline_json_path(const VcpkgPaths& paths, StringView baseline_commit_sha) + { + const auto baseline_json = paths.git_checkout_baseline(paths.get_filesystem(), baseline_commit_sha); + return paths.get_filesystem().exists(baseline_json) ? make_optional(baseline_json) : nullopt; + } +} namespace vcpkg::PortFileProvider { @@ -27,7 +57,7 @@ namespace vcpkg::PortFileProvider return Util::fmap(ports, [](auto&& kvpair) -> const SourceControlFileLocation* { return &kvpair.second; }); } - PathsPortFileProvider::PathsPortFileProvider(const vcpkg::VcpkgPaths& paths_, + PathsPortFileProvider::PathsPortFileProvider(const VcpkgPaths& paths_, const std::vector& overlay_ports_) : paths(paths_) { @@ -80,7 +110,7 @@ namespace vcpkg::PortFileProvider } else { - vcpkg::print_error_message(maybe_scf.error()); + print_error_message(maybe_scf.error()); Checks::exit_with_message( VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir)); } @@ -106,7 +136,7 @@ namespace vcpkg::PortFileProvider } else { - vcpkg::print_error_message(found_scf.error()); + print_error_message(found_scf.error()); Checks::exit_with_message( VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir)); } @@ -148,7 +178,7 @@ namespace vcpkg::PortFileProvider } else { - vcpkg::print_error_message(found_scf.error()); + print_error_message(found_scf.error()); Checks::exit_with_message( VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(port_directory)); } @@ -224,7 +254,7 @@ namespace vcpkg::PortFileProvider } else { - vcpkg::print_error_message(maybe_scf.error()); + print_error_message(maybe_scf.error()); Checks::exit_with_message( VCPKG_LINE_INFO, "Error: Failed to load port from %s", fs::u8string(ports_dir)); } @@ -257,4 +287,154 @@ namespace vcpkg::PortFileProvider return ret; } + + namespace details + { + struct BaselineProviderImpl + { + BaselineProviderImpl(const VcpkgPaths& paths, const std::string& baseline) + : paths(paths), baseline(baseline) + { + } + ~BaselineProviderImpl() { } + + const std::map>& get_baseline_cache() const + { + return baseline_cache.get_lazy([&]() -> auto { + auto maybe_baseline_file = get_baseline_json_path(paths, baseline); + Checks::check_exit(VCPKG_LINE_INFO, maybe_baseline_file.has_value(), "Couldn't find baseline.json"); + auto baseline_file = maybe_baseline_file.value_or_exit(VCPKG_LINE_INFO); + + auto maybe_baselines_map = parse_baseline_file(paths.get_filesystem(), "default", baseline_file); + Checks::check_exit(VCPKG_LINE_INFO, + maybe_baselines_map.has_value(), + "Error: Couldn't parse baseline `%s` from `%s`", + "default", + fs::u8string(baseline_file)); + auto baselines_map = *maybe_baselines_map.get(); + return std::move(baselines_map); + }); + } + + private: + const VcpkgPaths& paths; + const std::string baseline; + Lazy>> baseline_cache; + }; + + struct VersionedPortfileProviderImpl + { + std::map> versions_cache; + std::unordered_map git_tree_cache; + std::unordered_map control_cache; + + VersionedPortfileProviderImpl(const VcpkgPaths& paths) : paths(paths) { } + ~VersionedPortfileProviderImpl() { } + + const VcpkgPaths& get_paths() const { return paths; } + Files::Filesystem& get_filesystem() const { return paths.get_filesystem(); } + + private: + const VcpkgPaths& paths; + }; + } + + VersionedPortfileProvider::VersionedPortfileProvider(const VcpkgPaths& paths) + : m_impl(std::make_unique(paths)) + { + } + VersionedPortfileProvider::~VersionedPortfileProvider() { } + + const std::vector& VersionedPortfileProvider::get_port_versions(StringView port_name) const + { + auto cache_it = m_impl->versions_cache.find(port_name.to_string()); + if (cache_it != m_impl->versions_cache.end()) + { + return cache_it->second; + } + + auto maybe_versions_file_path = get_versions_json_path(m_impl->get_paths(), port_name); + Checks::check_exit(VCPKG_LINE_INFO, + maybe_versions_file_path.has_value(), + "Error: Couldn't find a versions database file: %s.json.", + port_name); + auto versions_file_path = maybe_versions_file_path.value_or_exit(VCPKG_LINE_INFO); + + auto maybe_version_entries = parse_versions_file(m_impl->get_filesystem(), port_name, versions_file_path); + Checks::check_exit(VCPKG_LINE_INFO, + maybe_version_entries.has_value(), + "Error: Couldn't parse versions from file: %s", + fs::u8string(versions_file_path)); + auto version_entries = maybe_version_entries.value_or_exit(VCPKG_LINE_INFO); + + auto port = port_name.to_string(); + for (auto&& version_entry : version_entries) + { + VersionSpec spec(port, version_entry.version, version_entry.scheme); + m_impl->versions_cache[port].push_back(spec); + m_impl->git_tree_cache.emplace(std::move(spec), std::move(version_entry.git_tree)); + } + return m_impl->versions_cache.at(port); + } + + ExpectedS VersionedPortfileProvider::get_control_file( + const VersionSpec& version_spec) const + { + auto cache_it = m_impl->control_cache.find(version_spec); + if (cache_it != m_impl->control_cache.end()) + { + return cache_it->second; + } + + // Pre-populate versions cache. + get_port_versions(version_spec.port_name); + + auto git_tree_cache_it = m_impl->git_tree_cache.find(version_spec); + if (git_tree_cache_it == m_impl->git_tree_cache.end()) + { + return Strings::concat("No git object SHA for entry %s at version %s.", + version_spec.port_name, + version_spec.version.to_string()); + } + + const std::string git_tree = git_tree_cache_it->second; + auto port_directory = + m_impl->get_paths().git_checkout_port(m_impl->get_filesystem(), version_spec.port_name, git_tree); + + auto maybe_control_file = Paragraphs::try_load_port(m_impl->get_filesystem(), port_directory); + if (auto scf = maybe_control_file.get()) + { + if (scf->get()->core_paragraph->name == version_spec.port_name) + { + return m_impl->control_cache + .emplace(version_spec, SourceControlFileLocation{std::move(*scf), std::move(port_directory)}) + .first->second; + } + return Strings::format("Error: Failed to load port from %s: names did not match: '%s' != '%s'", + fs::u8string(port_directory), + version_spec.port_name, + scf->get()->core_paragraph->name); + } + + print_error_message(maybe_control_file.error()); + return Strings::format( + "Error: Failed to load port %s from %s", version_spec.port_name, fs::u8string(port_directory)); + } + + BaselineProvider::BaselineProvider(const VcpkgPaths& paths, const std::string& baseline) + : m_impl(std::make_unique(paths, baseline)) + { + } + BaselineProvider::~BaselineProvider() { } + + Optional BaselineProvider::get_baseline_version(StringView port_name) const + { + const auto& cache = m_impl->get_baseline_cache(); + auto it = cache.find(port_name.to_string()); + if (it != cache.end()) + { + return it->second; + } + return nullopt; + } } diff --git a/toolsrc/src/vcpkg/registries.cpp b/toolsrc/src/vcpkg/registries.cpp index e481685d93..7794ed9b01 100644 --- a/toolsrc/src/vcpkg/registries.cpp +++ b/toolsrc/src/vcpkg/registries.cpp @@ -5,7 +5,9 @@ #include #include +#include #include +#include #include #include @@ -70,32 +72,6 @@ namespace } }; - struct VersionTDeserializer final : Json::IDeserializer - { - StringView type_name() const override { return "a version object"; } - View valid_fields() const override - { - static const StringView t[] = {"version-string", "port-version"}; - return t; - } - - Optional visit_object(Json::Reader& r, const Json::Object& obj) override - { - std::string version; - int port_version = 0; - - r.required_object_field(type_name(), obj, "version-string", version, version_deserializer); - r.optional_object_field(obj, "port-version", port_version, Json::NaturalNumberDeserializer::instance); - - return VersionT{std::move(version), port_version}; - } - - static Json::StringDeserializer version_deserializer; - static VersionTDeserializer instance; - }; - Json::StringDeserializer VersionTDeserializer::version_deserializer{"version"}; - VersionTDeserializer VersionTDeserializer::instance; - struct FilesystemVersionEntryDeserializer final : Json::IDeserializer> { StringView type_name() const override { return "a version entry object"; } @@ -109,7 +85,7 @@ namespace { fs::path registry_path; - auto version = VersionTDeserializer::instance.visit_object(r, obj); + auto version = get_versiont_deserializer_instance().visit_object(r, obj); r.required_object_field( "version entry", obj, "registry-path", registry_path, Json::PathDeserializer::instance); @@ -162,30 +138,6 @@ namespace const fs::path& registry_root; }; - struct BaselineDeserializer final : Json::IDeserializer>> - { - StringView type_name() const override { return "a baseline object"; } - - Optional visit_object(Json::Reader& r, const Json::Object& obj) override - { - std::map> result; - - for (auto pr : obj) - { - const auto& version_value = pr.second; - VersionT version; - r.visit_in_key(version_value, pr.first, version, VersionTDeserializer::instance); - - result.emplace(pr.first.to_string(), std::move(version)); - } - - return std::move(result); - } - - static BaselineDeserializer instance; - }; - BaselineDeserializer BaselineDeserializer::instance; - struct FilesystemRegistry final : RegistryImpl { std::unique_ptr get_port_entry(const VcpkgPaths& paths, StringView port_name) const override @@ -267,6 +219,12 @@ namespace Optional get_baseline_version(const VcpkgPaths& paths, StringView port_name) const override { + if (!paths.get_feature_flags().versions) + { + Checks::check_exit(VCPKG_LINE_INFO, + "This invocation failed because the `versions` feature flag is not enabled."); + } + const auto& baseline_cache = baseline.get([this, &paths] { return load_baseline_versions(paths); }); auto it = baseline_cache.find(port_name); if (it != baseline_cache.end()) @@ -310,26 +268,17 @@ namespace Checks::exit_with_message(VCPKG_LINE_INFO, "Error: `baseline.json` does not have a top-level object."); } - const auto& obj = value.first.object(); - auto baseline_value = obj.get("default"); - if (!baseline_value) + auto maybe_baseline_versions = parse_baseline_file(paths.get_filesystem(), "default", baseline_file); + if (auto baseline_versions = maybe_baseline_versions.get()) { - Checks::exit_with_message( - VCPKG_LINE_INFO, "Error: `baseline.json` does not contain the baseline \"%s\"", "default"); - } - - Json::Reader r; - std::map> result; - r.visit_in_key(*baseline_value, "default", result, BaselineDeserializer::instance); - - if (r.errors().empty()) - { - return result; + return std::move(*baseline_versions); } else { - Checks::exit_with_message( - VCPKG_LINE_INFO, "Error: failed to parse `baseline.json`:\n%s", Strings::join("\n", r.errors())); + Checks::exit_with_message(VCPKG_LINE_INFO, + "Error: failed to parse `%s`:\n%s", + fs::u8string(baseline_file), + maybe_baseline_versions.error()); } } diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp index a3ab21629e..5b33b8db44 100644 --- a/toolsrc/src/vcpkg/vcpkgpaths.cpp +++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp @@ -71,6 +71,15 @@ namespace return result; } + System::CmdLineBuilder git_cmd_builder(const VcpkgPaths& paths, + const fs::path& dot_git_dir, + const fs::path& work_tree) + { + return System::CmdLineBuilder() + .path_arg(paths.get_tool_exe(Tools::GIT)) + .string_arg(Strings::concat("--git-dir=", fs::u8string(dot_git_dir))) + .string_arg(Strings::concat("--work-tree=", fs::u8string(work_tree))); + } } // unnamed namespace namespace vcpkg @@ -353,6 +362,18 @@ If you wish to silence this error and use classic mode, you can: vcpkg_dir_info = vcpkg_dir / fs::u8path("info"); vcpkg_dir_updates = vcpkg_dir / fs::u8path("updates"); + // Versioning paths + const auto versioning_tmp = buildtrees / fs::u8path("versioning_tmp"); + const auto versioning_output = buildtrees / fs::u8path("versioning"); + + baselines_dot_git_dir = versioning_tmp / fs::u8path(".baselines.git"); + baselines_work_tree = versioning_tmp / fs::u8path("baselines-worktree"); + baselines_output = versioning_output / fs::u8path("baselines"); + + versions_dot_git_dir = versioning_tmp / fs::u8path(".versions.git"); + versions_work_tree = versioning_tmp / fs::u8path("versions-worktree"); + versions_output = versioning_output / fs::u8path("versions"); + ports_cmake = filesystem.canonical(VCPKG_LINE_INFO, scripts / fs::u8path("ports.cmake")); for (auto&& overlay_triplets_dir : args.overlay_triplets) @@ -456,6 +477,178 @@ If you wish to silence this error and use classic mode, you can: return m_pimpl->m_tool_cache->get_tool_version(*this, tool); } + void VcpkgPaths::git_checkout_subpath(const VcpkgPaths& paths, + StringView commit_sha, + const fs::path& subpath, + const fs::path& local_repo, + const fs::path& destination, + const fs::path& dot_git_dir, + const fs::path& work_tree) + { + Files::Filesystem& fs = paths.get_filesystem(); + fs.remove_all(work_tree, VCPKG_LINE_INFO); + fs.remove_all(destination, VCPKG_LINE_INFO); + fs.remove_all(dot_git_dir, VCPKG_LINE_INFO); + + // All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp} + // git clone --no-checkout --local {vcpkg_root} {dot_git_dir} + System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree) + .string_arg("clone") + .string_arg("--no-checkout") + .string_arg("--local") + .path_arg(local_repo) + .path_arg(dot_git_dir); + const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract()); + Checks::check_exit(VCPKG_LINE_INFO, + clone_output.exit_code == 0, + "Failed to clone temporary vcpkg instance.\n%s\n", + clone_output.output); + + // git checkout {commit-sha} -- {subpath} + System::CmdLineBuilder checkout_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree) + .string_arg("checkout") + .string_arg(commit_sha) + .string_arg("--") + .path_arg(subpath); + const auto checkout_output = System::cmd_execute_and_capture_output(checkout_cmd_builder.extract()); + Checks::check_exit(VCPKG_LINE_INFO, + checkout_output.exit_code == 0, + "Error: Failed to checkout %s:%s\n%s\n", + commit_sha, + fs::u8string(subpath), + checkout_output.output); + + const fs::path checked_out_path = work_tree / subpath; + const auto& containing_folder = destination.parent_path(); + if (!fs.exists(containing_folder)) + { + fs.create_directories(containing_folder, VCPKG_LINE_INFO); + } + + std::error_code ec; + fs.rename_or_copy(checked_out_path, destination, ".tmp", ec); + fs.remove_all(work_tree, VCPKG_LINE_INFO); + fs.remove_all(dot_git_dir, VCPKG_LINE_INFO); + if (ec) + { + System::printf(System::Color::error, + "Error: Couldn't move checked out files from %s to destination %s", + fs::u8string(checked_out_path), + fs::u8string(destination)); + Checks::exit_fail(VCPKG_LINE_INFO); + } + } + + void VcpkgPaths::git_checkout_object(const VcpkgPaths& paths, + StringView git_object, + const fs::path& local_repo, + const fs::path& destination, + const fs::path& dot_git_dir, + const fs::path& work_tree) + { + Files::Filesystem& fs = paths.get_filesystem(); + fs.remove_all(work_tree, VCPKG_LINE_INFO); + fs.remove_all(destination, VCPKG_LINE_INFO); + + if (!fs.exists(dot_git_dir)) + { + // All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp} + // git clone --no-checkout --local {vcpkg_root} {dot_git_dir} + System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree) + .string_arg("clone") + .string_arg("--no-checkout") + .string_arg("--local") + .path_arg(local_repo) + .path_arg(dot_git_dir); + const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract()); + Checks::check_exit(VCPKG_LINE_INFO, + clone_output.exit_code == 0, + "Failed to clone temporary vcpkg instance.\n%s\n", + clone_output.output); + } + else + { + System::CmdLineBuilder fetch_cmd_builder = + git_cmd_builder(paths, dot_git_dir, work_tree).string_arg("fetch"); + const auto fetch_output = System::cmd_execute_and_capture_output(fetch_cmd_builder.extract()); + Checks::check_exit(VCPKG_LINE_INFO, + fetch_output.exit_code == 0, + "Failed to update refs on temporary vcpkg repository.\n%s\n", + fetch_output.output); + } + + if (!fs.exists(work_tree)) + { + fs.create_directories(work_tree, VCPKG_LINE_INFO); + } + + // git checkout {tree_object} . + System::CmdLineBuilder checkout_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree) + .string_arg("checkout") + .string_arg(git_object) + .string_arg("."); + const auto checkout_output = System::cmd_execute_and_capture_output(checkout_cmd_builder.extract()); + Checks::check_exit(VCPKG_LINE_INFO, checkout_output.exit_code == 0, "Failed to checkout %s", git_object); + + const auto& containing_folder = destination.parent_path(); + if (!fs.exists(containing_folder)) + { + fs.create_directories(containing_folder, VCPKG_LINE_INFO); + } + + std::error_code ec; + fs.rename_or_copy(work_tree, destination, ".tmp", ec); + fs.remove_all(work_tree, VCPKG_LINE_INFO); + if (ec) + { + System::printf(System::Color::error, + "Error: Couldn't move checked out files from %s to destination %s", + fs::u8string(work_tree), + fs::u8string(destination)); + Checks::exit_fail(VCPKG_LINE_INFO); + } + } + + fs::path VcpkgPaths::git_checkout_baseline(Files::Filesystem& fs, StringView commit_sha) const + { + const fs::path local_repo = this->root; + const fs::path destination = this->baselines_output / fs::u8path(commit_sha) / fs::u8path("baseline.json"); + const fs::path baseline_subpath = fs::u8path("port_versions") / fs::u8path("baseline.json"); + + if (!fs.exists(destination)) + { + git_checkout_subpath(*this, + commit_sha, + baseline_subpath, + local_repo, + destination, + this->baselines_dot_git_dir, + this->baselines_work_tree); + } + return destination; + } + + fs::path VcpkgPaths::git_checkout_port(Files::Filesystem& fs, StringView port_name, StringView git_tree) const + { + /* Clone a new vcpkg repository instance using the local instance as base. + * + * The `--git-dir` directory will store all the Git metadata files, + * and the `--work-tree` is the directory where files will be checked out. + * + * Since we are checking a git tree object, all files will be checked out to the root of `work-tree`. + * Because of that, it makes sense to use the git hash as the name for the directory. + */ + const fs::path local_repo = this->root; + const fs::path destination = this->versions_output / fs::u8path(git_tree) / fs::u8path(port_name); + + if (!fs.exists(destination / "CONTROL") && !fs.exists(destination / "vcpkg.json")) + { + git_checkout_object( + *this, git_tree, local_repo, destination, this->versions_dot_git_dir, this->versions_work_tree); + } + return destination; + } + Optional VcpkgPaths::get_manifest() const { if (auto p = m_pimpl->m_manifest_doc.get()) diff --git a/toolsrc/src/vcpkg/versiondeserializers.cpp b/toolsrc/src/vcpkg/versiondeserializers.cpp new file mode 100644 index 0000000000..276b70e4fa --- /dev/null +++ b/toolsrc/src/vcpkg/versiondeserializers.cpp @@ -0,0 +1,210 @@ +#include + +using namespace vcpkg; +using namespace vcpkg::Versions; + +namespace +{ + struct VersionDbEntryDeserializer final : Json::IDeserializer + { + static constexpr StringLiteral VERSION_RELAXED = "version"; + static constexpr StringLiteral VERSION_SEMVER = "version-semver"; + static constexpr StringLiteral VERSION_STRING = "version-string"; + static constexpr StringLiteral VERSION_DATE = "version-date"; + static constexpr StringLiteral PORT_VERSION = "port-version"; + static constexpr StringLiteral GIT_TREE = "git-tree"; + + StringView type_name() const override { return "a version database entry"; } + View valid_fields() const override + { + static const StringView t[] = { + VERSION_RELAXED, VERSION_SEMVER, VERSION_STRING, VERSION_DATE, PORT_VERSION, GIT_TREE}; + return t; + } + + Optional visit_object(Json::Reader& r, const Json::Object& obj) override + { + std::string version; + int port_version = 0; + std::string git_tree; + Versions::Scheme version_scheme = Versions::Scheme::String; + + // Code copy-and-paste'd from sourceparagraph.cpp + static Json::StringDeserializer version_exact_deserializer{"an exact version string"}; + static Json::StringDeserializer version_relaxed_deserializer{"a relaxed version string"}; + static Json::StringDeserializer version_semver_deserializer{"a semantic version string"}; + static Json::StringDeserializer version_date_deserializer{"a date version string"}; + static Json::StringDeserializer git_tree_deserializer("a git object SHA"); + + bool has_exact = r.optional_object_field(obj, VERSION_STRING, version, version_exact_deserializer); + bool has_relax = r.optional_object_field(obj, VERSION_RELAXED, version, version_relaxed_deserializer); + bool has_semver = r.optional_object_field(obj, VERSION_SEMVER, version, version_semver_deserializer); + bool has_date = r.optional_object_field(obj, VERSION_DATE, version, version_date_deserializer); + int num_versions = (int)has_exact + (int)has_relax + (int)has_semver + (int)has_date; + if (num_versions == 0) + { + r.add_generic_error(type_name(), "expected a versioning field (example: ", VERSION_STRING, ")"); + } + else if (num_versions > 1) + { + r.add_generic_error(type_name(), "expected only one versioning field"); + } + else + { + if (has_exact) + version_scheme = Versions::Scheme::String; + else if (has_relax) + version_scheme = Versions::Scheme::Relaxed; + else if (has_semver) + version_scheme = Versions::Scheme::Semver; + else if (has_date) + version_scheme = Versions::Scheme::Date; + else + Checks::unreachable(VCPKG_LINE_INFO); + } + r.optional_object_field(obj, PORT_VERSION, port_version, Json::NaturalNumberDeserializer::instance); + r.required_object_field(type_name(), obj, GIT_TREE, git_tree, git_tree_deserializer); + + return VersionDbEntry(version, port_version, version_scheme, git_tree); + } + + static VersionDbEntryDeserializer instance; + }; + + struct VersionDbEntryArrayDeserializer final : Json::IDeserializer> + { + virtual StringView type_name() const override { return "an array of versions"; } + + virtual Optional> visit_array(Json::Reader& r, const Json::Array& arr) override + { + return r.array_elements(arr, VersionDbEntryDeserializer::instance); + } + + static VersionDbEntryArrayDeserializer instance; + }; + + VersionDbEntryDeserializer VersionDbEntryDeserializer::instance; + VersionDbEntryArrayDeserializer VersionDbEntryArrayDeserializer::instance; + + struct BaselineDeserializer final : Json::IDeserializer>> + { + StringView type_name() const override { return "a baseline object"; } + + Optional visit_object(Json::Reader& r, const Json::Object& obj) override + { + std::map> result; + + for (auto&& pr : obj) + { + const auto& version_value = pr.second; + VersionT version; + r.visit_in_key(version_value, pr.first, version, get_versiont_deserializer_instance()); + + result.emplace(pr.first.to_string(), std::move(version)); + } + + return std::move(result); + } + + static BaselineDeserializer instance; + }; + BaselineDeserializer BaselineDeserializer::instance; + + struct VersionTDeserializer final : Json::IDeserializer + { + StringView type_name() const override { return "a version object"; } + View valid_fields() const override + { + static const StringView t[] = {"version-string", "port-version"}; + return t; + } + + Optional visit_object(Json::Reader& r, const Json::Object& obj) override + { + std::string version; + int port_version = 0; + + r.required_object_field(type_name(), obj, "version-string", version, version_deserializer); + r.optional_object_field(obj, "port-version", port_version, Json::NaturalNumberDeserializer::instance); + + return VersionT{std::move(version), port_version}; + } + + static Json::StringDeserializer version_deserializer; + static VersionTDeserializer instance; + }; + Json::StringDeserializer VersionTDeserializer::version_deserializer{"version"}; + VersionTDeserializer VersionTDeserializer::instance; +} + +namespace vcpkg +{ + Json::IDeserializer& get_versiont_deserializer_instance() { return VersionTDeserializer::instance; } + + ExpectedS>> parse_baseline_file(Files::Filesystem& fs, + StringView baseline_name, + const fs::path& baseline_file_path) + { + if (!fs.exists(baseline_file_path)) + { + return Strings::format("Couldn't find `%s`", fs::u8string(baseline_file_path)); + } + + auto value = Json::parse_file(VCPKG_LINE_INFO, fs, baseline_file_path); + if (!value.first.is_object()) + { + return Strings::format("Error: `%s` does not have a top-level object.", fs::u8string(baseline_file_path)); + } + + const auto& obj = value.first.object(); + auto baseline_value = obj.get(baseline_name); + if (!baseline_value) + { + return Strings::format( + "Error: `%s` does not contain the baseline \"%s\"", fs::u8string(baseline_file_path), baseline_name); + } + + Json::Reader r; + std::map> result; + r.visit_in_key(*baseline_value, baseline_name, result, BaselineDeserializer::instance); + if (!r.errors().empty()) + { + return Strings::format( + "Error: failed to parse `%s`:\n%s", fs::u8string(baseline_file_path), Strings::join("\n", r.errors())); + } + return result; + } + + ExpectedS> parse_versions_file(Files::Filesystem& fs, + StringView port_name, + const fs::path& versions_file_path) + { + (void)port_name; + if (!fs.exists(versions_file_path)) + { + return Strings::format("Couldn't find the versions database file: %s", fs::u8string(versions_file_path)); + } + + auto versions_json = Json::parse_file(VCPKG_LINE_INFO, fs, versions_file_path); + if (!versions_json.first.is_object()) + { + return Strings::format("Error: `%s` does not have a top level object.", fs::u8string(versions_file_path)); + } + + const auto& versions_object = versions_json.first.object(); + auto maybe_versions_array = versions_object.get("versions"); + if (!maybe_versions_array || !maybe_versions_array->is_array()) + { + return Strings::format("Error: `%s` does not contain a versions array.", fs::u8string(versions_file_path)); + } + + std::vector db_entries; + // Avoid warning treated as error. + if (maybe_versions_array != nullptr) + { + Json::Reader r; + r.visit_in_key(*maybe_versions_array, "versions", db_entries, VersionDbEntryArrayDeserializer::instance); + } + return db_entries; + } +} diff --git a/toolsrc/src/vcpkg/versions.cpp b/toolsrc/src/vcpkg/versions.cpp new file mode 100644 index 0000000000..ac1829712c --- /dev/null +++ b/toolsrc/src/vcpkg/versions.cpp @@ -0,0 +1,34 @@ +#include + +namespace vcpkg::Versions +{ + VersionSpec::VersionSpec(const std::string& port_name, const VersionT& version, Scheme scheme) + : port_name(port_name), version(version), scheme(scheme) + { + } + + VersionSpec::VersionSpec(const std::string& port_name, + const std::string& version_string, + int port_version, + Scheme scheme) + : port_name(port_name), version(version_string, port_version), scheme(scheme) + { + } + + bool operator==(const VersionSpec& lhs, const VersionSpec& rhs) + { + return std::tie(lhs.port_name, lhs.version, lhs.scheme) == std::tie(rhs.port_name, rhs.version, rhs.scheme); + } + + bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs) { return !(lhs == rhs); } + + std::size_t VersionSpecHasher::operator()(const VersionSpec& key) const + { + using std::hash; + using std::size_t; + using std::string; + + return ((hash()(key.port_name) ^ (hash()(key.version.to_string()) << 1)) >> 1) ^ + (hash()(static_cast(key.scheme)) << 1); + } +} \ No newline at end of file diff --git a/toolsrc/windows-bootstrap/vcpkg.vcxproj b/toolsrc/windows-bootstrap/vcpkg.vcxproj index 5f897bc673..16de6e9c45 100644 --- a/toolsrc/windows-bootstrap/vcpkg.vcxproj +++ b/toolsrc/windows-bootstrap/vcpkg.vcxproj @@ -267,6 +267,7 @@ + @@ -355,6 +356,8 @@ + +