From 684989a1e48b6b7f0ac1c340a702236974762f05 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 14:02:13 -0700 Subject: [PATCH 01/34] use additional env param --- scripts/get_triplet_environment.cmake | 1 + toolsrc/include/vcpkg/build.h | 24 ++ toolsrc/include/vcpkg/sourceparagraph.h | 2 + toolsrc/src/vcpkg/build.cpp | 452 ++++++++++++++---------- toolsrc/src/vcpkg/sourceparagraph.cpp | 22 ++ triplets/x64-linux.cmake | 2 + 6 files changed, 311 insertions(+), 192 deletions(-) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index bc79b16ced3..24ff409058b 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -9,3 +9,4 @@ message("VCPKG_PLATFORM_TOOLSET=${VCPKG_PLATFORM_TOOLSET}") message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}") message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}") message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}") +message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}") diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 04cd7cf873e..e26597376f0 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -117,6 +117,29 @@ namespace vcpkg::Build std::string create_error_message(const BuildResult build_result, const PackageSpec& spec); std::string create_user_troubleshooting_message(const PackageSpec& spec); + enum class VcpkgTripletVar + { + TARGET_ARCHITECTURE = 0, + CMAKE_SYSTEM_NAME, + CMAKE_SYSTEM_VERSION, + PLATFORM_TOOLSET, + VISUAL_STUDIO_PATH, + CHAINLOAD_TOOLCHAIN_FILE, + BUILD_TYPE, + ENV_PASSTHROUGH, + }; + + const std::unordered_map VCPKG_OPTIONS = { + {"VCPKG_TARGET_ARCHITECTURE", VcpkgTripletVar::TARGET_ARCHITECTURE}, + {"VCPKG_CMAKE_SYSTEM_NAME", VcpkgTripletVar::CMAKE_SYSTEM_NAME}, + {"VCPKG_CMAKE_SYSTEM_VERSION", VcpkgTripletVar::CMAKE_SYSTEM_VERSION}, + {"VCPKG_PLATFORM_TOOLSET", VcpkgTripletVar::PLATFORM_TOOLSET}, + {"VCPKG_VISUAL_STUDIO_PATH", VcpkgTripletVar::VISUAL_STUDIO_PATH}, + {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, + {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, + {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + }; + /// /// Settings from the triplet file which impact the build environment and post-build checks /// @@ -135,6 +158,7 @@ namespace vcpkg::Build Optional visual_studio_path; Optional external_toolchain_file; Optional build_type; + std::vector passthrough_env_vars; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index 6232a3fd292..9fbd83475d8 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -23,6 +23,8 @@ namespace vcpkg std::vector filter_dependencies(const std::vector& deps, const Triplet& t); std::vector filter_dependencies_to_specs(const std::vector& deps, const Triplet& t); + std::vector filter_dependencies_to_features(const std::vector& deps, + const Triplet& t); // zlib[uwp] becomes Dependency{"zlib", "uwp"} std::vector expand_qualified_dependencies(const std::vector& depends); diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 68df1f965a0..e0c78f2da31 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -32,24 +32,14 @@ namespace vcpkg::Build::Command using Dependencies::InstallPlanAction; using Dependencies::InstallPlanType; - static constexpr StringLiteral OPTION_CHECKS_ONLY = "--checks-only"; - void perform_and_exit_ex(const FullPackageSpec& full_spec, const SourceControlFileLocation& scfl, const ParsedArguments& options, const VcpkgPaths& paths) { + const StatusParagraphs status_db = database_load_check(paths); const PackageSpec& spec = full_spec.package_spec; - const auto& scf = *scfl.source_control_file; - if (Util::Sets::contains(options.switches, OPTION_CHECKS_ONLY)) - { - const auto pre_build_info = Build::PreBuildInfo::from_triplet_file(paths, spec.triplet()); - const auto build_info = Build::read_build_info(paths.get_filesystem(), paths.build_info_file_path(spec)); - const size_t error_count = - PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, scfl.source_location); - Checks::check_exit(VCPKG_LINE_INFO, error_count == 0); - Checks::exit_success(VCPKG_LINE_INFO); - } + const SourceControlFile& scf = *scfl.source_control_file; Checks::check_exit(VCPKG_LINE_INFO, spec.name() == scf.core_paragraph->name, @@ -57,7 +47,6 @@ namespace vcpkg::Build::Command scf.core_paragraph->name, spec.name()); - const StatusParagraphs status_db = database_load_check(paths); const Build::BuildPackageOptions build_package_options{ Build::UseHeadVersion::NO, Build::AllowDownloads::YES, @@ -104,15 +93,11 @@ namespace vcpkg::Build::Command Checks::exit_success(VCPKG_LINE_INFO); } - static constexpr std::array BUILD_SWITCHES = {{ - {OPTION_CHECKS_ONLY, "Only run checks, do not rebuild package"}, - }}; - const CommandStructure COMMAND_STRUCTURE = { Help::create_example_string("build zlib:x64-windows"), 1, 1, - {BUILD_SWITCHES, {}}, + {{}, {}}, nullptr, }; @@ -230,6 +215,23 @@ namespace vcpkg::Build })); } + std::unordered_map make_env_passthrough(const PreBuildInfo& pre_build_info) + { + std::unordered_map env; + + for (auto&& env_var : pre_build_info.passthrough_env_vars) + { + auto env_val = System::get_environment_variable(env_var); + + if (env_val) + { + env[env_var] = env_val.value_or_exit(VCPKG_LINE_INFO); + } + } + + return env; + } + std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset) { if (pre_build_info.external_toolchain_file.has_value()) return ""; @@ -274,7 +276,7 @@ namespace vcpkg::Build return bcf; } - static void write_binary_control_file(const VcpkgPaths& paths, BinaryControlFile bcf) + static void write_binary_control_file(const VcpkgPaths& paths, const BinaryControlFile& bcf) { std::string start = Strings::serialize(bcf.core_paragraph); for (auto&& feature : bcf.features) @@ -285,23 +287,63 @@ namespace vcpkg::Build paths.get_filesystem().write_contents(binary_control_file, start, VCPKG_LINE_INFO); } + static std::vector get_dependencies(const SourceControlFile& scf, + const std::set& feature_list, + const Triplet& triplet) + { + return Util::fmap_flatten(feature_list, + [&](std::string const& feature) -> std::vector { + if (feature == "core") + { + return filter_dependencies_to_features(scf.core_paragraph->depends, triplet); + } + + auto maybe_feature = scf.find_feature(feature); + Checks::check_exit(VCPKG_LINE_INFO, maybe_feature.has_value()); + + return filter_dependencies_to_features(maybe_feature.get()->depends, triplet); + } + ); + } + + static std::vector get_dependency_names(const SourceControlFile& scf, + const std::set& feature_list, + const Triplet& triplet) + { + return Util::fmap(get_dependencies(scf, feature_list, triplet), + [&](const Features& feat) { + return feat.name; + } + ); + } + + static std::vector get_partials(const VcpkgPaths& paths, + const std::vector& dependencies, + const Triplet& triplet) + { + std::vector ret; + Files::Filesystem& fs = paths.get_filesystem(); + + for (const std::string& dependency : dependencies) + { + fs::path partial = + paths.buildtrees / dependency / triplet.canonical_name() / "partial_triplet.cmake"; + if (fs.is_regular_file(partial)) + { + ret.emplace_back(std::move(partial)); + } + } + + return ret; + } + static std::vector compute_required_feature_specs(const BuildPackageConfig& config, const StatusParagraphs& status_db) { const Triplet& triplet = config.triplet; const std::vector dep_strings = - Util::fmap_flatten(config.feature_list, [&](std::string const& feature) -> std::vector { - if (feature == "core") - { - return filter_dependencies(config.scf.core_paragraph->depends, triplet); - } - - auto maybe_feature = config.scf.find_feature(feature); - Checks::check_exit(VCPKG_LINE_INFO, maybe_feature.has_value()); - - return filter_dependencies(maybe_feature.get()->depends, triplet); - }); + get_dependency_names(config.scf, config.feature_list, triplet); auto dep_fspecs = FeatureSpec::from_strings_and_triplet(dep_strings, triplet); Util::sort_unique_erase(dep_fspecs); @@ -353,6 +395,139 @@ namespace vcpkg::Build return concurrency; } + static std::vector get_cmake_vars(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const BuildPackageConfig& config, + const Triplet& triplet, + const Toolset& toolset) + { +#if !defined(_WIN32) + // TODO: remove when vcpkg.exe is in charge for acquiring tools. Change introduced in vcpkg v0.0.107. + // bootstrap should have already downloaded ninja, but making sure it is present in case it was deleted. + vcpkg::Util::unused(paths.get_tool_exe(Tools::NINJA)); +#endif + + const fs::path& git_exe_path = paths.get_tool_exe(Tools::GIT); + + std::string all_features; + for (auto& feature : config.scf.feature_paragraphs) + { + all_features.append(feature->name + ";"); + } + + std::vector partials = + Util::fmap( + get_partials(paths, get_dependency_names(config.scf, config.feature_list, triplet), triplet), + [&](const fs::path& path) + { + return path.u8string(); + } + ); + + std::vector variables{ + {"CMD", "BUILD"}, + {"PORT", config.scf.core_paragraph->name}, + {"CURRENT_PORT_DIR", config.port_dir}, + {"TARGET_TRIPLET", triplet.canonical_name()}, + {"TARGET_TRIPLET_FILE", paths.get_triplet_file_path(triplet).u8string()}, + {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, + {"VCPKG_USE_HEAD_VERSION", Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, + {"DOWNLOADS", paths.downloads}, + {"_VCPKG_NO_DOWNLOADS", !Util::Enum::to_bool(config.build_package_options.allow_downloads) ? "1" : "0"}, + {"_VCPKG_DOWNLOAD_TOOL", to_string(config.build_package_options.download_tool)}, + {"FEATURES", Strings::join(";", config.feature_list)}, + {"ALL_FEATURES", all_features}, + {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())}, + {"VCPKG_PARTIAL_TRIPLETS", Strings::join(";", partials)}, + }; + + if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) + { + variables.push_back({"GIT", git_exe_path}); + } + + return variables; + } + + static std::string make_build_cmd(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const BuildPackageConfig& config, + const Triplet& triplet) + { + const Toolset& toolset = paths.get_toolset(pre_build_info); + const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); + std::vector variables = + get_cmake_vars(paths, pre_build_info, config, triplet, toolset); + + const std::string cmd_launch_cmake = System::make_cmake_cmd(cmake_exe_path, paths.ports_cmake, variables); + + std::string command = make_build_env_cmd(pre_build_info, toolset); + if (!command.empty()) + { +#ifdef _WIN32 + command.append(" & "); +#else + command.append(" && "); +#endif + } + + command.append(cmd_launch_cmake); + + return command; + } + + static std::string get_triplet_abi(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const Triplet& triplet) + { + static std::map s_hash_cache; + + const fs::path triplet_file_path = paths.get_triplet_file_path(triplet); + const auto& fs = paths.get_filesystem(); + + std::string hash; + + auto it_hash = s_hash_cache.find(triplet_file_path); + if (it_hash != s_hash_cache.end()) + { + hash = it_hash->second; + } + else + { + hash = Hash::get_file_hash(fs, triplet_file_path, "SHA1"); + + if (auto p = pre_build_info.external_toolchain_file.get()) + { + hash += "-"; + hash += Hash::get_file_hash(fs, *p, "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Linux") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "linux.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Darwin") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "osx.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "FreeBSD") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "freebsd.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Android") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "android.cmake", "SHA1"); + } + + s_hash_cache.emplace(triplet_file_path, hash); + } + + return hash; + } + static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, const PreBuildInfo& pre_build_info, const PackageSpec& spec, @@ -372,72 +547,16 @@ namespace vcpkg::Build System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); } -#if !defined(_WIN32) - // TODO: remove when vcpkg.exe is in charge for acquiring tools. Change introduced in vcpkg v0.0.107. - // bootstrap should have already downloaded ninja, but making sure it is present in case it was deleted. - vcpkg::Util::unused(paths.get_tool_exe(Tools::NINJA)); -#endif - - const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); - const fs::path& git_exe_path = paths.get_tool_exe(Tools::GIT); -#if defined(_WIN32) - const fs::path& powershell_exe_path = paths.get_tool_exe("powershell-core"); - if (!fs.exists(powershell_exe_path.parent_path() / "powershell.exe")) - { - fs.copy(powershell_exe_path, powershell_exe_path.parent_path() / "powershell.exe", fs::copy_options::none); - } -#endif - - std::string all_features; - for (auto& feature : config.scf.feature_paragraphs) - { - all_features.append(feature->name + ";"); - } - - const Toolset& toolset = paths.get_toolset(pre_build_info); - - std::vector variables{ - {"CMD", "BUILD"}, - {"PORT", config.scf.core_paragraph->name}, - {"CURRENT_PORT_DIR", config.port_dir}, - {"TARGET_TRIPLET", spec.triplet().canonical_name()}, - {"TARGET_TRIPLET_FILE", triplet_file_path}, - {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, - {"VCPKG_USE_HEAD_VERSION", Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, - {"DOWNLOADS", paths.downloads}, - {"_VCPKG_NO_DOWNLOADS", !Util::Enum::to_bool(config.build_package_options.allow_downloads) ? "1" : "0"}, - {"_VCPKG_DOWNLOAD_TOOL", to_string(config.build_package_options.download_tool)}, - {"FEATURES", Strings::join(";", config.feature_list)}, - {"ALL_FEATURES", all_features}, - {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())}, - }; - - if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) - { - variables.push_back({"GIT", git_exe_path}); - } - - const std::string cmd_launch_cmake = System::make_cmake_cmd(cmake_exe_path, paths.ports_cmake, variables); - - auto command = make_build_env_cmd(pre_build_info, toolset); - if (!command.empty()) - { -#ifdef _WIN32 - command.append(" & "); -#else - command.append(" && "); -#endif - } - command.append(cmd_launch_cmake); const auto timer = Chrono::ElapsedTimer::create_started(); - const int return_code = System::cmd_execute_clean( - command, - {} -#ifdef _WIN32 - , - powershell_exe_path.parent_path().u8string() + ";" -#endif - ); + + std::string command = + make_build_cmd(paths, pre_build_info, config, triplet); + std::unordered_map env = + make_env_passthrough(pre_build_info); + + const int return_code = + System::cmd_execute_clean(command, env); + const auto buildtimeus = timer.microseconds(); const auto spec_string = spec.to_string(); @@ -711,7 +830,7 @@ namespace vcpkg::Build { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); - auto archive_result = decompress_archive(paths, spec, archive_path); + int archive_result = decompress_archive(paths, spec, archive_path); if (archive_result != 0) { @@ -946,111 +1065,60 @@ namespace vcpkg::Build const std::string variable_name = s.at(0); const std::string variable_value = variable_with_no_value ? "" : s.at(1); - if (variable_name == "VCPKG_TARGET_ARCHITECTURE") + auto maybe_option = VCPKG_OPTIONS.find(variable_name); + if (maybe_option != VCPKG_OPTIONS.end()) { - pre_build_info.target_architecture = variable_value; - continue; + switch (maybe_option->second) + { + case VcpkgTripletVar::TARGET_ARCHITECTURE : + pre_build_info.target_architecture = variable_value; + break; + case VcpkgTripletVar::CMAKE_SYSTEM_NAME : + pre_build_info.cmake_system_name = variable_value; + break; + case VcpkgTripletVar::CMAKE_SYSTEM_VERSION : + pre_build_info.cmake_system_version = variable_value; + break; + case VcpkgTripletVar::PLATFORM_TOOLSET : + pre_build_info.platform_toolset = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::VISUAL_STUDIO_PATH : + pre_build_info.visual_studio_path = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE : + pre_build_info.external_toolchain_file = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::BUILD_TYPE : + if (variable_value.empty()) + pre_build_info.build_type = nullopt; + else if (Strings::case_insensitive_ascii_equals(variable_value, "debug")) + pre_build_info.build_type = ConfigurationType::DEBUG; + else if (Strings::case_insensitive_ascii_equals(variable_value, "release")) + pre_build_info.build_type = ConfigurationType::RELEASE; + else + Checks::exit_with_message( + VCPKG_LINE_INFO, "Unknown setting for VCPKG_BUILD_TYPE: %s", variable_value); + break; + case VcpkgTripletVar::ENV_PASSTHROUGH : + pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); + break; + } } - - if (variable_name == "VCPKG_CMAKE_SYSTEM_NAME") + else { - pre_build_info.cmake_system_name = variable_value; - continue; + Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown variable name %s", line); } - - if (variable_name == "VCPKG_CMAKE_SYSTEM_VERSION") - { - pre_build_info.cmake_system_version = variable_value; - continue; - } - - if (variable_name == "VCPKG_PLATFORM_TOOLSET") - { - pre_build_info.platform_toolset = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; - } - - if (variable_name == "VCPKG_VISUAL_STUDIO_PATH") - { - pre_build_info.visual_studio_path = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; - } - - if (variable_name == "VCPKG_CHAINLOAD_TOOLCHAIN_FILE") - { - pre_build_info.external_toolchain_file = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; - } - - if (variable_name == "VCPKG_BUILD_TYPE") - { - if (variable_value.empty()) - pre_build_info.build_type = nullopt; - else if (Strings::case_insensitive_ascii_equals(variable_value, "debug")) - pre_build_info.build_type = ConfigurationType::DEBUG; - else if (Strings::case_insensitive_ascii_equals(variable_value, "release")) - pre_build_info.build_type = ConfigurationType::RELEASE; - else - Checks::exit_with_message( - VCPKG_LINE_INFO, "Unknown setting for VCPKG_BUILD_TYPE: %s", variable_value); - continue; - } - - Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown variable name %s", line); } - pre_build_info.triplet_abi_tag = [&]() { - const auto& fs = paths.get_filesystem(); - static std::map s_hash_cache; - - auto it_hash = s_hash_cache.find(triplet_file_path); - if (it_hash != s_hash_cache.end()) - { - return it_hash->second; - } - auto hash = Hash::get_file_hash(fs, triplet_file_path, "SHA1"); - - if (auto p = pre_build_info.external_toolchain_file.get()) - { - hash += "-"; - hash += Hash::get_file_hash(fs, *p, "SHA1"); - } - else if (pre_build_info.cmake_system_name.empty() || - pre_build_info.cmake_system_name == "WindowsStore") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "windows.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Linux") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "linux.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Darwin") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "osx.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "FreeBSD") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "freebsd.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Android") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "android.cmake", "SHA1"); - } - - s_hash_cache.emplace(triplet_file_path, hash); - return hash; - }(); + pre_build_info.triplet_abi_tag = + get_triplet_abi(paths, pre_build_info, triplet); return pre_build_info; } + ExtendedBuildResult::ExtendedBuildResult(BuildResult code) : code(code) {} ExtendedBuildResult::ExtendedBuildResult(BuildResult code, std::unique_ptr&& bcf) : code(code), binary_control_file(std::move(bcf)) diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 9bc59cbe71b..1a52bd05f41 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -238,6 +238,28 @@ namespace vcpkg return ret; } + std::vector filter_dependencies_to_features(const std::vector& deps, + const Triplet& t) + { + std::vector ret; + for (auto&& dep : deps) + { + auto qualifiers = Strings::split(dep.qualifier, "&"); + if (std::all_of(qualifiers.begin(), qualifiers.end(), [&](const std::string& qualifier) { + if (qualifier.empty()) return true; + if (qualifier[0] == '!') + { + return t.canonical_name().find(qualifier.substr(1)) == std::string::npos; + } + return t.canonical_name().find(qualifier) != std::string::npos; + })) + { + ret.emplace_back(dep.depend); + } + } + return ret; + } + std::vector filter_dependencies_to_specs(const std::vector& deps, const Triplet& t) { return FeatureSpec::from_strings_and_triplet(filter_dependencies(deps, t), t); diff --git a/triplets/x64-linux.cmake b/triplets/x64-linux.cmake index 5196184638b..05cb0294057 100644 --- a/triplets/x64-linux.cmake +++ b/triplets/x64-linux.cmake @@ -3,3 +3,5 @@ set(VCPKG_CRT_LINKAGE dynamic) set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_ENV_PASSTHROUGH FC) From 6bef95b6f50b2bb602781d6d8898d27639294ff6 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 14:43:56 -0700 Subject: [PATCH 02/34] remove partials --- toolsrc/src/vcpkg/build.cpp | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index e0c78f2da31..dd2beec9d9f 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -317,26 +317,6 @@ namespace vcpkg::Build ); } - static std::vector get_partials(const VcpkgPaths& paths, - const std::vector& dependencies, - const Triplet& triplet) - { - std::vector ret; - Files::Filesystem& fs = paths.get_filesystem(); - - for (const std::string& dependency : dependencies) - { - fs::path partial = - paths.buildtrees / dependency / triplet.canonical_name() / "partial_triplet.cmake"; - if (fs.is_regular_file(partial)) - { - ret.emplace_back(std::move(partial)); - } - } - - return ret; - } - static std::vector compute_required_feature_specs(const BuildPackageConfig& config, const StatusParagraphs& status_db) { @@ -415,15 +395,6 @@ namespace vcpkg::Build all_features.append(feature->name + ";"); } - std::vector partials = - Util::fmap( - get_partials(paths, get_dependency_names(config.scf, config.feature_list, triplet), triplet), - [&](const fs::path& path) - { - return path.u8string(); - } - ); - std::vector variables{ {"CMD", "BUILD"}, {"PORT", config.scf.core_paragraph->name}, @@ -438,7 +409,6 @@ namespace vcpkg::Build {"FEATURES", Strings::join(";", config.feature_list)}, {"ALL_FEATURES", all_features}, {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())}, - {"VCPKG_PARTIAL_TRIPLETS", Strings::join(";", partials)}, }; if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) From a22242e837cc1af63ff754545512f5a754a70426 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 14:45:52 -0700 Subject: [PATCH 03/34] remove change to linux triplet --- triplets/x64-linux.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/triplets/x64-linux.cmake b/triplets/x64-linux.cmake index 05cb0294057..5196184638b 100644 --- a/triplets/x64-linux.cmake +++ b/triplets/x64-linux.cmake @@ -3,5 +3,3 @@ set(VCPKG_CRT_LINKAGE dynamic) set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Linux) - -set(VCPKG_ENV_PASSTHROUGH FC) From 44dcc3d4f3d2bc56cc9f6d0371a141ad0145d537 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 15:34:13 -0700 Subject: [PATCH 04/34] First pass at port settings --- scripts/get_triplet_environment.cmake | 3 ++ toolsrc/include/vcpkg/build.h | 48 ++++++++++++++------------- toolsrc/src/vcpkg/build.cpp | 21 +++++++++--- toolsrc/src/vcpkg/commands.ci.cpp | 9 ++++- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index 24ff409058b..e561492debd 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -1,4 +1,7 @@ include(${CMAKE_TRIPLET_FILE}) +if (DEFINED CMAKE_PORTFILE_SETTINGS) + include(${CMAKE_PORTFILE_SETTINGS} OPTIONAL) +endif() # GUID used as a flag - "cut here line" message("c35112b6-d1ba-415b-aa5d-81de856ef8eb") diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index e26597376f0..0d10d12a4f1 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -140,29 +140,6 @@ namespace vcpkg::Build {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, }; - /// - /// Settings from the triplet file which impact the build environment and post-build checks - /// - struct PreBuildInfo - { - /// - /// Runs the triplet file in a "capture" mode to create a PreBuildInfo - /// - static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, const Triplet& triplet); - - std::string triplet_abi_tag; - std::string target_architecture; - std::string cmake_system_name; - std::string cmake_system_version; - Optional platform_toolset; - Optional visual_studio_path; - Optional external_toolchain_file; - Optional build_type; - std::vector passthrough_env_vars; - }; - - std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); - struct ExtendedBuildResult { ExtendedBuildResult(BuildResult code); @@ -200,6 +177,31 @@ namespace vcpkg::Build const BuildPackageConfig& config, const StatusParagraphs& status_db); + /// + /// Settings from the triplet file which impact the build environment and post-build checks + /// + struct PreBuildInfo + { + /// + /// Runs the triplet file in a "capture" mode to create a PreBuildInfo + /// + static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, + const Triplet& triplet, + Optional port = nullopt); + + std::string triplet_abi_tag; + std::string target_architecture; + std::string cmake_system_name; + std::string cmake_system_version; + Optional platform_toolset; + Optional visual_studio_path; + Optional external_toolchain_file; + Optional build_type; + std::vector passthrough_env_vars; + }; + + std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); + enum class BuildPolicy { EMPTY_PACKAGE, diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index dd2beec9d9f..646da739825 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -782,7 +782,8 @@ namespace vcpkg::Build AbiEntry{status_it->get()->package.spec.name(), status_it->get()->package.abi}); } - const auto pre_build_info = PreBuildInfo::from_triplet_file(paths, triplet); + const auto pre_build_info = + PreBuildInfo::from_triplet_file(paths, triplet, config.scf.core_paragraph->name); auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); @@ -997,7 +998,9 @@ namespace vcpkg::Build return inner_create_buildinfo(*pghs.get()); } - PreBuildInfo PreBuildInfo::from_triplet_file(const VcpkgPaths& paths, const Triplet& triplet) + PreBuildInfo PreBuildInfo::from_triplet_file(const VcpkgPaths& paths, + const Triplet& triplet, + Optional port) { static constexpr CStringView FLAG_GUID = "c35112b6-d1ba-415b-aa5d-81de856ef8eb"; @@ -1005,11 +1008,19 @@ namespace vcpkg::Build const fs::path ports_cmake_script_path = paths.scripts / "get_triplet_environment.cmake"; const fs::path triplet_file_path = paths.get_triplet_file_path(triplet); + std::vector args{{"CMAKE_TRIPLET_FILE", triplet_file_path}}; + + if (port) + { + args.emplace_back( + "CMAKE_PORT_SETTINGS", + paths.ports / port.value_or_exit(VCPKG_LINE_INFO) / "port_settings.cmake"); + } + const auto cmd_launch_cmake = System::make_cmake_cmd(cmake_exe_path, ports_cmake_script_path, - { - {"CMAKE_TRIPLET_FILE", triplet_file_path}, - }); + args); + const auto ec_data = System::cmd_execute_and_capture_output(cmd_launch_cmake); Checks::check_exit(VCPKG_LINE_INFO, ec_data.exit_code == 0, ec_data.output); diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index c12c26ff7ba..493c052cb7e 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -254,7 +254,14 @@ namespace vcpkg::Commands::CI return {spec.name(), it->second}; }); const auto& pre_build_info = pre_build_info_cache.get_lazy( - triplet, [&]() { return Build::PreBuildInfo::from_triplet_file(paths, triplet); }); + triplet, + [&]() { + return Build::PreBuildInfo::from_triplet_file( + paths, + triplet, + scfl->source_control_file->core_paragraph->name); + } + ); auto maybe_tag_and_file = Build::compute_abi_tag(paths, build_config, pre_build_info, dependency_abis); From 64198a8109b9d1cffcc3830f70d0c3f2b066638d Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 15:51:50 -0700 Subject: [PATCH 05/34] Add to vcpkg.cmake --- scripts/get_triplet_environment.cmake | 4 ++-- scripts/ports.cmake | 1 + toolsrc/src/vcpkg/build.cpp | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index e561492debd..f76054556e3 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -1,6 +1,6 @@ include(${CMAKE_TRIPLET_FILE}) -if (DEFINED CMAKE_PORTFILE_SETTINGS) - include(${CMAKE_PORTFILE_SETTINGS} OPTIONAL) +if (DEFINED CMAKE_PORT_SETTINGS) + include(${CMAKE_PORT_SETTINGS} OPTIONAL) endif() # GUID used as a flag - "cut here line" diff --git a/scripts/ports.cmake b/scripts/ports.cmake index 860d08893fe..0c33fdffcc9 100644 --- a/scripts/ports.cmake +++ b/scripts/ports.cmake @@ -67,6 +67,7 @@ if(CMD MATCHES "^BUILD$") file(MAKE_DIRECTORY ${CURRENT_BUILDTREES_DIR} ${CURRENT_PACKAGES_DIR}) include(${CMAKE_TRIPLET_FILE}) + include(${CMAKE_PORT_SETTINGS} OPTIONAL) set(TRIPLET_SYSTEM_ARCH ${VCPKG_TARGET_ARCHITECTURE}) include(${CMAKE_CURRENT_LIST_DIR}/cmake/vcpkg_common_definitions.cmake) include(${CMAKE_CURRENT_LIST_DIR}/cmake/vcpkg_common_functions.cmake) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 646da739825..03e0c0b1d27 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -401,6 +401,7 @@ namespace vcpkg::Build {"CURRENT_PORT_DIR", config.port_dir}, {"TARGET_TRIPLET", triplet.canonical_name()}, {"TARGET_TRIPLET_FILE", paths.get_triplet_file_path(triplet).u8string()}, + {"CMAKE_PORT_SETTINGS", config.port_dir / "port_settings.cmake"}, {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, {"VCPKG_USE_HEAD_VERSION", Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, {"DOWNLOADS", paths.downloads}, From 7d9d457f589d82d6675c511636808a0fbee7a4e5 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 16:09:30 -0700 Subject: [PATCH 06/34] revert unecessary reordering --- toolsrc/include/vcpkg/build.h | 92 +++++++++++++++++------------------ 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 0d10d12a4f1..a0ad64ffd67 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -117,28 +117,53 @@ namespace vcpkg::Build std::string create_error_message(const BuildResult build_result, const PackageSpec& spec); std::string create_user_troubleshooting_message(const PackageSpec& spec); - enum class VcpkgTripletVar - { - TARGET_ARCHITECTURE = 0, - CMAKE_SYSTEM_NAME, - CMAKE_SYSTEM_VERSION, - PLATFORM_TOOLSET, - VISUAL_STUDIO_PATH, - CHAINLOAD_TOOLCHAIN_FILE, - BUILD_TYPE, - ENV_PASSTHROUGH, - }; + /// + /// Settings from the triplet file which impact the build environment and post-build checks + /// + struct PreBuildInfo + { + /// + /// Runs the triplet file in a "capture" mode to create a PreBuildInfo + /// + static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, + const Triplet& triplet, + Optional port = nullopt); - const std::unordered_map VCPKG_OPTIONS = { - {"VCPKG_TARGET_ARCHITECTURE", VcpkgTripletVar::TARGET_ARCHITECTURE}, - {"VCPKG_CMAKE_SYSTEM_NAME", VcpkgTripletVar::CMAKE_SYSTEM_NAME}, - {"VCPKG_CMAKE_SYSTEM_VERSION", VcpkgTripletVar::CMAKE_SYSTEM_VERSION}, - {"VCPKG_PLATFORM_TOOLSET", VcpkgTripletVar::PLATFORM_TOOLSET}, - {"VCPKG_VISUAL_STUDIO_PATH", VcpkgTripletVar::VISUAL_STUDIO_PATH}, - {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, - {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, - {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, - }; + std::string triplet_abi_tag; + std::string target_architecture; + std::string cmake_system_name; + std::string cmake_system_version; + Optional platform_toolset; + Optional visual_studio_path; + Optional external_toolchain_file; + Optional build_type; + std::vector passthrough_env_vars; + }; + + std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); + + enum class VcpkgTripletVar + { + TARGET_ARCHITECTURE = 0, + CMAKE_SYSTEM_NAME, + CMAKE_SYSTEM_VERSION, + PLATFORM_TOOLSET, + VISUAL_STUDIO_PATH, + CHAINLOAD_TOOLCHAIN_FILE, + BUILD_TYPE, + ENV_PASSTHROUGH, + }; + + const std::unordered_map VCPKG_OPTIONS = { + {"VCPKG_TARGET_ARCHITECTURE", VcpkgTripletVar::TARGET_ARCHITECTURE}, + {"VCPKG_CMAKE_SYSTEM_NAME", VcpkgTripletVar::CMAKE_SYSTEM_NAME}, + {"VCPKG_CMAKE_SYSTEM_VERSION", VcpkgTripletVar::CMAKE_SYSTEM_VERSION}, + {"VCPKG_PLATFORM_TOOLSET", VcpkgTripletVar::PLATFORM_TOOLSET}, + {"VCPKG_VISUAL_STUDIO_PATH", VcpkgTripletVar::VISUAL_STUDIO_PATH}, + {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, + {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, + {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + }; struct ExtendedBuildResult { @@ -177,31 +202,6 @@ namespace vcpkg::Build const BuildPackageConfig& config, const StatusParagraphs& status_db); - /// - /// Settings from the triplet file which impact the build environment and post-build checks - /// - struct PreBuildInfo - { - /// - /// Runs the triplet file in a "capture" mode to create a PreBuildInfo - /// - static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, - const Triplet& triplet, - Optional port = nullopt); - - std::string triplet_abi_tag; - std::string target_architecture; - std::string cmake_system_name; - std::string cmake_system_version; - Optional platform_toolset; - Optional visual_studio_path; - Optional external_toolchain_file; - Optional build_type; - std::vector passthrough_env_vars; - }; - - std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); - enum class BuildPolicy { EMPTY_PACKAGE, From d4ab567609495a5c239f537d7c1e200f7b8a19c0 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 10:10:36 -0700 Subject: [PATCH 07/34] first pass at abi additional files --- scripts/get_triplet_environment.cmake | 1 + toolsrc/include/vcpkg/build.h | 3 +++ toolsrc/src/vcpkg/build.cpp | 32 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index f76054556e3..ab5e8f8c346 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -13,3 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}") message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}") message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}") message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}") +message("VCPKG_ABI_ADDITIONAL_FILES=${VCPKG_ABI_ADDITIONAL_FILES}") diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index a0ad64ffd67..85d6ebed9a3 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -138,6 +138,7 @@ namespace vcpkg::Build Optional external_toolchain_file; Optional build_type; std::vector passthrough_env_vars; + std::vector additional_files; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); @@ -152,6 +153,7 @@ namespace vcpkg::Build CHAINLOAD_TOOLCHAIN_FILE, BUILD_TYPE, ENV_PASSTHROUGH, + ABI_ADDITIONAL_FILES, }; const std::unordered_map VCPKG_OPTIONS = { @@ -163,6 +165,7 @@ namespace vcpkg::Build {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + {"VCPKG_ABI_ADDITIONAL_FILES", VcpkgTripletVar::ABI_ADDITIONAL_FILES}, }; struct ExtendedBuildResult diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 03e0c0b1d27..daed6f6959e 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -496,6 +496,31 @@ namespace vcpkg::Build s_hash_cache.emplace(triplet_file_path, hash); } + for (const fs::path& additional_file : pre_build_info.additional_files) + { + it_hash = s_hash_cache.find(additional_file); + + if (it_hash != s_hash_cache.end()) + { + hash += "-"; + hash += it_hash->second; + } + else if (fs.is_regular_file(additional_file)) + { + std::string tmp_hash = Hash::get_file_hash(fs, additional_file, "SHA1"); + hash += "-"; + hash += tmp_hash; + + s_hash_cache.emplace(additional_file, tmp_hash); + } + else + { + Checks::exit_with_message( + VCPKG_LINE_INFO, + additional_file + " was listed as an additional file for calculating the abi, but was not found."); + } + } + return hash; } @@ -1087,6 +1112,13 @@ namespace vcpkg::Build case VcpkgTripletVar::ENV_PASSTHROUGH : pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; + case VcpkgTripletVar::ABI_ADDITIONAL_FILES : + pre_build_info.additional_files = Util::fmap(Strings::split(variable_value, ";"), + [](const std::string& path) + { + return fs::path{path}; + }); + break; } } else From e81d22ddec6887a497055d4804a004ca662b4526 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 10:18:20 -0700 Subject: [PATCH 08/34] Convert name of file to u8 string, to compile on windows --- toolsrc/src/vcpkg/build.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index daed6f6959e..b88d4fd1a73 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -517,7 +517,7 @@ namespace vcpkg::Build { Checks::exit_with_message( VCPKG_LINE_INFO, - additional_file + " was listed as an additional file for calculating the abi, but was not found."); + additional_file.u8string() + " was listed as an additional file for calculating the abi, but was not found."); } } From f0f615532f9aa03f1518f03009cff5c716e9b1fd Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 11:40:27 -0700 Subject: [PATCH 09/34] always calculate abi --- toolsrc/src/vcpkg/build.cpp | 144 ++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 71 deletions(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index b88d4fd1a73..47599bfca0c 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -623,8 +623,6 @@ namespace vcpkg::Build const PreBuildInfo& pre_build_info, Span dependency_abis) { - if (config.build_package_options.binary_caching == BinaryCaching::NO) return nullopt; - auto& fs = paths.get_filesystem(); const Triplet& triplet = config.triplet; const std::string& name = config.scf.core_paragraph->name; @@ -713,7 +711,7 @@ namespace vcpkg::Build } System::print2( - "Warning: binary caching disabled because abi keys are missing values:\n", + "Warning: abi keys are missing values:\n", Strings::join("", abi_tag_entries_missing, [](const AbiEntry& e) { return " " + e.key + "\n"; }), "\n"); @@ -814,15 +812,14 @@ namespace vcpkg::Build auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); const auto abi_tag_and_file = maybe_abi_tag_and_file.get(); + const fs::path archives_root_dir = paths.root / "archives"; + const std::string archive_name = abi_tag_and_file->tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; if (config.build_package_options.binary_caching == BinaryCaching::YES && abi_tag_and_file) { - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag_and_file->tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - if (fs.exists(archive_path)) { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); @@ -855,70 +852,75 @@ namespace vcpkg::Build } System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); - - ExtendedBuildResult result = do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config); - - std::error_code ec; - fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); - auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); - Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); - - if (result.code == BuildResult::SUCCEEDED) - { - const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); - - compress_archive(paths, spec, tmp_archive_path); - - fs.create_directories(archive_path.parent_path(), ec); - fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); - if (ec) - { - System::printf(System::Color::warning, - "Failed to store binary cache %s: %s\n", - archive_path.u8string(), - ec.message()); - } - else - System::printf("Stored binary cache: %s\n", archive_path.u8string()); - } - else if (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED) - { - if (!fs.exists(archive_tombstone_path)) - { - // Build failed, store all failure logs in the tombstone. - const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; - const auto tmp_log_path_destination = tmp_log_path / spec.name(); - const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; - fs.create_directories(tmp_log_path_destination, ec); - - for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) - { - if (log_file.path().extension() == ".log") - { - fs.copy_file(log_file.path(), - tmp_log_path_destination / log_file.path().filename(), - fs::stdfs::copy_options::none, - ec); - } - } - - compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); - - fs.create_directories(archive_tombstone_path.parent_path(), ec); - fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); - - // clean up temporary directory - fs.remove_all(tmp_log_path, ec); - } - } - - return result; } - return do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config); + ExtendedBuildResult result = + do_build_package_and_clean_buildtrees( + paths, + pre_build_info, + spec, + maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, + config); + + std::error_code ec; + fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); + auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; + fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); + + if (config.build_package_options.binary_caching == BinaryCaching::YES && + result.code == BuildResult::SUCCEEDED) + { + const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); + + compress_archive(paths, spec, tmp_archive_path); + + fs.create_directories(archive_path.parent_path(), ec); + fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); + if (ec) + { + System::printf(System::Color::warning, + "Failed to store binary cache %s: %s\n", + archive_path.u8string(), + ec.message()); + } + else + System::printf("Stored binary cache: %s\n", archive_path.u8string()); + } + else if (config.build_package_options.binary_caching == BinaryCaching::YES && + (result.code == BuildResult::BUILD_FAILED || + result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) + { + if (!fs.exists(archive_tombstone_path)) + { + // Build failed, store all failure logs in the tombstone. + const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; + const auto tmp_log_path_destination = tmp_log_path / spec.name(); + const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; + fs.create_directories(tmp_log_path_destination, ec); + + for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) + { + if (log_file.path().extension() == ".log") + { + fs.copy_file(log_file.path(), + tmp_log_path_destination / log_file.path().filename(), + fs::stdfs::copy_options::none, + ec); + } + } + + compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); + + fs.create_directories(archive_tombstone_path.parent_path(), ec); + fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); + + // clean up temporary directory + fs.remove_all(tmp_log_path, ec); + } + } + + return result; } const std::string& to_string(const BuildResult build_result) From 58958eb0ea47c6c423fe78f2cd6fd1e31cbcb082 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 14:27:18 -0700 Subject: [PATCH 10/34] sourceparagraph changes --- toolsrc/include/vcpkg/sourceparagraph.h | 10 ++++++++ toolsrc/src/vcpkg/sourceparagraph.cpp | 32 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index 9fbd83475d8..9a1c2c843c8 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -46,6 +46,12 @@ namespace vcpkg /// struct SourceParagraph { + enum TYPE : unsigned + { + PORT = 0, + SYS_TOOL, + }; + std::string name; std::string version; std::string description; @@ -54,6 +60,10 @@ namespace vcpkg std::vector supports; std::vector depends; std::vector default_features; + TYPE type; + + static TYPE type_from_string(const std::string& in); + static std::string string_from_type(const TYPE& in); }; /// diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 1a52bd05f41..26a7ab118f1 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -25,6 +25,7 @@ namespace vcpkg static const std::string SUPPORTS = "Supports"; static const std::string VERSION = "Version"; static const std::string HOMEPAGE = "Homepage"; + static const std::string TYPE = "Type"; } static Span get_list_of_valid_fields() @@ -36,6 +37,7 @@ namespace vcpkg SourceParagraphFields::MAINTAINER, SourceParagraphFields::BUILD_DEPENDS, SourceParagraphFields::HOMEPAGE, + SourceParagraphFields::TYPE, }; return valid_fields; @@ -98,6 +100,35 @@ namespace vcpkg } } + static SourceParagraph::TYPE type_from_string(const std::string& in) + { + if (Strings::equals(in, "port") || Strings::equals(in, "")) + { + return SourceParagraph::PORT; + } + + if (Strings::equals(in, "sys-tool")) + { + return SourceParagraph::SYS_TOOL; + } + + System::print2( + in, " is not a valid control file type. Valid types are:", + "\n port\n sys-tool"); + + Checks::exit_fail(VCPKG_LINE_INFO); + } + + static std::string string_from_type(const SourceParagraph::TYPE& in) + { + switch (in) + { + case SourceParagraph::PORT : return "port"; + case SourceParagraph::SYS_TOOL : return "sys-tool"; + default : Checks::exit_with_message(VCPKG_LINE_INFO, "Invalid CONTROL_TYPE value."); + } + } + static ParseExpected parse_source_paragraph(RawParagraph&& fields) { ParagraphParser parser(std::move(fields)); @@ -114,6 +145,7 @@ namespace vcpkg parse_comma_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS))); spgh->supports = parse_comma_list(parser.optional_field(SourceParagraphFields::SUPPORTS)); spgh->default_features = parse_comma_list(parser.optional_field(SourceParagraphFields::DEFAULTFEATURES)); + spgh->type = type_from_string(parser.optional_field(SourceParagraphFields::TYPE)); auto err = parser.error_info(spgh->name); if (err) From f18ffe996877a058da9e0208f92331c83517f6a0 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 16:04:05 -0700 Subject: [PATCH 11/34] Add type field --- toolsrc/include/vcpkg/binaryparagraph.h | 1 + toolsrc/src/vcpkg/binaryparagraph.cpp | 10 ++++++++-- toolsrc/src/vcpkg/sourceparagraph.cpp | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index 3315151c69d..fa49edaba63 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -31,6 +31,7 @@ namespace vcpkg std::vector default_features; std::vector depends; std::string abi; + SourceParagraph::TYPE type; }; struct BinaryControlFile diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index 4b80debab6a..ad7790fe1b3 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -23,6 +23,7 @@ namespace vcpkg static const std::string MAINTAINER = "Maintainer"; static const std::string DEPENDS = "Depends"; static const std::string DEFAULTFEATURES = "Default-Features"; + static const std::string TYPE = "Type"; } BinaryParagraph::BinaryParagraph() = default; @@ -60,6 +61,9 @@ namespace vcpkg this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES)); } + this->type = + SourceParagraph::type_from_string(parser.optional_field(Fields::TYPE)); + if (const auto err = parser.error_info(this->spec.to_string())) { System::print2(System::Color::error, "Error: while parsing the Binary Paragraph for ", this->spec, '\n'); @@ -72,14 +76,14 @@ namespace vcpkg } BinaryParagraph::BinaryParagraph(const SourceParagraph& spgh, const Triplet& triplet, const std::string& abi_tag) - : version(spgh.version), description(spgh.description), maintainer(spgh.maintainer), abi(abi_tag) + : version(spgh.version), description(spgh.description), maintainer(spgh.maintainer), abi(abi_tag), type(spgh.type) { this->spec = PackageSpec::from_name_and_triplet(spgh.name, triplet).value_or_exit(VCPKG_LINE_INFO); this->depends = filter_dependencies(spgh.depends, triplet); } BinaryParagraph::BinaryParagraph(const SourceParagraph& spgh, const FeatureParagraph& fpgh, const Triplet& triplet) - : version(), description(fpgh.description), maintainer(), feature(fpgh.name) + : version(), description(fpgh.description), maintainer(), feature(fpgh.name), type(spgh.type) { this->spec = PackageSpec::from_name_and_triplet(spgh.name, triplet).value_or_exit(VCPKG_LINE_INFO); this->depends = filter_dependencies(fpgh.depends, triplet); @@ -119,5 +123,7 @@ namespace vcpkg if (!pgh.maintainer.empty()) out_str.append("Maintainer: ").append(pgh.maintainer).push_back('\n'); if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n'); if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n'); + + out_str.append("Type: ").append(SourceParagraph::string_from_type(pgh.type)).push_back('\n'); } } diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 26a7ab118f1..6af93a99b8c 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -100,7 +100,7 @@ namespace vcpkg } } - static SourceParagraph::TYPE type_from_string(const std::string& in) + SourceParagraph::TYPE SourceParagraph::type_from_string(const std::string& in) { if (Strings::equals(in, "port") || Strings::equals(in, "")) { @@ -119,7 +119,7 @@ namespace vcpkg Checks::exit_fail(VCPKG_LINE_INFO); } - static std::string string_from_type(const SourceParagraph::TYPE& in) + std::string SourceParagraph::string_from_type(const SourceParagraph::TYPE& in) { switch (in) { @@ -145,7 +145,7 @@ namespace vcpkg parse_comma_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS))); spgh->supports = parse_comma_list(parser.optional_field(SourceParagraphFields::SUPPORTS)); spgh->default_features = parse_comma_list(parser.optional_field(SourceParagraphFields::DEFAULTFEATURES)); - spgh->type = type_from_string(parser.optional_field(SourceParagraphFields::TYPE)); + spgh->type = SourceParagraph::type_from_string(parser.optional_field(SourceParagraphFields::TYPE)); auto err = parser.error_info(spgh->name); if (err) From d39bd70d533c64e929d4399cb9a1bdbfe0efaecd Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Thu, 18 Jul 2019 13:24:31 -0700 Subject: [PATCH 12/34] add needs_rebuild, should probably be moved to somewhere else --- toolsrc/include/vcpkg/statusparagraphs.h | 2 ++ toolsrc/src/vcpkg/statusparagraphs.cpp | 25 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/toolsrc/include/vcpkg/statusparagraphs.h b/toolsrc/include/vcpkg/statusparagraphs.h index fa064de7edd..3ebad67de64 100644 --- a/toolsrc/include/vcpkg/statusparagraphs.h +++ b/toolsrc/include/vcpkg/statusparagraphs.h @@ -62,6 +62,8 @@ namespace vcpkg /// `true` if installed, `false` if not or not found. bool is_installed(const FeatureSpec& spec) const; + bool needs_rebuild(const PackageSpec& spec); + iterator insert(std::unique_ptr); friend void serialize(const StatusParagraphs& pgh, std::string& out_str); diff --git a/toolsrc/src/vcpkg/statusparagraphs.cpp b/toolsrc/src/vcpkg/statusparagraphs.cpp index c642af59b6a..f204ed568a8 100644 --- a/toolsrc/src/vcpkg/statusparagraphs.cpp +++ b/toolsrc/src/vcpkg/statusparagraphs.cpp @@ -118,6 +118,31 @@ namespace vcpkg return it != end() && (*it)->is_installed(); } + bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec) + { + auto it = find(spec); + if (it != end()) + { + for (const std::string& dep : (*it)->package.depends) + { + PackageSpec dep_spec = + PackageSpec::from_name_and_triplet( + dep, + spec.triplet()).value_or_exit(VCPKG_LINE_INFO); + + if (needs_rebuild(dep_spec)) + { + (*it)->state = InstallState::NEEDS_REBUILD; + return true; + } + } + + return (*it)->needs_rebuild(); + } + + return false; + } + StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr pgh) { Checks::check_exit(VCPKG_LINE_INFO, pgh != nullptr, "Inserted null paragraph"); From 3b808a48ce2fafab82bd7b53642a11a08b21c481 Mon Sep 17 00:00:00 2001 From: Curtis J Bezault Date: Fri, 19 Jul 2019 08:08:56 -0700 Subject: [PATCH 13/34] Update VERSION.txt --- toolsrc/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolsrc/VERSION.txt b/toolsrc/VERSION.txt index d7d695c69cb..0342c24ab35 100644 --- a/toolsrc/VERSION.txt +++ b/toolsrc/VERSION.txt @@ -1 +1 @@ -"2019.07.18" +"2019.07.19" From 459908ae14458a615bb0a8c278c799cabd34e558 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 10:07:39 -0700 Subject: [PATCH 14/34] add external file hashes to the binary paragraph --- scripts/get_triplet_environment.cmake | 2 +- toolsrc/include/vcpkg/binaryparagraph.h | 1 + toolsrc/include/vcpkg/build.h | 6 +- toolsrc/src/vcpkg/binaryparagraph.cpp | 34 ++++- toolsrc/src/vcpkg/build.cpp | 157 ++++++++++++++++-------- toolsrc/src/vcpkg/statusparagraphs.cpp | 42 +++---- 6 files changed, 161 insertions(+), 81 deletions(-) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index ab5e8f8c346..fa1e5cb3969 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -13,4 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}") message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}") message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}") message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}") -message("VCPKG_ABI_ADDITIONAL_FILES=${VCPKG_ABI_ADDITIONAL_FILES}") +message("VCPKG_EXTERNAL_FILES=${VCPKG_EXTERNAL_FILES}") diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index fa49edaba63..ec14f8a97d3 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -32,6 +32,7 @@ namespace vcpkg std::vector depends; std::string abi; SourceParagraph::TYPE type; + std::unordered_map external_files; }; struct BinaryControlFile diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 85d6ebed9a3..5b52f436f0f 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -138,7 +138,7 @@ namespace vcpkg::Build Optional external_toolchain_file; Optional build_type; std::vector passthrough_env_vars; - std::vector additional_files; + std::vector> external_files; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); @@ -153,7 +153,7 @@ namespace vcpkg::Build CHAINLOAD_TOOLCHAIN_FILE, BUILD_TYPE, ENV_PASSTHROUGH, - ABI_ADDITIONAL_FILES, + EXTERNAL_FILES, }; const std::unordered_map VCPKG_OPTIONS = { @@ -165,7 +165,7 @@ namespace vcpkg::Build {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, - {"VCPKG_ABI_ADDITIONAL_FILES", VcpkgTripletVar::ABI_ADDITIONAL_FILES}, + {"VCPKG_EXTERNAL_FILES", VcpkgTripletVar::EXTERNAL_FILES}, }; struct ExtendedBuildResult diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index ad7790fe1b3..2e9415dab72 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -23,7 +24,7 @@ namespace vcpkg static const std::string MAINTAINER = "Maintainer"; static const std::string DEPENDS = "Depends"; static const std::string DEFAULTFEATURES = "Default-Features"; - static const std::string TYPE = "Type"; + static const std::string EXTERNALFILES = "External-Files"; } BinaryParagraph::BinaryParagraph() = default; @@ -61,8 +62,23 @@ namespace vcpkg this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES)); } - this->type = - SourceParagraph::type_from_string(parser.optional_field(Fields::TYPE)); + std::vector external_files_or_hashes = + parse_comma_list(parser.optional_field(Fields::EXTERNALFILES)); + + if (external_files_or_hashes.size() % 2 != 0) + { + Checks::exit_with_message( + VCPKG_LINE_INFO, + "The External-Files field is not composed of key-value pairs for ", + this->spec); + } + + for (int i = 0; i < external_files_or_hashes.size(); i += 2) + { + external_files.emplace( + std::move(external_files_or_hashes[i]), + std::move(external_files_or_hashes[i+1])); + } if (const auto err = parser.error_info(this->spec.to_string())) { @@ -124,6 +140,16 @@ namespace vcpkg if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n'); if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n'); - out_str.append("Type: ").append(SourceParagraph::string_from_type(pgh.type)).push_back('\n'); + if (!pgh.external_files.empty()) + { + out_str.append("External-Files: "); + out_str.append(Strings::join(",", + Util::fmap( + pgh.external_files, + [](const std::pair& kv) + { + return kv.first + "," + kv.second; + }))).push_back('\n'); + } } } diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 47599bfca0c..110a571b078 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -261,10 +262,12 @@ namespace vcpkg::Build return BinaryParagraph(source_paragraph, feature_paragraph, triplet); } - static std::unique_ptr create_binary_control_file(const SourceParagraph& source_paragraph, - const Triplet& triplet, - const BuildInfo& build_info, - const std::string& abi_tag) + static std::unique_ptr create_binary_control_file( + const SourceParagraph& source_paragraph, + const Triplet& triplet, + const BuildInfo& build_info, + const PreBuildInfo& pre_build_info, + const std::string& abi_tag) { auto bcf = std::make_unique(); BinaryParagraph bpgh(source_paragraph, triplet, abi_tag); @@ -272,6 +275,14 @@ namespace vcpkg::Build { bpgh.version = *p_ver; } + + for (auto& file_hash : pre_build_info.external_files) + { + bpgh.external_files.emplace( + std::move(file_hash.first), + std::move(file_hash.second)); + } + bcf->core_paragraph = std::move(bpgh); return bcf; } @@ -447,6 +458,41 @@ namespace vcpkg::Build return command; } + static std::vector> get_external_file_hashes( + const VcpkgPaths& paths, + const std::vector& files) + { + static std::map s_hash_cache; + + const auto& fs = paths.get_filesystem(); + + std::vector> hashes; + for (const fs::path& external_file : files) + { + auto it_hash = s_hash_cache.find(external_file); + + if (it_hash != s_hash_cache.end()) + { + hashes.emplace_back(external_file, it_hash->second); + } + else if (fs.is_regular_file(external_file)) + { + auto emp = s_hash_cache.emplace(external_file.u8string(), + Hash::get_file_hash(fs, external_file, "SHA1")); + hashes.emplace_back(external_file, emp.first->second); + } + else + { + Checks::exit_with_message( + VCPKG_LINE_INFO, + external_file.u8string() + + " was listed as an additional file for calculating the abi, but was not found."); + } + } + + return hashes; + } + static std::string get_triplet_abi(const VcpkgPaths& paths, const PreBuildInfo& pre_build_info, const Triplet& triplet) @@ -496,31 +542,6 @@ namespace vcpkg::Build s_hash_cache.emplace(triplet_file_path, hash); } - for (const fs::path& additional_file : pre_build_info.additional_files) - { - it_hash = s_hash_cache.find(additional_file); - - if (it_hash != s_hash_cache.end()) - { - hash += "-"; - hash += it_hash->second; - } - else if (fs.is_regular_file(additional_file)) - { - std::string tmp_hash = Hash::get_file_hash(fs, additional_file, "SHA1"); - hash += "-"; - hash += tmp_hash; - - s_hash_cache.emplace(additional_file, tmp_hash); - } - else - { - Checks::exit_with_message( - VCPKG_LINE_INFO, - additional_file.u8string() + " was listed as an additional file for calculating the abi, but was not found."); - } - } - return hash; } @@ -572,7 +593,13 @@ namespace vcpkg::Build const size_t error_count = PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir); - auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); + std::unique_ptr bcf = + create_binary_control_file( + *config.scf.core_paragraph, + triplet, + build_info, + pre_build_info, + abi_tag); if (error_count != 0) { @@ -629,7 +656,9 @@ namespace vcpkg::Build std::vector abi_tag_entries(dependency_abis.begin(), dependency_abis.end()); - abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); + // Sorted here as the order of dependency_abis is the only + // non-deterministicly ordered set of AbiEntries + Util::sort(abi_tag_entries); // If there is an unusually large number of files in the port then // something suspicious is going on. Rather than hash all of them @@ -637,13 +666,16 @@ namespace vcpkg::Build const int max_port_file_count = 100; // the order of recursive_directory_iterator is undefined so save the names to sort - std::vector port_files; + std::vector> hashes_files; for (auto& port_file : fs::stdfs::recursive_directory_iterator(config.port_dir)) { if (fs::is_regular_file(status(port_file))) { - port_files.push_back(port_file); - if (port_files.size() > max_port_file_count) + hashes_files.emplace_back( + vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"), + port_file.path().filename()); + + if (hashes_files.size() > max_port_file_count) { abi_tag_entries.emplace_back(AbiEntry{"no_hash_max_portfile", ""}); break; @@ -651,25 +683,45 @@ namespace vcpkg::Build } } - if (port_files.size() <= max_port_file_count) + if (hashes_files.size() <= max_port_file_count) { - std::sort(port_files.begin(), port_files.end()); + Util::sort(hashes_files); - int counter = 0; - for (auto& port_file : port_files) + for (auto& hash_file : hashes_files) { - // When vcpkg takes a dependency on C++17 it can use fs::relative, - // which will give a stable ordering and better names in the key entry. - // this is not available in the filesystem TS so instead number the files for the key. - std::string key = Strings::format("file_%03d", counter++); - if (Debug::g_debugging) - { - System::print2("[DEBUG] mapping ", key, " from ", port_file.u8string(), "\n"); - } - abi_tag_entries.emplace_back(AbiEntry{key, vcpkg::Hash::get_file_hash(fs, port_file, "SHA1")}); + // We've already sorted by hash so it's safe to write down the + // filename, which will be consistent across machines. + abi_tag_entries.emplace_back( + AbiEntry{ + std::move(hash_file.second), + std::move(hash_file.first) + }); } } + //Make a copy of the external files and their hashes, and sort by hash + auto additional_file_hashes = pre_build_info.external_files; + std::sort( + additional_file_hashes.begin(), + additional_file_hashes.end(), + [](const std::pair& l, + const std::pair& r) + { + return l.second < r.second || + (l.second == r.second && l.first < r.first); + }); + + for (auto& hash_file : additional_file_hashes) + { + abi_tag_entries.emplace_back( + AbiEntry{ + std::move(hash_file.first), + std::move(hash_file.second) + }); + } + + abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); + abi_tag_entries.emplace_back(AbiEntry{ "vcpkg_fixup_cmake_targets", vcpkg::Hash::get_file_hash(fs, paths.scripts / "cmake" / "vcpkg_fixup_cmake_targets.cmake", "SHA1")}); @@ -682,8 +734,6 @@ namespace vcpkg::Build if (config.build_package_options.use_head_version == UseHeadVersion::YES) abi_tag_entries.emplace_back(AbiEntry{"head", ""}); - Util::sort(abi_tag_entries); - const std::string full_abi_info = Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; }); @@ -1114,12 +1164,15 @@ namespace vcpkg::Build case VcpkgTripletVar::ENV_PASSTHROUGH : pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; - case VcpkgTripletVar::ABI_ADDITIONAL_FILES : - pre_build_info.additional_files = Util::fmap(Strings::split(variable_value, ";"), + case VcpkgTripletVar::EXTERNAL_FILES : + pre_build_info.external_files = + get_external_file_hashes( + paths, + Util::fmap(Strings::split(variable_value, ";"), [](const std::string& path) { return fs::path{path}; - }); + })); break; } } diff --git a/toolsrc/src/vcpkg/statusparagraphs.cpp b/toolsrc/src/vcpkg/statusparagraphs.cpp index f204ed568a8..4f01b991818 100644 --- a/toolsrc/src/vcpkg/statusparagraphs.cpp +++ b/toolsrc/src/vcpkg/statusparagraphs.cpp @@ -118,30 +118,30 @@ namespace vcpkg return it != end() && (*it)->is_installed(); } - bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec) - { - auto it = find(spec); - if (it != end()) - { - for (const std::string& dep : (*it)->package.depends) - { - PackageSpec dep_spec = - PackageSpec::from_name_and_triplet( - dep, - spec.triplet()).value_or_exit(VCPKG_LINE_INFO); + //bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec) + //{ + // auto it = find(spec); + // if (it != end()) + // { + // for (const std::string& dep : (*it)->package.depends) + // { + // PackageSpec dep_spec = + // PackageSpec::from_name_and_triplet( + // dep, + // spec.triplet()).value_or_exit(VCPKG_LINE_INFO); - if (needs_rebuild(dep_spec)) - { - (*it)->state = InstallState::NEEDS_REBUILD; - return true; - } - } + // if (needs_rebuild(dep_spec)) + // { + // (*it)->state = InstallState::NEEDS_REBUILD; + // return true; + // } + // } - return (*it)->needs_rebuild(); - } + // return (*it)->needs_rebuild(); + // } - return false; - } + // return false; + //} StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr pgh) { From 2f2a45595fa925edeace250b694d70095c42b5fa Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 15:26:13 -0700 Subject: [PATCH 15/34] Prompt rebuild if external hash changes --- toolsrc/include/vcpkg/binaryparagraph.h | 11 ++++++++++ toolsrc/src/vcpkg/binaryparagraph.cpp | 27 +++++++++++++++++++++++++ toolsrc/src/vcpkg/build.cpp | 12 +++++++---- toolsrc/src/vcpkg/dependencies.cpp | 8 ++++++-- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index ec14f8a97d3..a95a680903a 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -7,6 +7,13 @@ namespace vcpkg { + enum class ConsistencyState : unsigned + { + UNKNOWN = 0, + CONSISTENT, + INCONSISTENT, + }; + /// /// Built package metadata /// @@ -23,6 +30,8 @@ namespace vcpkg std::string dir() const; + bool is_consistent() const; + PackageSpec spec; std::string version; std::string description; @@ -33,6 +42,8 @@ namespace vcpkg std::string abi; SourceParagraph::TYPE type; std::unordered_map external_files; + + mutable ConsistencyState consistency = ConsistencyState::UNKNOWN; }; struct BinaryControlFile diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index 2e9415dab72..88d257e7e23 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -1,6 +1,7 @@ #include "pch.h" #include +#include #include #include #include @@ -27,6 +28,32 @@ namespace vcpkg static const std::string EXTERNALFILES = "External-Files"; } + bool BinaryParagraph::is_consistent() const + { + switch (consistency) + { + case ConsistencyState::UNKNOWN : + for (const auto& file_hash : external_files) + { + const auto& realfs = Files::get_real_filesystem(); + + if (realfs.is_regular_file(file_hash.first) && + Hash::get_file_hash(realfs, file_hash.first, "SHA1") != file_hash.second) + { + consistency = ConsistencyState::INCONSISTENT; + return false; + } + } + + consistency = ConsistencyState::CONSISTENT; + return true; + case ConsistencyState::CONSISTENT : return true; + case ConsistencyState::INCONSISTENT : return false; + } + + Checks::unreachable(VCPKG_LINE_INFO); + } + BinaryParagraph::BinaryParagraph() = default; BinaryParagraph::BinaryParagraph(std::unordered_map fields) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 110a571b078..1f0dda5900f 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -475,11 +475,15 @@ namespace vcpkg::Build { hashes.emplace_back(external_file, it_hash->second); } - else if (fs.is_regular_file(external_file)) + else if (Files::get_real_filesystem().is_regular_file(external_file)) { - auto emp = s_hash_cache.emplace(external_file.u8string(), - Hash::get_file_hash(fs, external_file, "SHA1")); - hashes.emplace_back(external_file, emp.first->second); + auto emp = s_hash_cache.emplace( + external_file.u8string(), + Hash::get_file_hash( + Files::get_real_filesystem(), + external_file, "SHA1")); + + hashes.emplace_back(external_file.u8string(), emp.first->second); } else { diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index b604c9acfe3..50c310dbbc1 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -664,13 +664,17 @@ namespace vcpkg::Dependencies if (auto p_installed = cluster.installed.get()) { - if (p_installed->original_features.find(feature) != p_installed->original_features.end()) + if (p_installed->original_features.find(feature) != p_installed->original_features.end() && + p_installed->ipv.core->package.is_consistent()) { return MarkPlusResult::SUCCESS; } } - // This feature was or will be uninstalled, therefore we need to rebuild + //The feature was not previously installed or the external files of the + //port are no longer consistent with the last installation of this port + //(they've either been modified or removed). Mark the cluster + //(aka the entire port) to be removed before re-adding it. mark_minus(cluster, graph, graph_plan, prevent_default_features); return follow_plus_dependencies(feature, cluster, graph, graph_plan, prevent_default_features); From e1813766047b17cf8d80dc4babed4d1195df3b56 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 15:31:53 -0700 Subject: [PATCH 16/34] fix condition --- toolsrc/src/vcpkg/binaryparagraph.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index 88d257e7e23..7448d456b4b 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -37,7 +37,7 @@ namespace vcpkg { const auto& realfs = Files::get_real_filesystem(); - if (realfs.is_regular_file(file_hash.first) && + if (!realfs.is_regular_file(file_hash.first) || Hash::get_file_hash(realfs, file_hash.first, "SHA1") != file_hash.second) { consistency = ConsistencyState::INCONSISTENT; From f64d2e9103c8a181ad89cb5d485863a5772105a1 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 15:33:36 -0700 Subject: [PATCH 17/34] remove accidental comment --- toolsrc/src/vcpkg/statusparagraphs.cpp | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/toolsrc/src/vcpkg/statusparagraphs.cpp b/toolsrc/src/vcpkg/statusparagraphs.cpp index 4f01b991818..c642af59b6a 100644 --- a/toolsrc/src/vcpkg/statusparagraphs.cpp +++ b/toolsrc/src/vcpkg/statusparagraphs.cpp @@ -118,31 +118,6 @@ namespace vcpkg return it != end() && (*it)->is_installed(); } - //bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec) - //{ - // auto it = find(spec); - // if (it != end()) - // { - // for (const std::string& dep : (*it)->package.depends) - // { - // PackageSpec dep_spec = - // PackageSpec::from_name_and_triplet( - // dep, - // spec.triplet()).value_or_exit(VCPKG_LINE_INFO); - - // if (needs_rebuild(dep_spec)) - // { - // (*it)->state = InstallState::NEEDS_REBUILD; - // return true; - // } - // } - - // return (*it)->needs_rebuild(); - // } - - // return false; - //} - StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr pgh) { Checks::check_exit(VCPKG_LINE_INFO, pgh != nullptr, "Inserted null paragraph"); From 829f99b50624778ed1534e50fb04912d782fa4df Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 15:36:13 -0700 Subject: [PATCH 18/34] remove needs_rebuild from statusparagraphs --- toolsrc/include/vcpkg/statusparagraphs.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/toolsrc/include/vcpkg/statusparagraphs.h b/toolsrc/include/vcpkg/statusparagraphs.h index 3ebad67de64..fa064de7edd 100644 --- a/toolsrc/include/vcpkg/statusparagraphs.h +++ b/toolsrc/include/vcpkg/statusparagraphs.h @@ -62,8 +62,6 @@ namespace vcpkg /// `true` if installed, `false` if not or not found. bool is_installed(const FeatureSpec& spec) const; - bool needs_rebuild(const PackageSpec& spec); - iterator insert(std::unique_ptr); friend void serialize(const StatusParagraphs& pgh, std::string& out_str); From 81909e47d1f1563926e78d2e0f0764a3dc1bef03 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 15:38:09 -0700 Subject: [PATCH 19/34] Remove types from this PR --- toolsrc/include/vcpkg/binaryparagraph.h | 1 - toolsrc/include/vcpkg/sourceparagraph.h | 10 --------- toolsrc/src/vcpkg/binaryparagraph.cpp | 4 ++-- toolsrc/src/vcpkg/sourceparagraph.cpp | 30 ------------------------- 4 files changed, 2 insertions(+), 43 deletions(-) diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index a95a680903a..8099367c2e1 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -40,7 +40,6 @@ namespace vcpkg std::vector default_features; std::vector depends; std::string abi; - SourceParagraph::TYPE type; std::unordered_map external_files; mutable ConsistencyState consistency = ConsistencyState::UNKNOWN; diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index 9a1c2c843c8..9fbd83475d8 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -46,12 +46,6 @@ namespace vcpkg /// struct SourceParagraph { - enum TYPE : unsigned - { - PORT = 0, - SYS_TOOL, - }; - std::string name; std::string version; std::string description; @@ -60,10 +54,6 @@ namespace vcpkg std::vector supports; std::vector depends; std::vector default_features; - TYPE type; - - static TYPE type_from_string(const std::string& in); - static std::string string_from_type(const TYPE& in); }; /// diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index 7448d456b4b..9b1a8e201ff 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -119,14 +119,14 @@ namespace vcpkg } BinaryParagraph::BinaryParagraph(const SourceParagraph& spgh, const Triplet& triplet, const std::string& abi_tag) - : version(spgh.version), description(spgh.description), maintainer(spgh.maintainer), abi(abi_tag), type(spgh.type) + : version(spgh.version), description(spgh.description), maintainer(spgh.maintainer), abi(abi_tag) { this->spec = PackageSpec::from_name_and_triplet(spgh.name, triplet).value_or_exit(VCPKG_LINE_INFO); this->depends = filter_dependencies(spgh.depends, triplet); } BinaryParagraph::BinaryParagraph(const SourceParagraph& spgh, const FeatureParagraph& fpgh, const Triplet& triplet) - : version(), description(fpgh.description), maintainer(), feature(fpgh.name), type(spgh.type) + : version(), description(fpgh.description), maintainer(), feature(fpgh.name) { this->spec = PackageSpec::from_name_and_triplet(spgh.name, triplet).value_or_exit(VCPKG_LINE_INFO); this->depends = filter_dependencies(fpgh.depends, triplet); diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 6af93a99b8c..5542ef92368 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -100,35 +100,6 @@ namespace vcpkg } } - SourceParagraph::TYPE SourceParagraph::type_from_string(const std::string& in) - { - if (Strings::equals(in, "port") || Strings::equals(in, "")) - { - return SourceParagraph::PORT; - } - - if (Strings::equals(in, "sys-tool")) - { - return SourceParagraph::SYS_TOOL; - } - - System::print2( - in, " is not a valid control file type. Valid types are:", - "\n port\n sys-tool"); - - Checks::exit_fail(VCPKG_LINE_INFO); - } - - std::string SourceParagraph::string_from_type(const SourceParagraph::TYPE& in) - { - switch (in) - { - case SourceParagraph::PORT : return "port"; - case SourceParagraph::SYS_TOOL : return "sys-tool"; - default : Checks::exit_with_message(VCPKG_LINE_INFO, "Invalid CONTROL_TYPE value."); - } - } - static ParseExpected parse_source_paragraph(RawParagraph&& fields) { ParagraphParser parser(std::move(fields)); @@ -145,7 +116,6 @@ namespace vcpkg parse_comma_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS))); spgh->supports = parse_comma_list(parser.optional_field(SourceParagraphFields::SUPPORTS)); spgh->default_features = parse_comma_list(parser.optional_field(SourceParagraphFields::DEFAULTFEATURES)); - spgh->type = SourceParagraph::type_from_string(parser.optional_field(SourceParagraphFields::TYPE)); auto err = parser.error_info(spgh->name); if (err) From 45cd8cda5a8339ec91693c720dd7df5b496b8856 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 15:49:48 -0700 Subject: [PATCH 20/34] move the pre_build_info --- toolsrc/src/vcpkg/build.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index f810d537a6b..70c664b753d 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -603,7 +603,7 @@ namespace vcpkg::Build *config.scf.core_paragraph, triplet, build_info, - pre_build_info, + std::move(pre_build_info), abi_tag); if (error_count != 0) From d68b9a08b1edfe80b77fd65afd21faffc81b162b Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 16:28:00 -0700 Subject: [PATCH 21/34] only use filename --- toolsrc/include/vcpkg/build.h | 2 +- toolsrc/src/vcpkg/build.cpp | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 0445675f026..2d5e18a43a7 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -138,7 +138,7 @@ namespace vcpkg::Build Optional external_toolchain_file; Optional build_type; std::vector passthrough_env_vars; - std::vector> external_files; + std::vector> external_files; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 70c664b753d..9d5b490e35d 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -281,7 +281,7 @@ namespace vcpkg::Build for (auto& file_hash : pre_build_info.external_files) { bpgh.external_files.emplace( - std::move(file_hash.first), + file_hash.first.u8string(), std::move(file_hash.second)); } @@ -459,7 +459,7 @@ namespace vcpkg::Build return command; } - static std::vector> get_external_file_hashes( + static std::vector> get_external_file_hashes( const VcpkgPaths& paths, const std::vector& files) { @@ -467,7 +467,7 @@ namespace vcpkg::Build const auto& fs = paths.get_filesystem(); - std::vector> hashes; + std::vector> hashes; for (const fs::path& external_file : files) { auto it_hash = s_hash_cache.find(external_file); @@ -705,7 +705,16 @@ namespace vcpkg::Build } //Make a copy of the external files and their hashes, and sort by hash - auto additional_file_hashes = pre_build_info.external_files; + std::vector> additional_file_hashes + = Util::fmap(pre_build_info.external_files, + [](const std::pair& file_hash) + { + return std::pair{ + file_hash.first.filename().u8string(), + file_hash.second + }; + }); + std::sort( additional_file_hashes.begin(), additional_file_hashes.end(), From 0c7669d009548616aeb754276deea974ba7a53c3 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 24 Jul 2019 14:24:49 -0700 Subject: [PATCH 22/34] store fs::path instead of std::string --- toolsrc/include/vcpkg/binaryparagraph.h | 4 ++-- toolsrc/src/vcpkg/binaryparagraph.cpp | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index 8099367c2e1..d75e4abb1dc 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -3,7 +3,7 @@ #include #include -#include +#include namespace vcpkg { @@ -40,7 +40,7 @@ namespace vcpkg std::vector default_features; std::vector depends; std::string abi; - std::unordered_map external_files; + std::map external_files; mutable ConsistencyState consistency = ConsistencyState::UNKNOWN; }; diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index 9b1a8e201ff..f571e0e8e66 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -100,7 +100,9 @@ namespace vcpkg this->spec); } - for (int i = 0; i < external_files_or_hashes.size(); i += 2) + for (decltype(external_files_or_hashes)::size_type i = 0; + i < external_files_or_hashes.size(); + i += 2) { external_files.emplace( std::move(external_files_or_hashes[i]), From be59ecce61a14ba9ca0f65f6eeb7db83782bac22 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 24 Jul 2019 14:43:44 -0700 Subject: [PATCH 23/34] Use correct types --- toolsrc/src/vcpkg/build.cpp | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index a5383b2a8d7..ff302507741 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -711,27 +711,19 @@ namespace vcpkg::Build [](const std::pair& file_hash) { return std::pair{ - file_hash.first.filename().u8string(), - file_hash.second + file_hash.second, + file_hash.first.filename().u8string() }; }); - std::sort( - additional_file_hashes.begin(), - additional_file_hashes.end(), - [](const std::pair& l, - const std::pair& r) - { - return l.second < r.second || - (l.second == r.second && l.first < r.first); - }); + std::sort(additional_file_hashes.begin(), additional_file_hashes.end()); for (auto& hash_file : additional_file_hashes) { abi_tag_entries.emplace_back( AbiEntry{ - std::move(hash_file.first), - std::move(hash_file.second) + std::move(hash_file.second), + std::move(hash_file.first) }); } From 67a749b9892b1c68609dd41dd014dd8de2cdbb9b Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 24 Jul 2019 14:46:55 -0700 Subject: [PATCH 24/34] turn path into string --- toolsrc/src/vcpkg/build.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index ff302507741..80ac3ffe693 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -679,7 +679,7 @@ namespace vcpkg::Build { hashes_files.emplace_back( vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"), - port_file.path().filename()); + port_file.path().filename().u8string()); if (hashes_files.size() > max_port_file_count) { From f81aa47176dd805ccceea0fceb8c4e2158e5c554 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 24 Jul 2019 14:53:50 -0700 Subject: [PATCH 25/34] convert path to string --- toolsrc/src/vcpkg/binaryparagraph.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index f571e0e8e66..9be2310c69a 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -175,9 +175,9 @@ namespace vcpkg out_str.append(Strings::join(",", Util::fmap( pgh.external_files, - [](const std::pair& kv) + [](const std::pair& kv) { - return kv.first + "," + kv.second; + return kv.first.u8string() + "," + kv.second; }))).push_back('\n'); } } From 9da7c5c99ee42e382895dbd0dafdd29beaa61075 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Thu, 25 Jul 2019 09:38:05 -0700 Subject: [PATCH 26/34] Make comment better --- toolsrc/src/vcpkg/build.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 80ac3ffe693..69a313c1a2f 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -705,7 +705,8 @@ namespace vcpkg::Build } } - //Make a copy of the external files and their hashes, and sort by hash + //Make a copy of the external file names and their hashes, and sort by + //hash. std::vector> additional_file_hashes = Util::fmap(pre_build_info.external_files, [](const std::pair& file_hash) From 0c7d8f414669c6e025794c374f2837e5fa24d02b Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Fri, 9 Aug 2019 14:21:58 -0700 Subject: [PATCH 27/34] Change purpose of this PR to just overriding the abi --- toolsrc/include/vcpkg/binaryparagraph.h | 12 --- toolsrc/include/vcpkg/build.h | 7 +- toolsrc/src/vcpkg/binaryparagraph.cpp | 59 ----------- toolsrc/src/vcpkg/build.cpp | 128 ++++++++---------------- toolsrc/src/vcpkg/dependencies.cpp | 7 +- 5 files changed, 46 insertions(+), 167 deletions(-) diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index a68fcfb368c..0fbd382fa21 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -8,13 +8,6 @@ namespace vcpkg { - enum class ConsistencyState : unsigned - { - UNKNOWN = 0, - CONSISTENT, - INCONSISTENT, - }; - /// /// Built package metadata /// @@ -31,8 +24,6 @@ namespace vcpkg std::string dir() const; - bool is_consistent() const; - PackageSpec spec; std::string version; std::string description; @@ -41,9 +32,6 @@ namespace vcpkg std::vector default_features; std::vector depends; std::string abi; - std::map external_files; - - mutable ConsistencyState consistency = ConsistencyState::UNKNOWN; }; struct BinaryControlFile diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index c393d989b3e..cd1821b32b7 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -137,8 +137,9 @@ namespace vcpkg::Build Optional visual_studio_path; Optional external_toolchain_file; Optional build_type; + Optional public_abi_override; + Optional port; std::vector passthrough_env_vars; - std::vector> external_files; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); @@ -153,7 +154,7 @@ namespace vcpkg::Build CHAINLOAD_TOOLCHAIN_FILE, BUILD_TYPE, ENV_PASSTHROUGH, - EXTERNAL_FILES, + PUBLIC_ABI_OVERRIDE, }; const std::unordered_map VCPKG_OPTIONS = { @@ -165,7 +166,7 @@ namespace vcpkg::Build {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, - {"VCPKG_EXTERNAL_FILES", VcpkgTripletVar::EXTERNAL_FILES}, + {"VCPKG_PUBLIC_ABI_OVERRIDE", VcpkgTripletVar::PUBLIC_ABI_OVERRIDE}, }; struct ExtendedBuildResult diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index ef194f9f34b..231380b1980 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -25,33 +25,6 @@ namespace vcpkg static const std::string MAINTAINER = "Maintainer"; static const std::string DEPENDS = "Depends"; static const std::string DEFAULTFEATURES = "Default-Features"; - static const std::string EXTERNALFILES = "External-Files"; - } - - bool BinaryParagraph::is_consistent() const - { - switch (consistency) - { - case ConsistencyState::UNKNOWN : - for (const auto& file_hash : external_files) - { - const auto& realfs = Files::get_real_filesystem(); - - if (!realfs.is_regular_file(file_hash.first) || - Hash::get_file_hash(realfs, file_hash.first, "SHA1") != file_hash.second) - { - consistency = ConsistencyState::INCONSISTENT; - return false; - } - } - - consistency = ConsistencyState::CONSISTENT; - return true; - case ConsistencyState::CONSISTENT : return true; - case ConsistencyState::INCONSISTENT : return false; - } - - Checks::unreachable(VCPKG_LINE_INFO); } BinaryParagraph::BinaryParagraph() = default; @@ -89,26 +62,6 @@ namespace vcpkg this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES)); } - std::vector external_files_or_hashes = - parse_comma_list(parser.optional_field(Fields::EXTERNALFILES)); - - if (external_files_or_hashes.size() % 2 != 0) - { - Checks::exit_with_message( - VCPKG_LINE_INFO, - "The External-Files field is not composed of key-value pairs for ", - this->spec); - } - - for (decltype(external_files_or_hashes)::size_type i = 0; - i < external_files_or_hashes.size(); - i += 2) - { - external_files.emplace( - std::move(external_files_or_hashes[i]), - std::move(external_files_or_hashes[i+1])); - } - if (const auto err = parser.error_info(this->spec.to_string())) { System::print2(System::Color::error, "Error: while parsing the Binary Paragraph for ", this->spec, '\n'); @@ -168,17 +121,5 @@ namespace vcpkg if (!pgh.maintainer.empty()) out_str.append("Maintainer: ").append(pgh.maintainer).push_back('\n'); if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n'); if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n'); - - if (!pgh.external_files.empty()) - { - out_str.append("External-Files: "); - out_str.append(Strings::join(",", - Util::fmap( - pgh.external_files, - [](const std::pair& kv) - { - return kv.first.u8string() + "," + kv.second; - }))).push_back('\n'); - } } } diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 6f14f46f8c8..147bbe796c7 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -277,13 +277,6 @@ namespace vcpkg::Build bpgh.version = *p_ver; } - for (auto& file_hash : pre_build_info.external_files) - { - bpgh.external_files.emplace( - file_hash.first.u8string(), - std::move(file_hash.second)); - } - bcf->core_paragraph = std::move(bpgh); return bcf; } @@ -451,45 +444,6 @@ namespace vcpkg::Build return command; } - static std::vector> get_external_file_hashes( - const VcpkgPaths& paths, - const std::vector& files) - { - static std::map s_hash_cache; - - const auto& fs = paths.get_filesystem(); - - std::vector> hashes; - for (const fs::path& external_file : files) - { - auto it_hash = s_hash_cache.find(external_file); - - if (it_hash != s_hash_cache.end()) - { - hashes.emplace_back(external_file, it_hash->second); - } - else if (Files::get_real_filesystem().is_regular_file(external_file)) - { - auto emp = s_hash_cache.emplace( - external_file.u8string(), - Hash::get_file_hash( - Files::get_real_filesystem(), - external_file, "SHA1")); - - hashes.emplace_back(external_file.u8string(), emp.first->second); - } - else - { - Checks::exit_with_message( - VCPKG_LINE_INFO, - external_file.u8string() + - " was listed as an additional file for calculating the abi, but was not found."); - } - } - - return hashes; - } - static std::string get_triplet_abi(const VcpkgPaths& paths, const PreBuildInfo& pre_build_info, const Triplet& triplet) @@ -645,6 +599,15 @@ namespace vcpkg::Build const PreBuildInfo& pre_build_info, Span dependency_abis) { + if (pre_build_info.public_abi_override) + { + return AbiTagAndFile + { + "override", + pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO) + }; + } + auto& fs = paths.get_filesystem(); const Triplet& triplet = config.triplet; const std::string& name = config.scf.core_paragraph->name; @@ -694,29 +657,6 @@ namespace vcpkg::Build } } - //Make a copy of the external file names and their hashes, and sort by - //hash. - std::vector> additional_file_hashes - = Util::fmap(pre_build_info.external_files, - [](const std::pair& file_hash) - { - return std::pair{ - file_hash.second, - file_hash.first.filename().u8string() - }; - }); - - std::sort(additional_file_hashes.begin(), additional_file_hashes.end()); - - for (auto& hash_file : additional_file_hashes) - { - abi_tag_entries.emplace_back( - AbiEntry{ - std::move(hash_file.second), - std::move(hash_file.first) - }); - } - abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); abi_tag_entries.emplace_back(AbiEntry{ @@ -835,11 +775,16 @@ namespace vcpkg::Build if (!required_fspecs.empty()) { - return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(required_fspecs)}; + return { + BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, + std::move(required_fspecs) + }; } const PackageSpec spec = - PackageSpec::from_name_and_triplet(config.scf.core_paragraph->name, triplet).value_or_exit(VCPKG_LINE_INFO); + PackageSpec::from_name_and_triplet( + config.scf.core_paragraph->name, + triplet).value_or_exit(VCPKG_LINE_INFO); std::vector dependency_abis; @@ -856,15 +801,27 @@ namespace vcpkg::Build const auto pre_build_info = PreBuildInfo::from_triplet_file(paths, triplet, config.scfl); auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); + if (!maybe_abi_tag_and_file) + { + return do_build_package_and_clean_buildtrees( + paths, + pre_build_info, + spec, + AbiTagAndFile{}.tag, + config); + } + std::error_code ec; const auto abi_tag_and_file = maybe_abi_tag_and_file.get(); const fs::path archives_root_dir = paths.root / "archives"; const std::string archive_name = abi_tag_and_file->tag + ".zip"; const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; const fs::path archive_path = archives_root_dir / archive_subpath; const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); + const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - if (config.build_package_options.binary_caching == BinaryCaching::YES && abi_tag_and_file) + if (config.build_package_options.binary_caching == BinaryCaching::YES) { if (fs.exists(archive_path)) { @@ -900,20 +857,19 @@ namespace vcpkg::Build System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); } + fs.create_directories(abi_package_dir, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Coud not create directory %s", abi_package_dir.u8string()); + fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); + ExtendedBuildResult result = do_build_package_and_clean_buildtrees( paths, pre_build_info, spec, - maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, + maybe_abi_tag_and_file.value_or_exit(VCPKG_LINE_INFO).tag, config); - std::error_code ec; - fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); - auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); - Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); - if (config.build_package_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED) { @@ -1098,6 +1054,8 @@ namespace vcpkg::Build PreBuildInfo pre_build_info; + pre_build_info.port = port; + const auto e = lines.cend(); auto cur = std::find(lines.cbegin(), e, FLAG_GUID); if (cur != e) ++cur; @@ -1154,15 +1112,9 @@ namespace vcpkg::Build case VcpkgTripletVar::ENV_PASSTHROUGH : pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; - case VcpkgTripletVar::EXTERNAL_FILES : - pre_build_info.external_files = - get_external_file_hashes( - paths, - Util::fmap(Strings::split(variable_value, ";"), - [](const std::string& path) - { - return fs::path{path}; - })); + case VcpkgTripletVar::PUBLIC_ABI_OVERRIDE : + pre_build_info.public_abi_override = + variable_value.empty() ? nullopt : Optional{variable_value}; break; } } diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index c175cc86f0a..8c8b2f8101a 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -663,16 +663,13 @@ namespace vcpkg::Dependencies if (auto p_installed = cluster.installed.get()) { - if (p_installed->original_features.find(feature) != p_installed->original_features.end() && - p_installed->ipv.core->package.is_consistent()) + if (p_installed->original_features.find(feature) != p_installed->original_features.end()) { return MarkPlusResult::SUCCESS; } } - //The feature was not previously installed or the external files of the - //port are no longer consistent with the last installation of this port - //(they've either been modified or removed). Mark the cluster + //The feature was not previously installed. Mark the cluster //(aka the entire port) to be removed before re-adding it. mark_minus(cluster, graph, graph_plan, prevent_default_features); From ad82c38cc1e8cc87f026cbd179e8bfaa62ce3553 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Fri, 9 Aug 2019 14:40:09 -0700 Subject: [PATCH 28/34] Actually only override the PUBLIC ABI, not the private one --- scripts/get_triplet_environment.cmake | 2 +- toolsrc/src/vcpkg/build.cpp | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index 8ae365812e6..a838624ca85 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -13,4 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}") message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}") message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}") message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}") -message("VCPKG_EXTERNAL_FILES=${VCPKG_EXTERNAL_FILES}") +message("VCPKG_EXTERNAL_FILES=${VCPKG_PUBLIC_ABI_OVERRIDE}") diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index de8fbc75281..7e14a09263f 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -611,15 +611,6 @@ namespace vcpkg::Build const PreBuildInfo& pre_build_info, Span dependency_abis) { - if (pre_build_info.public_abi_override) - { - return AbiTagAndFile - { - "override", - pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO) - }; - } - auto& fs = paths.get_filesystem(); const Triplet& triplet = config.triplet; const std::string& name = config.scf.core_paragraph->name; @@ -823,7 +814,7 @@ namespace vcpkg::Build paths, pre_build_info, spec, - AbiTagAndFile{}.tag, + pre_build_info.public_abi_override.value_or(AbiTagAndFile{}.tag), config); } @@ -883,7 +874,7 @@ namespace vcpkg::Build paths, pre_build_info, spec, - maybe_abi_tag_and_file.value_or_exit(VCPKG_LINE_INFO).tag, + pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), config); if (config.build_package_options.binary_caching == BinaryCaching::YES && @@ -1129,8 +1120,7 @@ namespace vcpkg::Build pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; case VcpkgTripletVar::PUBLIC_ABI_OVERRIDE : - pre_build_info.public_abi_override = - variable_value.empty() ? nullopt : Optional{variable_value}; + pre_build_info.public_abi_override = variable_value; break; } } From f9561a08db1add3052d4759adf051fc903e2e3e4 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Fri, 9 Aug 2019 14:47:58 -0700 Subject: [PATCH 29/34] remove unnecessary include, fix variable extraction --- scripts/get_triplet_environment.cmake | 2 +- toolsrc/include/vcpkg/binaryparagraph.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index a838624ca85..0457cee9bde 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -13,4 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}") message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}") message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}") message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}") -message("VCPKG_EXTERNAL_FILES=${VCPKG_PUBLIC_ABI_OVERRIDE}") +message("VCPKG_PUBLIC_ABI_OVERRIDE=${VCPKG_PUBLIC_ABI_OVERRIDE}") diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index 0fbd382fa21..45720538412 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -4,8 +4,6 @@ #include #include -#include - namespace vcpkg { /// From f0dd7b62964df89eb3cd50678f9a8ef98392be1d Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Fri, 9 Aug 2019 14:51:16 -0700 Subject: [PATCH 30/34] fix missing abi key mistake --- toolsrc/src/vcpkg/build.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 7e14a09263f..385013b9b28 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -1120,7 +1120,8 @@ namespace vcpkg::Build pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; case VcpkgTripletVar::PUBLIC_ABI_OVERRIDE : - pre_build_info.public_abi_override = variable_value; + pre_build_info.public_abi_override = + variable_value.empty() ? nullopt : Optional{variable_value}; break; } } From 5bd45366fb0558616a6cb46cdbca6810da5afa4c Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Fri, 9 Aug 2019 15:15:22 -0700 Subject: [PATCH 31/34] Add public abi override into the private abi --- toolsrc/src/vcpkg/build.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 385013b9b28..1e5394593c2 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -675,6 +675,15 @@ namespace vcpkg::Build const std::string features = Strings::join(";", config.feature_list); abi_tag_entries.emplace_back(AbiEntry{"features", features}); + if (pre_build_info.public_abi_override) + { + abi_tag_entries.emplace_back( + AbiEntry{ + "public_abi_override", + pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO) + }); + } + if (config.build_package_options.use_head_version == UseHeadVersion::YES) abi_tag_entries.emplace_back(AbiEntry{"head", ""}); From 5c11033ad7a52960500c8b604a4b7e7b4fb2a1b3 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Mon, 12 Aug 2019 10:22:05 -0700 Subject: [PATCH 32/34] Fix formatting, hash override, don't move pre_build_info --- toolsrc/src/vcpkg/binaryparagraph.cpp | 1 - toolsrc/src/vcpkg/build.cpp | 85 +++++++++------------------ 2 files changed, 28 insertions(+), 58 deletions(-) diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index 231380b1980..b4cd9cc4b4b 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 1e5394593c2..e0dc603ea38 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -263,12 +263,11 @@ namespace vcpkg::Build return BinaryParagraph(source_paragraph, feature_paragraph, triplet); } - static std::unique_ptr create_binary_control_file( - const SourceParagraph& source_paragraph, - const Triplet& triplet, - const BuildInfo& build_info, - const PreBuildInfo& pre_build_info, - const std::string& abi_tag) + static std::unique_ptr create_binary_control_file(const SourceParagraph& source_paragraph, + const Triplet& triplet, + const BuildInfo& build_info, + const PreBuildInfo& pre_build_info, + const std::string& abi_tag) { auto bcf = std::make_unique(); BinaryParagraph bpgh(source_paragraph, triplet, abi_tag); @@ -554,12 +553,7 @@ namespace vcpkg::Build PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir); std::unique_ptr bcf = - create_binary_control_file( - *config.scf.core_paragraph, - triplet, - build_info, - std::move(pre_build_info), - abi_tag); + create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, pre_build_info, abi_tag); if (error_count != 0) { @@ -632,9 +626,8 @@ namespace vcpkg::Build { if (fs::is_regular_file(status(port_file))) { - hashes_files.emplace_back( - vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"), - port_file.path().filename().u8string()); + hashes_files.emplace_back(vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"), + port_file.path().filename().u8string()); if (hashes_files.size() > max_port_file_count) { @@ -652,16 +645,12 @@ namespace vcpkg::Build { // We've already sorted by hash so it's safe to write down the // filename, which will be consistent across machines. - abi_tag_entries.emplace_back( - AbiEntry{ - std::move(hash_file.second), - std::move(hash_file.first) - }); + abi_tag_entries.emplace_back(AbiEntry{std::move(hash_file.second), std::move(hash_file.first)}); } } abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); - + #if defined(_WIN32) abi_tag_entries.emplace_back(AbiEntry{"powershell", paths.get_tool_version("powershell-core")}); #endif @@ -677,11 +666,9 @@ namespace vcpkg::Build if (pre_build_info.public_abi_override) { - abi_tag_entries.emplace_back( - AbiEntry{ - "public_abi_override", - pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO) - }); + abi_tag_entries.emplace_back(AbiEntry{ + "public_abi_override", + Hash::get_string_hash(pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO), "SHA1")}); } if (config.build_package_options.use_head_version == UseHeadVersion::YES) @@ -791,16 +778,11 @@ namespace vcpkg::Build if (!required_fspecs.empty()) { - return { - BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, - std::move(required_fspecs) - }; + return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(required_fspecs)}; } const PackageSpec spec = - PackageSpec::from_name_and_triplet( - config.scf.core_paragraph->name, - triplet).value_or_exit(VCPKG_LINE_INFO); + PackageSpec::from_name_and_triplet(config.scf.core_paragraph->name, triplet).value_or_exit(VCPKG_LINE_INFO); std::vector dependency_abis; @@ -820,11 +802,7 @@ namespace vcpkg::Build if (!maybe_abi_tag_and_file) { return do_build_package_and_clean_buildtrees( - paths, - pre_build_info, - spec, - pre_build_info.public_abi_override.value_or(AbiTagAndFile{}.tag), - config); + paths, pre_build_info, spec, pre_build_info.public_abi_override.value_or(AbiTagAndFile{}.tag), config); } std::error_code ec; @@ -878,16 +856,10 @@ namespace vcpkg::Build fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); - ExtendedBuildResult result = - do_build_package_and_clean_buildtrees( - paths, - pre_build_info, - spec, - pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), - config); + ExtendedBuildResult result = do_build_package_and_clean_buildtrees( + paths, pre_build_info, spec, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), config); - if (config.build_package_options.binary_caching == BinaryCaching::YES && - result.code == BuildResult::SUCCEEDED) + if (config.build_package_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED) { const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); @@ -898,16 +870,15 @@ namespace vcpkg::Build if (ec) { System::printf(System::Color::warning, - "Failed to store binary cache %s: %s\n", - archive_path.u8string(), - ec.message()); + "Failed to store binary cache %s: %s\n", + archive_path.u8string(), + ec.message()); } else System::printf("Stored binary cache: %s\n", archive_path.u8string()); } else if (config.build_package_options.binary_caching == BinaryCaching::YES && - (result.code == BuildResult::BUILD_FAILED || - result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) + (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) { if (!fs.exists(archive_tombstone_path)) { @@ -922,9 +893,9 @@ namespace vcpkg::Build if (log_file.path().extension() == ".log") { fs.copy_file(log_file.path(), - tmp_log_path_destination / log_file.path().filename(), - fs::stdfs::copy_options::none, - ec); + tmp_log_path_destination / log_file.path().filename(), + fs::stdfs::copy_options::none, + ec); } } @@ -1125,10 +1096,10 @@ namespace vcpkg::Build Checks::exit_with_message( VCPKG_LINE_INFO, "Unknown setting for VCPKG_BUILD_TYPE: %s", variable_value); break; - case VcpkgTripletVar::ENV_PASSTHROUGH : + case VcpkgTripletVar::ENV_PASSTHROUGH: pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; - case VcpkgTripletVar::PUBLIC_ABI_OVERRIDE : + case VcpkgTripletVar::PUBLIC_ABI_OVERRIDE: pre_build_info.public_abi_override = variable_value.empty() ? nullopt : Optional{variable_value}; break; From 44e08d28e777faa08a773054b2dac2d5cf949569 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Mon, 12 Aug 2019 11:19:16 -0700 Subject: [PATCH 33/34] remove uneeded header --- toolsrc/src/vcpkg/binaryparagraph.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index b4cd9cc4b4b..8b188609801 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -1,7 +1,6 @@ #include "pch.h" #include -#include #include #include #include From c605f4eea632878dc495fd280a41734bd14e851f Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Mon, 12 Aug 2019 11:40:03 -0700 Subject: [PATCH 34/34] drop uneeded parameter from create_binary_control_file --- toolsrc/src/vcpkg/build.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index e0dc603ea38..199a7db1a09 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -266,7 +266,6 @@ namespace vcpkg::Build static std::unique_ptr create_binary_control_file(const SourceParagraph& source_paragraph, const Triplet& triplet, const BuildInfo& build_info, - const PreBuildInfo& pre_build_info, const std::string& abi_tag) { auto bcf = std::make_unique(); @@ -553,7 +552,7 @@ namespace vcpkg::Build PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir); std::unique_ptr bcf = - create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, pre_build_info, abi_tag); + create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); if (error_count != 0) {