diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index 21464bb2e63..0457cee9bde 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -13,3 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}") message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}") message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}") message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}") +message("VCPKG_PUBLIC_ABI_OVERRIDE=${VCPKG_PUBLIC_ABI_OVERRIDE}") diff --git a/scripts/ports.cmake b/scripts/ports.cmake index 58510567b26..86b2719d561 100644 --- a/scripts/ports.cmake +++ b/scripts/ports.cmake @@ -68,6 +68,7 @@ if(CMD MATCHES "^BUILD$") include(${CMAKE_TRIPLET_FILE}) include(${ENV_OVERRIDES_FILE} OPTIONAL) + set(TRIPLET_SYSTEM_ARCH ${VCPKG_TARGET_ARCHITECTURE}) include(${CMAKE_CURRENT_LIST_DIR}/cmake/vcpkg_common_definitions.cmake) include(${CMAKE_CURRENT_LIST_DIR}/cmake/vcpkg_common_functions.cmake) diff --git a/toolsrc/VERSION.txt b/toolsrc/VERSION.txt index d7d695c69cb..0342c24ab35 100644 --- a/toolsrc/VERSION.txt +++ b/toolsrc/VERSION.txt @@ -1 +1 @@ -"2019.07.18" +"2019.07.19" diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index 7734859f700..45720538412 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -4,8 +4,6 @@ #include #include -#include - namespace vcpkg { /// diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index c57cbfad333..cd1821b32b7 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -137,6 +137,8 @@ namespace vcpkg::Build Optional visual_studio_path; Optional external_toolchain_file; Optional build_type; + Optional public_abi_override; + Optional port; std::vector passthrough_env_vars; }; @@ -152,6 +154,7 @@ namespace vcpkg::Build CHAINLOAD_TOOLCHAIN_FILE, BUILD_TYPE, ENV_PASSTHROUGH, + PUBLIC_ABI_OVERRIDE, }; const std::unordered_map VCPKG_OPTIONS = { @@ -163,6 +166,7 @@ namespace vcpkg::Build {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + {"VCPKG_PUBLIC_ABI_OVERRIDE", VcpkgTripletVar::PUBLIC_ABI_OVERRIDE}, }; struct ExtendedBuildResult diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index b809db0bc0b..d7ba5fd189c 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -273,6 +274,7 @@ namespace vcpkg::Build { bpgh.version = *p_ver; } + bcf->core_paragraph = std::move(bpgh); return bcf; } @@ -549,7 +551,8 @@ namespace vcpkg::Build const size_t error_count = PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir); - auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); + std::unique_ptr bcf = + create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); if (error_count != 0) { @@ -601,18 +604,15 @@ namespace vcpkg::Build const PreBuildInfo& pre_build_info, Span dependency_abis) { - if (config.build_package_options.binary_caching == BinaryCaching::NO) return nullopt; - auto& fs = paths.get_filesystem(); const Triplet& triplet = config.triplet; const std::string& name = config.scf.core_paragraph->name; std::vector abi_tag_entries(dependency_abis.begin(), dependency_abis.end()); -#if defined(_WIN32) - abi_tag_entries.emplace_back(AbiEntry{"powershell", paths.get_tool_version("powershell-core")}); -#endif - abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); + // Sorted here as the order of dependency_abis is the only + // non-deterministicly ordered set of AbiEntries + Util::sort(abi_tag_entries); // If there is an unusually large number of files in the port then // something suspicious is going on. Rather than hash all of them @@ -620,13 +620,15 @@ namespace vcpkg::Build const int max_port_file_count = 100; // the order of recursive_directory_iterator is undefined so save the names to sort - std::vector port_files; + std::vector> hashes_files; for (auto& port_file : fs::stdfs::recursive_directory_iterator(config.port_dir)) { if (fs::is_regular_file(fs.status(VCPKG_LINE_INFO, port_file))) { - port_files.push_back(port_file); - if (port_files.size() > max_port_file_count) + hashes_files.emplace_back(vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"), + port_file.path().filename().u8string()); + + if (hashes_files.size() > max_port_file_count) { abi_tag_entries.emplace_back(AbiEntry{"no_hash_max_portfile", ""}); break; @@ -634,25 +636,24 @@ namespace vcpkg::Build } } - if (port_files.size() <= max_port_file_count) + if (hashes_files.size() <= max_port_file_count) { - std::sort(port_files.begin(), port_files.end()); + Util::sort(hashes_files); - int counter = 0; - for (auto& port_file : port_files) + for (auto& hash_file : hashes_files) { - // When vcpkg takes a dependency on C++17 it can use fs::relative, - // which will give a stable ordering and better names in the key entry. - // this is not available in the filesystem TS so instead number the files for the key. - std::string key = Strings::format("file_%03d", counter++); - if (Debug::g_debugging) - { - System::print2("[DEBUG] mapping ", key, " from ", port_file.u8string(), "\n"); - } - abi_tag_entries.emplace_back(AbiEntry{key, vcpkg::Hash::get_file_hash(fs, port_file, "SHA1")}); + // We've already sorted by hash so it's safe to write down the + // filename, which will be consistent across machines. + abi_tag_entries.emplace_back(AbiEntry{std::move(hash_file.second), std::move(hash_file.first)}); } } + abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); + +#if defined(_WIN32) + abi_tag_entries.emplace_back(AbiEntry{"powershell", paths.get_tool_version("powershell-core")}); +#endif + abi_tag_entries.emplace_back(AbiEntry{ "vcpkg_fixup_cmake_targets", vcpkg::Hash::get_file_hash(fs, paths.scripts / "cmake" / "vcpkg_fixup_cmake_targets.cmake", "SHA1")}); @@ -662,11 +663,16 @@ namespace vcpkg::Build const std::string features = Strings::join(";", config.feature_list); abi_tag_entries.emplace_back(AbiEntry{"features", features}); + if (pre_build_info.public_abi_override) + { + abi_tag_entries.emplace_back(AbiEntry{ + "public_abi_override", + Hash::get_string_hash(pre_build_info.public_abi_override.value_or_exit(VCPKG_LINE_INFO), "SHA1")}); + } + if (config.build_package_options.use_head_version == UseHeadVersion::YES) abi_tag_entries.emplace_back(AbiEntry{"head", ""}); - Util::sort(abi_tag_entries); - const std::string full_abi_info = Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; }); @@ -694,7 +700,7 @@ namespace vcpkg::Build } System::print2( - "Warning: binary caching disabled because abi keys are missing values:\n", + "Warning: abi keys are missing values:\n", Strings::join("", abi_tag_entries_missing, [](const AbiEntry& e) { return " " + e.key + "\n"; }), "\n"); @@ -792,17 +798,24 @@ namespace vcpkg::Build const auto pre_build_info = PreBuildInfo::from_triplet_file(paths, triplet, config.scfl); auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); - - const auto abi_tag_and_file = maybe_abi_tag_and_file.get(); - - if (config.build_package_options.binary_caching == BinaryCaching::YES && abi_tag_and_file) + if (!maybe_abi_tag_and_file) { - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag_and_file->tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + return do_build_package_and_clean_buildtrees( + paths, pre_build_info, spec, pre_build_info.public_abi_override.value_or(AbiTagAndFile{}.tag), config); + } + std::error_code ec; + const auto abi_tag_and_file = maybe_abi_tag_and_file.get(); + const fs::path archives_root_dir = paths.root / "archives"; + const std::string archive_name = abi_tag_and_file->tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name(); + const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; + + if (config.build_package_options.binary_caching == BinaryCaching::YES) + { if (fs.exists(archive_path)) { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); @@ -835,70 +848,67 @@ namespace vcpkg::Build } System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); - - ExtendedBuildResult result = do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config); - - std::error_code ec; - fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); - auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); - Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); - - if (result.code == BuildResult::SUCCEEDED) - { - const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); - - compress_archive(paths, spec, tmp_archive_path); - - fs.create_directories(archive_path.parent_path(), ec); - fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); - if (ec) - { - System::printf(System::Color::warning, - "Failed to store binary cache %s: %s\n", - archive_path.u8string(), - ec.message()); - } - else - System::printf("Stored binary cache: %s\n", archive_path.u8string()); - } - else if (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED) - { - if (!fs.exists(archive_tombstone_path)) - { - // Build failed, store all failure logs in the tombstone. - const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; - const auto tmp_log_path_destination = tmp_log_path / spec.name(); - const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; - fs.create_directories(tmp_log_path_destination, ec); - - for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) - { - if (log_file.path().extension() == ".log") - { - fs.copy_file(log_file.path(), - tmp_log_path_destination / log_file.path().filename(), - fs::stdfs::copy_options::none, - ec); - } - } - - compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); - - fs.create_directories(archive_tombstone_path.parent_path(), ec); - fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); - - // clean up temporary directory - fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); - } - } - - return result; } - return do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config); + fs.create_directories(abi_package_dir, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Coud not create directory %s", abi_package_dir.u8string()); + fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); + + ExtendedBuildResult result = do_build_package_and_clean_buildtrees( + paths, pre_build_info, spec, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), config); + + if (config.build_package_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED) + { + const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); + + compress_archive(paths, spec, tmp_archive_path); + + fs.create_directories(archive_path.parent_path(), ec); + fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); + if (ec) + { + System::printf(System::Color::warning, + "Failed to store binary cache %s: %s\n", + archive_path.u8string(), + ec.message()); + } + else + System::printf("Stored binary cache: %s\n", archive_path.u8string()); + } + else if (config.build_package_options.binary_caching == BinaryCaching::YES && + (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) + { + if (!fs.exists(archive_tombstone_path)) + { + // Build failed, store all failure logs in the tombstone. + const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; + const auto tmp_log_path_destination = tmp_log_path / spec.name(); + const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; + fs.create_directories(tmp_log_path_destination, ec); + + for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) + { + if (log_file.path().extension() == ".log") + { + fs.copy_file(log_file.path(), + tmp_log_path_destination / log_file.path().filename(), + fs::stdfs::copy_options::none, + ec); + } + } + + compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); + + fs.create_directories(archive_tombstone_path.parent_path(), ec); + fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); + + // clean up temporary directory + fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); + } + } + + return result; } const std::string& to_string(const BuildResult build_result) @@ -1030,6 +1040,8 @@ namespace vcpkg::Build PreBuildInfo pre_build_info; + pre_build_info.port = port; + const auto e = lines.cend(); auto cur = std::find(lines.cbegin(), e, FLAG_GUID); if (cur != e) ++cur; @@ -1086,6 +1098,10 @@ namespace vcpkg::Build case VcpkgTripletVar::ENV_PASSTHROUGH: pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; + case VcpkgTripletVar::PUBLIC_ABI_OVERRIDE: + pre_build_info.public_abi_override = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; } } else diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index 9fccf950e20..09f35d2b129 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -670,7 +670,8 @@ namespace vcpkg::Dependencies } } - // This feature was or will be uninstalled, therefore we need to rebuild + //The feature was not previously installed. Mark the cluster + //(aka the entire port) to be removed before re-adding it. mark_minus(cluster, graph, graph_plan, prevent_default_features); return follow_plus_dependencies(feature, cluster, graph, graph_plan, prevent_default_features); diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 298095df60d..ebb9cd4f469 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -26,6 +26,7 @@ namespace vcpkg static const std::string SUPPORTS = "Supports"; static const std::string VERSION = "Version"; static const std::string HOMEPAGE = "Homepage"; + static const std::string TYPE = "Type"; } static Span get_list_of_valid_fields() @@ -37,6 +38,7 @@ namespace vcpkg SourceParagraphFields::MAINTAINER, SourceParagraphFields::BUILD_DEPENDS, SourceParagraphFields::HOMEPAGE, + SourceParagraphFields::TYPE, }; return valid_fields;