mirror/vcpkg
1
0
Fork 0
mirror of https://github.com/microsoft/vcpkg.git synced 2024-11-28 09:19:02 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

add external file hashes to the binary paragraph

Browse Source
This commit is contained in:
Curtis.Bezault 2019-07-23 10:07:39 -07:00
parent d39bd70d53
commit 459908ae14
6 changed files with 161 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/get_triplet_environment.cmake
View File

@ -13,4 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}")
message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}")
message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}")
message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}")
message("VCPKG_ABI_ADDITIONAL_FILES=${VCPKG_ABI_ADDITIONAL_FILES}")
message("VCPKG_EXTERNAL_FILES=${VCPKG_EXTERNAL_FILES}")

1
toolsrc/include/vcpkg/binaryparagraph.h
View File

@ -32,6 +32,7 @@ namespace vcpkg
std::vector<std::string> depends;
std::string abi;
SourceParagraph::TYPE type;
std::unordered_map<std::string, std::string> external_files;
};
struct BinaryControlFile

6
toolsrc/include/vcpkg/build.h
View File

@ -138,7 +138,7 @@ namespace vcpkg::Build
Optional<std::string> external_toolchain_file;
Optional<ConfigurationType> build_type;
std::vector<std::string> passthrough_env_vars;
std::vector<fs::path> additional_files;
std::vector<std::pair<std::string, std::string>> external_files;
};
std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset);
@ -153,7 +153,7 @@ namespace vcpkg::Build
CHAINLOAD_TOOLCHAIN_FILE,
BUILD_TYPE,
ENV_PASSTHROUGH,
ABI_ADDITIONAL_FILES,
EXTERNAL_FILES,
};
const std::unordered_map<std::string, VcpkgTripletVar> VCPKG_OPTIONS = {
@ -165,7 +165,7 @@ namespace vcpkg::Build
{"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE},
{"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE},
{"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH},
{"VCPKG_ABI_ADDITIONAL_FILES", VcpkgTripletVar::ABI_ADDITIONAL_FILES},
{"VCPKG_EXTERNAL_FILES", VcpkgTripletVar::EXTERNAL_FILES},
};
struct ExtendedBuildResult

34
toolsrc/src/vcpkg/binaryparagraph.cpp
View File

@ -2,6 +2,7 @@
#include <vcpkg/base/checks.h>
#include <vcpkg/base/system.print.h>
#include <vcpkg/base/util.h>
#include <vcpkg/binaryparagraph.h>
#include <vcpkg/parse.h>
@ -23,7 +24,7 @@ namespace vcpkg
static const std::string MAINTAINER = "Maintainer";
static const std::string DEPENDS = "Depends";
static const std::string DEFAULTFEATURES = "Default-Features";
static const std::string TYPE = "Type";
static const std::string EXTERNALFILES = "External-Files";
}
BinaryParagraph::BinaryParagraph() = default;
@ -61,8 +62,23 @@ namespace vcpkg
this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES));
}
this->type =
SourceParagraph::type_from_string(parser.optional_field(Fields::TYPE));
std::vector<std::string> external_files_or_hashes =
parse_comma_list(parser.optional_field(Fields::EXTERNALFILES));
if (external_files_or_hashes.size() % 2 != 0)
{
Checks::exit_with_message(
VCPKG_LINE_INFO,
"The External-Files field is not composed of key-value pairs for ",
this->spec);
}
for (int i = 0; i < external_files_or_hashes.size(); i += 2)
{
external_files.emplace(
std::move(external_files_or_hashes[i]),
std::move(external_files_or_hashes[i+1]));
}
if (const auto err = parser.error_info(this->spec.to_string()))
{
@ -124,6 +140,16 @@ namespace vcpkg
if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n');
if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n');
out_str.append("Type: ").append(SourceParagraph::string_from_type(pgh.type)).push_back('\n');
if (!pgh.external_files.empty())
{
out_str.append("External-Files: ");
out_str.append(Strings::join(",",
Util::fmap(
pgh.external_files,
[](const std::pair<std::string, std::string>& kv)
{
return kv.first + "," + kv.second;
}))).push_back('\n');
}
}
}

157
toolsrc/src/vcpkg/build.cpp
View File

@ -9,6 +9,7 @@
#include <vcpkg/base/system.debug.h>
#include <vcpkg/base/system.print.h>
#include <vcpkg/base/system.process.h>
#include <vcpkg/base/util.h>
#include <vcpkg/build.h>
#include <vcpkg/commands.h>
@ -261,10 +262,12 @@ namespace vcpkg::Build
return BinaryParagraph(source_paragraph, feature_paragraph, triplet);
}
static std::unique_ptr<BinaryControlFile> create_binary_control_file(const SourceParagraph& source_paragraph,
const Triplet& triplet,
const BuildInfo& build_info,
const std::string& abi_tag)
static std::unique_ptr<BinaryControlFile> create_binary_control_file(
const SourceParagraph& source_paragraph,
const Triplet& triplet,
const BuildInfo& build_info,
const PreBuildInfo& pre_build_info,
const std::string& abi_tag)
{
auto bcf = std::make_unique<BinaryControlFile>();
BinaryParagraph bpgh(source_paragraph, triplet, abi_tag);
@ -272,6 +275,14 @@ namespace vcpkg::Build
{
bpgh.version = *p_ver;
}
for (auto& file_hash : pre_build_info.external_files)
{
bpgh.external_files.emplace(
std::move(file_hash.first),
std::move(file_hash.second));
}
bcf->core_paragraph = std::move(bpgh);
return bcf;
}
@ -447,6 +458,41 @@ namespace vcpkg::Build
return command;
}
static std::vector<std::pair<std::string, std::string>> get_external_file_hashes(
const VcpkgPaths& paths,
const std::vector<fs::path>& files)
{
static std::map<fs::path, std::string> s_hash_cache;
const auto& fs = paths.get_filesystem();
std::vector<std::pair<std::string, std::string>> hashes;
for (const fs::path& external_file : files)
{
auto it_hash = s_hash_cache.find(external_file);
if (it_hash != s_hash_cache.end())
{
hashes.emplace_back(external_file, it_hash->second);
}
else if (fs.is_regular_file(external_file))
{
auto emp = s_hash_cache.emplace(external_file.u8string(),
Hash::get_file_hash(fs, external_file, "SHA1"));
hashes.emplace_back(external_file, emp.first->second);
}
else
{
Checks::exit_with_message(
VCPKG_LINE_INFO,
external_file.u8string() +
" was listed as an additional file for calculating the abi, but was not found.");
}
}
return hashes;
}
static std::string get_triplet_abi(const VcpkgPaths& paths,
const PreBuildInfo& pre_build_info,
const Triplet& triplet)
@ -496,31 +542,6 @@ namespace vcpkg::Build
s_hash_cache.emplace(triplet_file_path, hash);
}
for (const fs::path& additional_file : pre_build_info.additional_files)
{
it_hash = s_hash_cache.find(additional_file);
if (it_hash != s_hash_cache.end())
{
hash += "-";
hash += it_hash->second;
}
else if (fs.is_regular_file(additional_file))
{
std::string tmp_hash = Hash::get_file_hash(fs, additional_file, "SHA1");
hash += "-";
hash += tmp_hash;
s_hash_cache.emplace(additional_file, tmp_hash);
}
else
{
Checks::exit_with_message(
VCPKG_LINE_INFO,
additional_file.u8string() + " was listed as an additional file for calculating the abi, but was not found.");
}
}
return hash;
}
@ -572,7 +593,13 @@ namespace vcpkg::Build
const size_t error_count =
PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir);
auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag);
std::unique_ptr<BinaryControlFile> bcf =
create_binary_control_file(
*config.scf.core_paragraph,
triplet,
build_info,
pre_build_info,
abi_tag);
if (error_count != 0)
{
@ -629,7 +656,9 @@ namespace vcpkg::Build
std::vector<AbiEntry> abi_tag_entries(dependency_abis.begin(), dependency_abis.end());
abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)});
// Sorted here as the order of dependency_abis is the only
// non-deterministicly ordered set of AbiEntries
Util::sort(abi_tag_entries);
// If there is an unusually large number of files in the port then
// something suspicious is going on. Rather than hash all of them
@ -637,13 +666,16 @@ namespace vcpkg::Build
const int max_port_file_count = 100;
// the order of recursive_directory_iterator is undefined so save the names to sort
std::vector<fs::path> port_files;
std::vector<std::pair<std::string, std::string>> hashes_files;
for (auto& port_file : fs::stdfs::recursive_directory_iterator(config.port_dir))
{
if (fs::is_regular_file(status(port_file)))
{
port_files.push_back(port_file);
if (port_files.size() > max_port_file_count)
hashes_files.emplace_back(
vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"),
port_file.path().filename());
if (hashes_files.size() > max_port_file_count)
{
abi_tag_entries.emplace_back(AbiEntry{"no_hash_max_portfile", ""});
break;
@ -651,25 +683,45 @@ namespace vcpkg::Build
}
}
if (port_files.size() <= max_port_file_count)
if (hashes_files.size() <= max_port_file_count)
{
std::sort(port_files.begin(), port_files.end());
Util::sort(hashes_files);
int counter = 0;
for (auto& port_file : port_files)
for (auto& hash_file : hashes_files)
{
// When vcpkg takes a dependency on C++17 it can use fs::relative,
// which will give a stable ordering and better names in the key entry.
// this is not available in the filesystem TS so instead number the files for the key.
std::string key = Strings::format("file_%03d", counter++);
if (Debug::g_debugging)
{
System::print2("[DEBUG] mapping ", key, " from ", port_file.u8string(), "\n");
}
abi_tag_entries.emplace_back(AbiEntry{key, vcpkg::Hash::get_file_hash(fs, port_file, "SHA1")});
// We've already sorted by hash so it's safe to write down the
// filename, which will be consistent across machines.
abi_tag_entries.emplace_back(
AbiEntry{
std::move(hash_file.second),
std::move(hash_file.first)
});
}
}
//Make a copy of the external files and their hashes, and sort by hash
auto additional_file_hashes = pre_build_info.external_files;
std::sort(
additional_file_hashes.begin(),
additional_file_hashes.end(),
[](const std::pair<fs::path, std::string>& l,
const std::pair<fs::path, std::string>& r)
{
return l.second < r.second ||
(l.second == r.second && l.first < r.first);
});
for (auto& hash_file : additional_file_hashes)
{
abi_tag_entries.emplace_back(
AbiEntry{
std::move(hash_file.first),
std::move(hash_file.second)
});
}
abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)});
abi_tag_entries.emplace_back(AbiEntry{
"vcpkg_fixup_cmake_targets",
vcpkg::Hash::get_file_hash(fs, paths.scripts / "cmake" / "vcpkg_fixup_cmake_targets.cmake", "SHA1")});
@ -682,8 +734,6 @@ namespace vcpkg::Build
if (config.build_package_options.use_head_version == UseHeadVersion::YES)
abi_tag_entries.emplace_back(AbiEntry{"head", ""});
Util::sort(abi_tag_entries);
const std::string full_abi_info =
Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; });
@ -1114,12 +1164,15 @@ namespace vcpkg::Build
case VcpkgTripletVar::ENV_PASSTHROUGH :
pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";");
break;
case VcpkgTripletVar::ABI_ADDITIONAL_FILES :
pre_build_info.additional_files = Util::fmap(Strings::split(variable_value, ";"),
case VcpkgTripletVar::EXTERNAL_FILES :
pre_build_info.external_files =
get_external_file_hashes(
paths,
Util::fmap(Strings::split(variable_value, ";"),
[](const std::string& path)
{
return fs::path{path};
});
}));
break;
}
}

42
toolsrc/src/vcpkg/statusparagraphs.cpp
View File

@ -118,30 +118,30 @@ namespace vcpkg
return it != end() && (*it)->is_installed();
}
bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec)
{
auto it = find(spec);
if (it != end())
{
for (const std::string& dep : (*it)->package.depends)
{
PackageSpec dep_spec =
PackageSpec::from_name_and_triplet(
dep,
spec.triplet()).value_or_exit(VCPKG_LINE_INFO);
//bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec)
//{
// auto it = find(spec);
// if (it != end())
// {
// for (const std::string& dep : (*it)->package.depends)
// {
// PackageSpec dep_spec =
// PackageSpec::from_name_and_triplet(
// dep,
// spec.triplet()).value_or_exit(VCPKG_LINE_INFO);
if (needs_rebuild(dep_spec))
{
(*it)->state = InstallState::NEEDS_REBUILD;
return true;
}
}
// if (needs_rebuild(dep_spec))
// {
// (*it)->state = InstallState::NEEDS_REBUILD;
// return true;
// }
// }
return (*it)->needs_rebuild();
}
// return (*it)->needs_rebuild();
// }
return false;
}
// return false;
//}
StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr<StatusParagraph> pgh)
{