From 8fd34506c3f2d06af8abd8cfbb543ad16e79c6a3 Mon Sep 17 00:00:00 2001 From: Phil Christensen Date: Thu, 21 Feb 2019 22:24:20 -0800 Subject: [PATCH] [vcpkg] improve xunit xml output used in CI tests --- toolsrc/include/vcpkg/install.h | 1 - toolsrc/src/vcpkg/commands.ci.cpp | 235 +++++++++++++++++++++++++----- toolsrc/src/vcpkg/install.cpp | 10 +- 3 files changed, 204 insertions(+), 42 deletions(-) diff --git a/toolsrc/include/vcpkg/install.h b/toolsrc/include/vcpkg/install.h index b7acbf15f8d..2e92764dc50 100644 --- a/toolsrc/include/vcpkg/install.h +++ b/toolsrc/include/vcpkg/install.h @@ -37,7 +37,6 @@ namespace vcpkg::Install std::string total_elapsed_time; void print() const; - static std::string xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, Build::BuildResult code); std::string xunit_results() const; }; diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 5ca96274427..4ff503e1caf 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -52,24 +52,176 @@ namespace vcpkg::Commands::CI nullptr, }; + struct XunitTestResults + { + public: + + XunitTestResults() + { + m_assembly_run_datetime = Chrono::CTime::get_current_date_time(); + } + + void add_test_results(const std::string& spec, const Build::BuildResult& build_result, const Chrono::ElapsedTime& elapsed_time, const std::string& abi_tag) + { + m_collections.back().tests.push_back({ spec, build_result, elapsed_time, abi_tag }); + } + + // Starting a new test collection + void push_collection( const std::string& name) + { + m_collections.push_back({name}); + } + + void collection_time(const vcpkg::Chrono::ElapsedTime& time) + { + m_collections.back().time = time; + } + + const std::string& build_xml() + { + m_xml.clear(); + xml_start_assembly(); + + for (const auto& collection : m_collections) + { + xml_start_collection(collection); + for (const auto& test : collection.tests) + { + xml_test(test); + } + xml_finish_collection(); + } + + xml_finish_assembly(); + return m_xml; + } + + void assembly_time(const vcpkg::Chrono::ElapsedTime& assembly_time) + { + m_assembly_time = assembly_time; + } + + private: + + struct XunitTest + { + std::string name; + vcpkg::Build::BuildResult result; + vcpkg::Chrono::ElapsedTime time; + std::string abi_tag; + }; + + struct XunitCollection + { + std::string name; + vcpkg::Chrono::ElapsedTime time; + std::vector tests; + }; + + void xml_start_assembly() + { + std::string datetime; + if (m_assembly_run_datetime) + { + auto rawDateTime = m_assembly_run_datetime.get()->to_string(); + // The expected format is "yyyy-mm-ddThh:mm:ss.0Z" + // 0123456789012345678901 + datetime = Strings::format(R"(run-date="%s" run-time="%s")", + rawDateTime.substr(0, 10), rawDateTime.substr(11, 8)); + } + + std::string time = Strings::format(R"(time="%lld")", m_assembly_time.as().count()); + + m_xml += Strings::format( + R"()" "\n" + R"( )" "\n" + , datetime, time); + } + void xml_finish_assembly() + { + m_xml += " \n" + "\n"; + } + + void xml_start_collection(const XunitCollection& collection) + { + m_xml += Strings::format(R"( )" + "\n", + collection.name, + collection.time.as().count()); + } + void xml_finish_collection() + { + m_xml += " \n"; + } + + void xml_test(const XunitTest& test) + { + std::string message_block; + const char* result_string = ""; + switch (test.result) + { + case BuildResult::POST_BUILD_CHECKS_FAILED: + case BuildResult::FILE_CONFLICTS: + case BuildResult::BUILD_FAILED: + result_string = "Fail"; + message_block = Strings::format("", to_string(test.result)); + break; + case BuildResult::EXCLUDED: + case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: + result_string = "Skip"; + message_block = Strings::format("", to_string(test.result)); + break; + case BuildResult::SUCCEEDED: + result_string = "Pass"; + break; + default: + Checks::exit_fail(VCPKG_LINE_INFO); + break; + } + + std::string traits_block; + if (test.abi_tag != "") // only adding if there is a known abi tag + { + traits_block = Strings::format(R"()", test.abi_tag); + } + + m_xml += Strings::format(R"( %s%s)" + "\n", + test.name, + test.name, + test.time.as().count(), + result_string, + traits_block, + message_block); + } + + Optional m_assembly_run_datetime; + vcpkg::Chrono::ElapsedTime m_assembly_time; + std::vector m_collections; + + std::string m_xml; + }; + + struct UnknownCIPortsResults { std::vector unknown; std::map known; std::map> features; + std::map abi_tag_map; }; - static UnknownCIPortsResults find_unknown_ports_for_ci(const VcpkgPaths& paths, + static std::unique_ptr find_unknown_ports_for_ci(const VcpkgPaths& paths, const std::set& exclusions, const Dependencies::PortFileProvider& provider, const std::vector& fspecs, const bool purge_tombstones) { - UnknownCIPortsResults ret; + auto ret = std::make_unique(); auto& fs = paths.get_filesystem(); - std::map abi_tag_map; std::set will_fail; const Build::BuildPackageOptions build_options = { @@ -103,9 +255,9 @@ namespace vcpkg::Commands::CI auto dependency_abis = Util::fmap(p->computed_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry { - auto it = abi_tag_map.find(spec); + auto it = ret->abi_tag_map.find(spec); - if (it == abi_tag_map.end()) + if (it == ret->abi_tag_map.end()) return {spec.name(), ""}; else return {spec.name(), it->second}; @@ -118,13 +270,13 @@ namespace vcpkg::Commands::CI if (auto tag_and_file = maybe_tag_and_file.get()) { abi = tag_and_file->tag; - abi_tag_map.emplace(p->spec, abi); + ret->abi_tag_map.emplace(p->spec, abi); } } else if (auto ipv = p->installed_package.get()) { abi = ipv->core->package.abi; - if (!abi.empty()) abi_tag_map.emplace(p->spec, abi); + if (!abi.empty()) ret->abi_tag_map.emplace(p->spec, abi); } std::string state; @@ -143,35 +295,35 @@ namespace vcpkg::Commands::CI bool b_will_build = false; - ret.features.emplace(p->spec, + ret->features.emplace(p->spec, std::vector {p->feature_list.begin(), p->feature_list.end()}); if (Util::Sets::contains(exclusions, p->spec.name())) { - ret.known.emplace(p->spec, BuildResult::EXCLUDED); + ret->known.emplace(p->spec, BuildResult::EXCLUDED); will_fail.emplace(p->spec); } else if (std::any_of(p->computed_dependencies.begin(), p->computed_dependencies.end(), [&](const PackageSpec& spec) { return Util::Sets::contains(will_fail, spec); })) { - ret.known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES); + ret->known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES); will_fail.emplace(p->spec); } else if (fs.exists(archive_path)) { state += "pass"; - ret.known.emplace(p->spec, BuildResult::SUCCEEDED); + ret->known.emplace(p->spec, BuildResult::SUCCEEDED); } else if (fs.exists(archive_tombstone_path)) { state += "fail"; - ret.known.emplace(p->spec, BuildResult::BUILD_FAILED); + ret->known.emplace(p->spec, BuildResult::BUILD_FAILED); will_fail.emplace(p->spec); } else { - ret.unknown.push_back({p->spec, {p->feature_list.begin(), p->feature_list.end()}}); + ret->unknown.push_back({p->spec, {p->feature_list.begin(), p->feature_list.end()}}); b_will_build = true; } @@ -229,23 +381,29 @@ namespace vcpkg::Commands::CI }; std::vector> all_known_results; + std::map abi_tag_map; + + XunitTestResults xunitTestResults; std::vector all_ports = Install::get_all_port_names(paths); std::vector results; + auto timer = Chrono::ElapsedTimer::create_started(); for (const Triplet& triplet : triplets) { Input::check_triplet(triplet, paths); + xunitTestResults.push_collection(triplet.canonical_name()); + Dependencies::PackageGraph pgraph(paths_port_file, status_db); std::vector specs = PackageSpec::to_package_specs(all_ports, triplet); // Install the default features for every package - auto all_fspecs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); }); + auto all_feature_specs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); }); auto split_specs = - find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_fspecs, purge_tombstones); - auto fspecs = FullPackageSpec::to_feature_specs(split_specs.unknown); + find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_feature_specs, purge_tombstones); + auto feature_specs = FullPackageSpec::to_feature_specs(split_specs->unknown); - for (auto&& fspec : fspecs) + for (auto&& fspec : feature_specs) pgraph.install(fspec); Dependencies::CreateInstallPlanOptions serialize_options; @@ -284,7 +442,7 @@ namespace vcpkg::Commands::CI p->plan_type = InstallPlanType::EXCLUDED; } - for (auto&& feature : split_specs.features[p->spec]) + for (auto&& feature : split_specs->features[p->spec]) if (p->feature_list.find(feature) == p->feature_list.end()) { pgraph.install({p->spec, feature}); @@ -304,13 +462,32 @@ namespace vcpkg::Commands::CI } else { + auto collection_timer = Chrono::ElapsedTimer::create_started(); auto summary = Install::perform(action_plan, Install::KeepGoing::YES, paths, status_db); + auto collection_time_elapsed = collection_timer.elapsed(); + + // Adding results for ports that were built or pulled from an archive for (auto&& result : summary.results) - split_specs.known.erase(result.spec); - results.push_back({triplet, std::move(summary)}); - all_known_results.emplace_back(std::move(split_specs.known)); + { + split_specs->known.erase(result.spec); + xunitTestResults.add_test_results(result.spec.to_string(), result.build_result.code, result.timing, split_specs->abi_tag_map.at(result.spec)); + } + + // Adding results for ports that were not built because they have known states + for (auto&& port : split_specs->known) + { + xunitTestResults.add_test_results(port.first.to_string(), port.second, Chrono::ElapsedTime{}, split_specs->abi_tag_map.at(port.first)); + } + + all_known_results.emplace_back(std::move(split_specs->known)); + abi_tag_map.insert(split_specs->abi_tag_map.begin(), split_specs->abi_tag_map.end()); + + results.push_back({ triplet, std::move(summary)}); + + xunitTestResults.collection_time( collection_time_elapsed ); } } + xunitTestResults.assembly_time(timer.elapsed()); for (auto&& result : results) { @@ -322,21 +499,7 @@ namespace vcpkg::Commands::CI auto it_xunit = options.settings.find(OPTION_XUNIT); if (it_xunit != options.settings.end()) { - std::string xunit_doc = "\n"; - - for (auto&& result : results) - xunit_doc += result.summary.xunit_results(); - for (auto&& known_result : all_known_results) - { - for (auto&& result : known_result) - { - xunit_doc += - Install::InstallSummary::xunit_result(result.first, Chrono::ElapsedTime {}, result.second); - } - } - - xunit_doc += "\n"; - paths.get_filesystem().write_contents(fs::u8path(it_xunit->second), xunit_doc); + paths.get_filesystem().write_contents(fs::u8path(it_xunit->second), xunitTestResults.build_xml()); } Checks::exit_success(VCPKG_LINE_INFO); diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 1cfa2bf7199..4348768718a 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -719,9 +719,9 @@ namespace vcpkg::Install return nullptr; } - std::string InstallSummary::xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, BuildResult code) + static std::string xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, BuildResult code) { - std::string inner_block; + std::string message_block; const char* result_string = ""; switch (code) { @@ -729,12 +729,12 @@ namespace vcpkg::Install case BuildResult::FILE_CONFLICTS: case BuildResult::BUILD_FAILED: result_string = "Fail"; - inner_block = Strings::format("", to_string(code)); + message_block = Strings::format("", to_string(code)); break; case BuildResult::EXCLUDED: case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: result_string = "Skip"; - inner_block = Strings::format("", to_string(code)); + message_block = Strings::format("", to_string(code)); break; case BuildResult::SUCCEEDED: result_string = "Pass"; break; default: Checks::exit_fail(VCPKG_LINE_INFO); @@ -746,7 +746,7 @@ namespace vcpkg::Install spec, time.as().count(), result_string, - inner_block); + message_block); } std::string InstallSummary::xunit_results() const