[vcpkg] improve xunit xml output used in CI tests

toolsrc/include/vcpkg/install.h

@@ -37,7 +37,6 @@ namespace vcpkg::Install
         std::string total_elapsed_time;
 
         void print() const;
-        static std::string xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, Build::BuildResult code);
         std::string xunit_results() const;
     };
 

toolsrc/src/vcpkg/commands.ci.cpp

@@ -52,24 +52,176 @@ namespace vcpkg::Commands::CI
         nullptr,
     };
 
+    struct XunitTestResults
+    {
+    public:
+
+        XunitTestResults()
+        {
+            m_assembly_run_datetime = Chrono::CTime::get_current_date_time();
+        }
+
+        void add_test_results(const std::string& spec, const Build::BuildResult& build_result, const Chrono::ElapsedTime& elapsed_time, const std::string& abi_tag)
+        {
+            m_collections.back().tests.push_back({ spec, build_result, elapsed_time, abi_tag });
+        }
+
+        // Starting a new test collection
+        void push_collection( const std::string& name)
+        {
+            m_collections.push_back({name});
+        }
+
+        void collection_time(const vcpkg::Chrono::ElapsedTime& time)
+        {
+            m_collections.back().time = time;
+        }
+
+        const std::string& build_xml()
+        {
+            m_xml.clear();
+            xml_start_assembly();
+
+            for (const auto& collection : m_collections)
+            {
+                xml_start_collection(collection);
+                for (const auto& test : collection.tests)
+                {
+                    xml_test(test);
+                }
+                xml_finish_collection();
+            }
+
+            xml_finish_assembly();
+            return m_xml;
+        }
+
+        void assembly_time(const vcpkg::Chrono::ElapsedTime& assembly_time)
+        {
+            m_assembly_time = assembly_time;
+        }
+
+    private:
+
+        struct XunitTest
+        {
+            std::string name;
+            vcpkg::Build::BuildResult result;
+            vcpkg::Chrono::ElapsedTime time;
+            std::string abi_tag;
+        };
+
+        struct XunitCollection
+        {
+            std::string name;
+            vcpkg::Chrono::ElapsedTime time;
+            std::vector<XunitTest> tests;
+        };
+
+        void xml_start_assembly()
+        {
+            std::string datetime;
+            if (m_assembly_run_datetime)
+            {
+                auto rawDateTime = m_assembly_run_datetime.get()->to_string();
+                // The expected format is "yyyy-mm-ddThh:mm:ss.0Z"
+                //                         0123456789012345678901
+                datetime = Strings::format(R"(run-date="%s" run-time="%s")",
+                    rawDateTime.substr(0, 10), rawDateTime.substr(11, 8));
+            }
+
+            std::string time = Strings::format(R"(time="%lld")", m_assembly_time.as<std::chrono::seconds>().count());
+
+            m_xml += Strings::format(
+                R"(<assemblies>)" "\n"
+                R"(  <assembly name="vcpkg" %s %s>)"  "\n"
+                , datetime, time);
+        }
+        void xml_finish_assembly()
+        {
+            m_xml += "  </assembly>\n"
+                "</assemblies>\n";
+        }
+
+        void xml_start_collection(const XunitCollection& collection)
+        {
+            m_xml += Strings::format(R"(    <collection name="%s" time="%lld">)"
+                "\n",
+                collection.name,
+                collection.time.as<std::chrono::seconds>().count());
+        }
+        void xml_finish_collection()
+        {
+            m_xml += "    </collection>\n";
+        }
+
+        void xml_test(const XunitTest& test)
+        {
+            std::string message_block;
+            const char* result_string = "";
+            switch (test.result)
+            {
+            case BuildResult::POST_BUILD_CHECKS_FAILED:
+            case BuildResult::FILE_CONFLICTS:
+            case BuildResult::BUILD_FAILED:
+                result_string = "Fail";
+                message_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", to_string(test.result));
+                break;
+            case BuildResult::EXCLUDED:
+            case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES:
+                result_string = "Skip";
+                message_block = Strings::format("<reason><![CDATA[%s]]></reason>", to_string(test.result));
+                break;
+            case BuildResult::SUCCEEDED:
+                result_string = "Pass";
+                break;
+            default:
+                Checks::exit_fail(VCPKG_LINE_INFO);
+                break;
+            }
+
+            std::string traits_block;
+            if (test.abi_tag != "") // only adding if there is a known abi tag
+            {
+                traits_block = Strings::format(R"(<traits><trait name="abi_tag" value="%s" /></traits>)", test.abi_tag);
+            }
+
+            m_xml += Strings::format(R"(      <test name="%s" method="%s" time="%lld" result="%s">%s%s</test>)"
+                "\n",
+                test.name,
+                test.name,
+                test.time.as<std::chrono::seconds>().count(),
+                result_string,
+                traits_block,
+                message_block);
+        }
+
+        Optional<vcpkg::Chrono::CTime> m_assembly_run_datetime;
+        vcpkg::Chrono::ElapsedTime m_assembly_time;
+        std::vector<XunitCollection> m_collections;
+
+        std::string m_xml;
+    };
+
+
     struct UnknownCIPortsResults
     {
         std::vector<FullPackageSpec> unknown;
         std::map<PackageSpec, Build::BuildResult> known;
         std::map<PackageSpec, std::vector<std::string>> features;
+        std::map<PackageSpec, std::string> abi_tag_map;
     };
 
-    static UnknownCIPortsResults find_unknown_ports_for_ci(const VcpkgPaths& paths,
+    static std::unique_ptr<UnknownCIPortsResults> find_unknown_ports_for_ci(const VcpkgPaths& paths,
                                                            const std::set<std::string>& exclusions,
                                                            const Dependencies::PortFileProvider& provider,
                                                            const std::vector<FeatureSpec>& fspecs,
                                                            const bool purge_tombstones)
     {
-        UnknownCIPortsResults ret;
+        auto ret = std::make_unique<UnknownCIPortsResults>();
 
         auto& fs = paths.get_filesystem();
 
-        std::map<PackageSpec, std::string> abi_tag_map;
         std::set<PackageSpec> will_fail;
 
         const Build::BuildPackageOptions build_options = {
@@ -103,9 +255,9 @@ namespace vcpkg::Commands::CI
 
                     auto dependency_abis =
                         Util::fmap(p->computed_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry {
-                            auto it = abi_tag_map.find(spec);
+                            auto it = ret->abi_tag_map.find(spec);
 
-                            if (it == abi_tag_map.end())
+                            if (it == ret->abi_tag_map.end())
                                 return {spec.name(), ""};
                             else
                                 return {spec.name(), it->second};
@@ -118,13 +270,13 @@ namespace vcpkg::Commands::CI
                     if (auto tag_and_file = maybe_tag_and_file.get())
                     {
                         abi = tag_and_file->tag;
-                        abi_tag_map.emplace(p->spec, abi);
+                        ret->abi_tag_map.emplace(p->spec, abi);
                     }
                 }
                 else if (auto ipv = p->installed_package.get())
                 {
                     abi = ipv->core->package.abi;
-                    if (!abi.empty()) abi_tag_map.emplace(p->spec, abi);
+                    if (!abi.empty()) ret->abi_tag_map.emplace(p->spec, abi);
                 }
 
                 std::string state;
@@ -143,35 +295,35 @@ namespace vcpkg::Commands::CI
 
                 bool b_will_build = false;
 
-                ret.features.emplace(p->spec,
+                ret->features.emplace(p->spec,
                                      std::vector<std::string> {p->feature_list.begin(), p->feature_list.end()});
 
                 if (Util::Sets::contains(exclusions, p->spec.name()))
                 {
-                    ret.known.emplace(p->spec, BuildResult::EXCLUDED);
+                    ret->known.emplace(p->spec, BuildResult::EXCLUDED);
                     will_fail.emplace(p->spec);
                 }
                 else if (std::any_of(p->computed_dependencies.begin(),
                                      p->computed_dependencies.end(),
                                      [&](const PackageSpec& spec) { return Util::Sets::contains(will_fail, spec); }))
                 {
-                    ret.known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES);
+                    ret->known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES);
                     will_fail.emplace(p->spec);
                 }
                 else if (fs.exists(archive_path))
                 {
                     state += "pass";
-                    ret.known.emplace(p->spec, BuildResult::SUCCEEDED);
+                    ret->known.emplace(p->spec, BuildResult::SUCCEEDED);
                 }
                 else if (fs.exists(archive_tombstone_path))
                 {
                     state += "fail";
-                    ret.known.emplace(p->spec, BuildResult::BUILD_FAILED);
+                    ret->known.emplace(p->spec, BuildResult::BUILD_FAILED);
                     will_fail.emplace(p->spec);
                 }
                 else
                 {
-                    ret.unknown.push_back({p->spec, {p->feature_list.begin(), p->feature_list.end()}});
+                    ret->unknown.push_back({p->spec, {p->feature_list.begin(), p->feature_list.end()}});
                     b_will_build = true;
                 }
 
@@ -229,23 +381,29 @@ namespace vcpkg::Commands::CI
         };
 
         std::vector<std::map<PackageSpec, BuildResult>> all_known_results;
+        std::map<PackageSpec, std::string> abi_tag_map;
+
+        XunitTestResults xunitTestResults;
 
         std::vector<std::string> all_ports = Install::get_all_port_names(paths);
         std::vector<TripletAndSummary> results;
+        auto timer = Chrono::ElapsedTimer::create_started();
         for (const Triplet& triplet : triplets)
         {
             Input::check_triplet(triplet, paths);
 
+            xunitTestResults.push_collection(triplet.canonical_name());
+
             Dependencies::PackageGraph pgraph(paths_port_file, status_db);
 
             std::vector<PackageSpec> specs = PackageSpec::to_package_specs(all_ports, triplet);
             // Install the default features for every package
-            auto all_fspecs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); });
+            auto all_feature_specs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); });
             auto split_specs =
-                find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_fspecs, purge_tombstones);
+                find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_feature_specs, purge_tombstones);
-            auto fspecs = FullPackageSpec::to_feature_specs(split_specs.unknown);
+            auto feature_specs = FullPackageSpec::to_feature_specs(split_specs->unknown);
 
-            for (auto&& fspec : fspecs)
+            for (auto&& fspec : feature_specs)
                 pgraph.install(fspec);
 
             Dependencies::CreateInstallPlanOptions serialize_options;
@@ -284,7 +442,7 @@ namespace vcpkg::Commands::CI
                                 p->plan_type = InstallPlanType::EXCLUDED;
                             }
 
-                            for (auto&& feature : split_specs.features[p->spec])
+                            for (auto&& feature : split_specs->features[p->spec])
                                 if (p->feature_list.find(feature) == p->feature_list.end())
                                 {
                                     pgraph.install({p->spec, feature});
@@ -304,13 +462,32 @@ namespace vcpkg::Commands::CI
             }
             else
             {
+                auto collection_timer = Chrono::ElapsedTimer::create_started();
                 auto summary = Install::perform(action_plan, Install::KeepGoing::YES, paths, status_db);
+                auto collection_time_elapsed = collection_timer.elapsed();
+
+                // Adding results for ports that were built or pulled from an archive
                 for (auto&& result : summary.results)
-                    split_specs.known.erase(result.spec);
+                {
-                results.push_back({triplet, std::move(summary)});
+                    split_specs->known.erase(result.spec);
-                all_known_results.emplace_back(std::move(split_specs.known));
+                    xunitTestResults.add_test_results(result.spec.to_string(), result.build_result.code, result.timing, split_specs->abi_tag_map.at(result.spec));
+                }
+
+                // Adding results for ports that were not built because they have known states
+                for (auto&& port : split_specs->known)
+                {
+                    xunitTestResults.add_test_results(port.first.to_string(), port.second, Chrono::ElapsedTime{}, split_specs->abi_tag_map.at(port.first));
+                }
+
+                all_known_results.emplace_back(std::move(split_specs->known));
+                abi_tag_map.insert(split_specs->abi_tag_map.begin(), split_specs->abi_tag_map.end());
+
+                results.push_back({ triplet, std::move(summary)});
+
+                xunitTestResults.collection_time( collection_time_elapsed );
             }
         }
+        xunitTestResults.assembly_time(timer.elapsed());
 
         for (auto&& result : results)
         {
@@ -322,21 +499,7 @@ namespace vcpkg::Commands::CI
         auto it_xunit = options.settings.find(OPTION_XUNIT);
         if (it_xunit != options.settings.end())
         {
-            std::string xunit_doc = "<assemblies><assembly><collection>\n";
+            paths.get_filesystem().write_contents(fs::u8path(it_xunit->second), xunitTestResults.build_xml());
-
-            for (auto&& result : results)
-                xunit_doc += result.summary.xunit_results();
-            for (auto&& known_result : all_known_results)
-            {
-                for (auto&& result : known_result)
-                {
-                    xunit_doc +=
-                        Install::InstallSummary::xunit_result(result.first, Chrono::ElapsedTime {}, result.second);
-                }
-            }
-
-            xunit_doc += "</collection></assembly></assemblies>\n";
-            paths.get_filesystem().write_contents(fs::u8path(it_xunit->second), xunit_doc);
         }
 
         Checks::exit_success(VCPKG_LINE_INFO);

toolsrc/src/vcpkg/install.cpp

@@ -719,9 +719,9 @@ namespace vcpkg::Install
         return nullptr;
     }
 
-    std::string InstallSummary::xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, BuildResult code)
+    static std::string xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, BuildResult code)
     {
-        std::string inner_block;
+        std::string message_block;
         const char* result_string = "";
         switch (code)
         {
@@ -729,12 +729,12 @@ namespace vcpkg::Install
             case BuildResult::FILE_CONFLICTS:
             case BuildResult::BUILD_FAILED:
                 result_string = "Fail";
-                inner_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", to_string(code));
+                message_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", to_string(code));
                 break;
             case BuildResult::EXCLUDED:
             case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES:
                 result_string = "Skip";
-                inner_block = Strings::format("<reason><![CDATA[%s]]></reason>", to_string(code));
+                message_block = Strings::format("<reason><![CDATA[%s]]></reason>", to_string(code));
                 break;
             case BuildResult::SUCCEEDED: result_string = "Pass"; break;
             default: Checks::exit_fail(VCPKG_LINE_INFO);
@@ -746,7 +746,7 @@ namespace vcpkg::Install
                                spec,
                                time.as<std::chrono::seconds>().count(),
                                result_string,
-                               inner_block);
+                               message_block);
     }
 
     std::string InstallSummary::xunit_results() const