// This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // // Copyright (C) 2020 Intel Corporation #include "../test_precomp.hpp" #ifdef HAVE_ONNX #include #include #include #include namespace { struct ONNXInitPath { ONNXInitPath() { const char* env_path = getenv("OPENCV_GAPI_ONNX_MODEL_PATH"); if (env_path) cvtest::addDataSearchPath(env_path); } }; static ONNXInitPath g_init_path; cv::Mat initMatrixRandU(int type, cv::Size sz_in) { cv::Mat in_mat1 = cv::Mat(sz_in, type); if (CV_MAT_DEPTH(type) < CV_32F) { cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); } else { const int fscale = 256; // avoid bits near ULP, generate stable test input cv::Mat in_mat32s(in_mat1.size(), CV_MAKE_TYPE(CV_32S, CV_MAT_CN(type))); cv::randu(in_mat32s, cv::Scalar::all(0), cv::Scalar::all(255 * fscale)); in_mat32s.convertTo(in_mat1, type, 1.0f / fscale, 0); } return in_mat1; } } namespace opencv_test { namespace { // FIXME: taken from the DNN module void normAssert(cv::InputArray ref, cv::InputArray test, const char *comment /*= ""*/, double l1 = 0.00001, double lInf = 0.0001) { double normL1 = cvtest::norm(ref, test, cv::NORM_L1) / ref.getMat().total(); EXPECT_LE(normL1, l1) << comment; double normInf = cvtest::norm(ref, test, cv::NORM_INF); EXPECT_LE(normInf, lInf) << comment; } std::string findModel(const std::string &model_name) { return findDataFile("vision/classification/squeezenet/model/" + model_name + ".onnx", false); } inline void preprocess(const cv::Mat& src, cv::Mat& dst, const cv::Scalar& mean, const cv::Scalar& std) { int new_h = 224; int new_w = 224; cv::Mat tmp, nmat, cvt; cv::resize(src, dst, cv::Size(new_w, new_h)); dst.convertTo(cvt, CV_32F, 1.f / 255); nmat = cvt - mean; tmp = nmat / std; dst.create(cv::Size(new_w, new_h * src.channels()), CV_32F); std::vector planes; for (int i = 0; i < src.channels(); ++i) { planes.push_back(dst.rowRange(i * new_h, (i + 1) * new_h)); } cv::split(tmp, planes); } void InferONNX(const std::string& model_path, const cv::Mat& in, cv::Mat& out, const cv::Scalar& mean, const cv::Scalar& std) { // FIXME: It must be a FIXTURE test! Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "test"); Ort::SessionOptions session_options; Ort::Session session(env, model_path.data(), session_options); auto input_node_dims = // 0 - one input session.GetInputTypeInfo(0).GetTensorTypeAndShapeInfo().GetShape(); auto output_node_dims = // 0 - one output session.GetOutputTypeInfo(0).GetTensorTypeAndShapeInfo().GetShape(); Ort::AllocatorWithDefaultOptions allocator; char* in_node_name_p = session.GetInputName(0, allocator); char* out_node_name_p = session.GetOutputName(0, allocator); std::string in_node_name(in_node_name_p); std::string out_node_name(out_node_name_p); allocator.Free(in_node_name_p); allocator.Free(out_node_name_p); auto memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); cv::Mat dst; preprocess(in, dst, mean, std); out.create(std::vector(output_node_dims.begin(), output_node_dims.end()), CV_32F); // empty output Mat auto in_tensor = Ort::Value::CreateTensor(memory_info, dst.ptr(), dst.total(), input_node_dims.data(), input_node_dims.size()); auto out_tensor = Ort::Value::CreateTensor(memory_info, out.ptr(), out.total(), output_node_dims.data(), output_node_dims.size()); std::vector in_names = {in_node_name.data()}; std::vector out_names = {out_node_name.data()}; session.Run(Ort::RunOptions{nullptr}, in_names.data(), &in_tensor, session.GetInputCount(), out_names.data(), &out_tensor, session.GetOutputCount()); } } // anonymous namespace TEST(ONNX, Infer) { cv::Mat in_mat1, out_gapi, out_onnx; std::string model_path = findModel("squeezenet1.0-9"); // NOTE: All tests chek "random" image // Ideally it should be a real image in_mat1 = initMatrixRandU(CV_8UC3, cv::Size{640, 480}); cv::Scalar mean = { 0.485, 0.456, 0.406 }; cv::Scalar std = { 0.229, 0.224, 0.225 }; // ONNX_API code InferONNX(model_path, in_mat1, out_onnx, mean, std); // G_API code G_API_NET(SqueezNet, , "squeeznet"); cv::GMat in; cv::GMat out = cv::gapi::infer(in); cv::GComputation comp(cv::GIn(in), cv::GOut(out)); // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({mean},{std}); comp.apply(cv::gin(in_mat1), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate ASSERT_EQ(1000u, out_onnx.total()); ASSERT_EQ(1000u, out_gapi.total()); normAssert(out_onnx, out_gapi, "Test classification output"); } TEST(ONNX, InferROI) { cv::Mat in_mat1, out_gapi, out_onnx; std::string model_path = findModel("squeezenet1.0-9"); in_mat1 = initMatrixRandU(CV_8UC3, cv::Size{640, 480}); cv::Scalar mean = { 0.485, 0.456, 0.406 }; // squeeznet mean cv::Scalar std = { 0.229, 0.224, 0.225 }; // squeeznet std cv::Rect ROI(cv::Point{0, 0}, cv::Size{250, 250}); // ONNX_API code InferONNX(model_path, in_mat1(ROI), out_onnx, mean, std); // G_API code G_API_NET(SqueezNet, , "squeeznet"); cv::GMat in; cv::GOpaque rect; cv::GMat out = cv::gapi::infer(rect, in); cv::GComputation comp(cv::GIn(in, rect), cv::GOut(out)); auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({mean},{std}); comp.apply(cv::gin(in_mat1, ROI), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate ASSERT_EQ(1000u, out_onnx.total()); ASSERT_EQ(1000u, out_gapi.total()); normAssert(out_onnx, out_gapi, "Test classification output"); } TEST(ONNX, InferROIList) { cv::Mat in_mat1; std::string model_path = findModel("squeezenet1.0-9"); in_mat1 = initMatrixRandU(CV_8UC3, cv::Size{640, 480}); cv::Scalar mean = { 0.485, 0.456, 0.406 }; // squeeznet mean cv::Scalar std = { 0.229, 0.224, 0.225 }; // squeeznet std std::vector rois = { cv::Rect(cv::Point{ 0, 0}, cv::Size{80, 120}), cv::Rect(cv::Point{50, 100}, cv::Size{250, 360}), }; std::vector out_gapi; std::vector out_onnx(rois.size()); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { InferONNX(model_path, in_mat1(rois[i]), out_onnx[i], mean, std); } // G_API code G_API_NET(SqueezNet, , "squeeznet"); cv::GMat in; cv::GArray rr; cv::GArray out = cv::gapi::infer(rr, in); cv::GComputation comp(cv::GIn(in, rr), cv::GOut(out)); auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({mean},{std}); comp.apply(cv::gin(in_mat1, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate for (size_t i = 0; i < rois.size(); ++i) { ASSERT_EQ(1000u, out_onnx[i].total()); ASSERT_EQ(1000u, out_gapi[i].total()); normAssert(out_onnx[i], out_gapi[i], "Test classification output"); } } TEST(ONNX, Infer2ROIList) { cv::Mat in_mat1; std::string model_path = findModel("squeezenet1.0-9"); in_mat1 = initMatrixRandU(CV_8UC3, cv::Size{640, 480}); cv::Scalar mean = { 0.485, 0.456, 0.406 }; // squeeznet mean cv::Scalar std = { 0.229, 0.224, 0.225 }; // squeeznet std std::vector rois = { cv::Rect(cv::Point{ 0, 0}, cv::Size{80, 120}), cv::Rect(cv::Point{50, 100}, cv::Size{250, 360}), }; std::vector out_gapi; std::vector out_onnx(rois.size()); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { InferONNX(model_path, in_mat1(rois[i]), out_onnx[i], mean, std); } // G_API code G_API_NET(SqueezNet, , "squeeznet"); cv::GMat in; cv::GArray rr; cv::GArray out = cv::gapi::infer2(in,rr); cv::GComputation comp(cv::GIn(in, rr), cv::GOut(out)); auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({mean},{std}); comp.apply(cv::gin(in_mat1, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate for (size_t i = 0; i < rois.size(); ++i) { ASSERT_EQ(1000u, out_onnx[i].total()); ASSERT_EQ(1000u, out_gapi[i].total()); normAssert(out_onnx[i], out_gapi[i], "Test classification output"); } } } // namespace opencv_test #endif // HAVE_ONNX