diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index 9d5acf04e3..c05a7088cd 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -121,7 +121,9 @@ PERF_TEST_P_(DNNTestNetwork, Inception_5h) PERF_TEST_P_(DNNTestNetwork, ENet) { - if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE) || + (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16)) + throw SkipTestException(""); processNet("dnn/Enet-model-best.net", "", "enet.yml", Mat(cv::Size(512, 256), CV_32FC3)); } @@ -232,7 +234,8 @@ const tuple testCases[] = { tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL_FP16), #endif tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_CPU), - tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL) + tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL), + tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL_FP16) }; INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, testing::ValuesIn(testCases)); diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 2752d7ab2a..2bcd357e2e 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -147,7 +147,9 @@ TEST_P(DNNTestNetwork, Inception_5h) TEST_P(DNNTestNetwork, ENet) { - if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE) || + (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16)) + throw SkipTestException(""); processNet("dnn/Enet-model-best.net", "", Size(512, 512), "l367_Deconvolution", target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_enet.yml" : "dnn/halide_scheduler_enet.yml", @@ -161,9 +163,11 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe) throw SkipTestException(""); Mat sample = imread(findDataFile("dnn/street.png", false)); Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); + float l1 = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.0007 : 0.0; + float lInf = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.011 : 0.0; processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt", - inp, "detection_out"); + inp, "detection_out", "", l1, lInf); } TEST_P(DNNTestNetwork, MobileNet_SSD_TensorFlow) @@ -173,15 +177,17 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_TensorFlow) throw SkipTestException(""); Mat sample = imread(findDataFile("dnn/street.png", false)); Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); + float l1 = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.008 : 0.0; + float lInf = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.06 : 0.0; processNet("dnn/ssd_mobilenet_v1_coco.pb", "dnn/ssd_mobilenet_v1_coco.pbtxt", - inp, "detection_out"); + inp, "detection_out", "", l1, lInf); } TEST_P(DNNTestNetwork, SSD_VGG16) { - if (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL || - backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU || - backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU) + if ((backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) || + (backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU) || + (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)) throw SkipTestException(""); processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", Size(300, 300), "detection_out"); @@ -236,14 +242,17 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow) throw SkipTestException(""); Mat sample = imread(findDataFile("dnn/street.png", false)); Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); + float l1 = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.008 : 0.0; + float lInf = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.07 : 0.0; processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "dnn/ssd_inception_v2_coco_2017_11_17.pbtxt", - inp, "detection_out"); + inp, "detection_out", "", l1, lInf); } TEST_P(DNNTestNetwork, DenseNet_121) { - if (backend == DNN_BACKEND_HALIDE || - backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) + if ((backend == DNN_BACKEND_HALIDE) || + (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) || + (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16)) throw SkipTestException(""); processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", Size(224, 224), "", "caffe"); } @@ -258,7 +267,8 @@ const tuple testCases[] = { tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL), tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL_FP16), #endif - tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL) + tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL), + tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL_FP16) }; INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, testing::ValuesIn(testCases)); diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 82b395a120..eaf95acc9d 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -104,7 +104,11 @@ TEST_P(Reproducibility_AlexNet, Accuracy) ASSERT_FALSE(net.empty()); } - net.setPreferableTarget(get<1>(GetParam())); + int targetId = get<1>(GetParam()); + const float l1 = 1e-5; + const float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 3e-3 : 1e-4; + + net.setPreferableTarget(targetId); Mat sample = imread(_tf("grace_hopper_227.png")); ASSERT_TRUE(!sample.empty()); @@ -112,10 +116,11 @@ TEST_P(Reproducibility_AlexNet, Accuracy) net.setInput(blobFromImage(sample, 1.0f, Size(227, 227), Scalar(), false), "data"); Mat out = net.forward("prob"); Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy")); - normAssert(ref, out); + normAssert(ref, out, "", l1, lInf); } -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_AlexNet, Combine(testing::Bool(), availableDnnTargets())); +INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_AlexNet, Combine(testing::Bool(), + Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16))); #if !defined(_WIN32) || defined(_WIN64) TEST(Reproducibility_FCN, Accuracy) @@ -176,8 +181,11 @@ TEST_P(Reproducibility_MobileNet_SSD, Accuracy) const string proto = findDataFile("dnn/MobileNetSSD_deploy.prototxt", false); const string model = findDataFile("dnn/MobileNetSSD_deploy.caffemodel", false); Net net = readNetFromCaffe(proto, model); + int targetId = GetParam(); + const float l1 = (targetId == DNN_TARGET_OPENCL_FP16) ? 1.5e-4 : 1e-5; + const float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 4e-4 : 1e-4; - net.setPreferableTarget(GetParam()); + net.setPreferableTarget(targetId); Mat sample = imread(_tf("street.png")); @@ -185,8 +193,10 @@ TEST_P(Reproducibility_MobileNet_SSD, Accuracy) net.setInput(inp); Mat out = net.forward(); + const float scores_diff = (targetId == DNN_TARGET_OPENCL_FP16) ? 4e-4 : 1e-5; + const float boxes_iou_diff = (targetId == DNN_TARGET_OPENCL_FP16) ? 5e-3 : 1e-4; Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy")); - normAssertDetections(ref, out); + normAssertDetections(ref, out, "", 0.0, scores_diff, boxes_iou_diff); // Check that detections aren't preserved. inp.setTo(0.0f); @@ -212,10 +222,12 @@ TEST_P(Reproducibility_MobileNet_SSD, Accuracy) // a single sample in batch. The first numbers of detection vectors are batch id. outBatch = outBatch.reshape(1, outBatch.total() / 7); EXPECT_EQ(outBatch.rows, 2 * numDetections); - normAssert(outBatch.rowRange(0, numDetections), ref); - normAssert(outBatch.rowRange(numDetections, 2 * numDetections).colRange(1, 7), ref.colRange(1, 7)); + normAssert(outBatch.rowRange(0, numDetections), ref, "", l1, lInf); + normAssert(outBatch.rowRange(numDetections, 2 * numDetections).colRange(1, 7), ref.colRange(1, 7), + "", l1, lInf); } -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_MobileNet_SSD, availableDnnTargets()); +INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_MobileNet_SSD, + Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16)); typedef testing::TestWithParam Reproducibility_ResNet50; TEST_P(Reproducibility_ResNet50, Accuracy) @@ -226,6 +238,9 @@ TEST_P(Reproducibility_ResNet50, Accuracy) int targetId = GetParam(); net.setPreferableTarget(targetId); + float l1 = (targetId == DNN_TARGET_OPENCL_FP16) ? 3e-5 : 1e-5; + float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 6e-3 : 1e-4; + Mat input = blobFromImage(imread(_tf("googlenet_0.png")), 1.0f, Size(224,224), Scalar(), false); ASSERT_TRUE(!input.empty()); @@ -233,20 +248,21 @@ TEST_P(Reproducibility_ResNet50, Accuracy) Mat out = net.forward(); Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); - normAssert(ref, out); + normAssert(ref, out, "", l1, lInf); - if (targetId == DNN_TARGET_OPENCL) + if (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) { UMat out_umat; net.forward(out_umat); - normAssert(ref, out_umat, "out_umat"); + normAssert(ref, out_umat, "out_umat", l1, lInf); std::vector out_umats; net.forward(out_umats); - normAssert(ref, out_umats[0], "out_umat_vector"); + normAssert(ref, out_umats[0], "out_umat_vector", l1, lInf); } } -INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50, availableDnnTargets()); +INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50, + Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16)); typedef testing::TestWithParam Reproducibility_SqueezeNet_v1_1; TEST_P(Reproducibility_SqueezeNet_v1_1, Accuracy) diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 397aadfa08..84d224e33d 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -295,26 +295,32 @@ TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8) INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, availableDnnTargets()); +typedef testing::TestWithParam Test_TensorFlow_fp16; + +TEST_P(Test_TensorFlow_fp16, tests) +{ + int targetId = GetParam(); + const float l1 = 7e-4; + const float lInf = 1e-2; + runTensorFlowNet("fp16_single_conv", targetId, false, l1, lInf); + runTensorFlowNet("fp16_deconvolution", targetId, false, l1, lInf); + runTensorFlowNet("fp16_max_pool_odd_same", targetId, false, l1, lInf); + runTensorFlowNet("fp16_padding_valid", targetId, false, l1, lInf); + runTensorFlowNet("fp16_eltwise_add_mul", targetId, false, l1, lInf); + runTensorFlowNet("fp16_max_pool_odd_valid", targetId, false, l1, lInf); + runTensorFlowNet("fp16_pad_and_concat", targetId, false, l1, lInf); + runTensorFlowNet("fp16_max_pool_even", targetId, false, l1, lInf); + runTensorFlowNet("fp16_padding_same", targetId, false, l1, lInf); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_fp16, + Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16)); + TEST(Test_TensorFlow, defun) { runTensorFlowNet("defun_dropout"); } -TEST(Test_TensorFlow, fp16) -{ - const float l1 = 1e-3; - const float lInf = 1e-2; - runTensorFlowNet("fp16_single_conv", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_deconvolution", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_max_pool_odd_same", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_padding_valid", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_eltwise_add_mul", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_max_pool_odd_valid", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_pad_and_concat", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_max_pool_even", DNN_TARGET_CPU, false, l1, lInf); - runTensorFlowNet("fp16_padding_same", DNN_TARGET_CPU, false, l1, lInf); -} - TEST(Test_TensorFlow, quantized) { runTensorFlowNet("uint8_single_conv");