Merge pull request #20290 from wjj19950828:add_paddle_humanseg_demo

Add paddle humanseg demo

* fixed onnx resize op bug

* add humanseg demo for PaddlePaddle sample

* update README.md and flake8 format

* update func name

* update README.md for enviroment setup

* update README.md in the way install paddle2onnx

* update README.md

* update README.md

* add paddleseg in requirements.txt

* deal with comments

* replace picture

samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md

@@ -1,6 +1,6 @@
-# Run PaddlePaddle model by OpenCV
+# Run PaddlePaddle model using OpenCV
 
-This tutorial shows how to run PaddlePaddle model by opencv.
+These two demonstrations show how to inference PaddlePaddle model using OpenCV.
 
 ## Environment Setup
 
@@ -10,16 +10,69 @@ pip install paddlehub
 pip install paddle2onnx
 ```
 
-## Run PaddlePaddle model demo
+## 1. Run PaddlePaddle ResNet50 using OpenCV
 
-Run the example code as below,
+### Run PaddlePaddle model demo
+
+Run the code sample as follows:
 
 ```shell
 python paddle_resnet50.py
 ```
 
-there are 3 part of this execution
+There are three parts to the process:
 
-- 1. Export PaddlePaddle ResNet50 model to onnx format;
-- 2. Use `cv2.dnn.readNetFromONNX` load model file;
-- 3. Preprocess image file and do inference.
+1. Export PaddlePaddle ResNet50 model to onnx format.
+2. Use `cv2.dnn.readNetFromONNX` to load the model file.
+3. Preprocess image file and do the inference.
+
+## 2. Run PaddleSeg Portrait Segmentation using OpenCV
+
+### Convert to ONNX Model
+
+#### 1. Get Paddle Inference model
+
+For more details, please refer to [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/contrib/HumanSeg/README.md).
+
+```shell
+wget https://x2paddle.bj.bcebos.com/inference/models/humanseg_hrnet18_small_v1.zip
+unzip humanseg_hrnet18_small_v1.zip
+```
+
+Notes:
+
+* The exported model must have a fixed input shape, as dynamic is not supported at this moment.
+
+#### 2. Convert to ONNX model using paddle2onnx
+
+To convert the model, use the following command:
+
+```
+paddle2onnx --model_dir humanseg_hrnet18_small_v1 \
+            --model_filename model.pdmodel \
+            --params_filename model.pdiparams \
+            --opset_version 11 \
+            --save_file humanseg_hrnet18_tiny.onnx
+```
+
+The converted model can be found in the current directory by the name `humanseg_hrnet18_tiny.onnx` .
+
+### Run PaddleSeg Portrait Segmentation demo
+
+Run the code sample as follows:
+
+```shell
+python paddle_humanseg.py
+```
+
+There are three parts to the process:
+
+1. Use `cv2.dnn.readNetFromONNX` to load the model file.
+2. Preprocess image file and do inference.
+3. Postprocess image file and visualize.
+
+The resulting file can be found at `data/result_test_human.jpg` .
+
+### Portrait segmentation visualization
+
+<img src="../../../../data/messi5.jpg" width="50%" height="50%"><img src="./data/result_test_human.jpg" width="50%" height="50%">

samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py

@@ -0,0 +1,112 @@
+import os
+import paddlehub.vision.transforms as T
+import numpy as np
+import cv2 as cv
+
+
+def get_color_map_list(num_classes):
+    """
+    Returns the color map for visualizing the segmentation mask,
+    which can support arbitrary number of classes.
+
+    Args:
+        num_classes (int): Number of classes.
+
+    Returns:
+        (list). The color map.
+    """
+
+    num_classes += 1
+    color_map = num_classes * [0, 0, 0]
+    for i in range(0, num_classes):
+        j = 0
+        lab = i
+        while lab:
+            color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j))
+            color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j))
+            color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j))
+            j += 1
+            lab >>= 3
+    color_map = color_map[3:]
+    return color_map
+
+
+def visualize(image, result, save_dir=None, weight=0.6):
+    """
+    Convert predict result to color image, and save added image.
+
+    Args:
+        image (str): The path of origin image.
+        result (np.ndarray): The predict result of image.
+        save_dir (str): The directory for saving visual image. Default: None.
+        weight (float): The image weight of visual image, and the result weight is (1 - weight). Default: 0.6
+
+    Returns:
+        vis_result (np.ndarray): If `save_dir` is None, return the visualized result.
+    """
+
+    color_map = get_color_map_list(256)
+    color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)]
+    color_map = np.array(color_map).astype("uint8")
+    # Use OpenCV LUT for color mapping
+    c1 = cv.LUT(result, color_map[:, 0])
+    c2 = cv.LUT(result, color_map[:, 1])
+    c3 = cv.LUT(result, color_map[:, 2])
+    pseudo_img = np.dstack((c1, c2, c3))
+
+    im = cv.imread(image)
+    vis_result = cv.addWeighted(im, weight, pseudo_img, 1 - weight, 0)
+
+    if save_dir is not None:
+        if not os.path.exists(save_dir):
+            os.makedirs(save_dir)
+        image_name = os.path.split(image)[-1]
+        out_path = os.path.join(save_dir, image_name)
+        cv.imwrite(out_path, vis_result)
+    else:
+        return vis_result
+
+
+def preprocess(image_path):
+    ''' preprocess input image file to np.ndarray
+
+    Args:
+        image_path(str): Path of input image file
+
+    Returns:
+        ProcessedImage(numpy.ndarray): A numpy.ndarray
+                variable which shape is (1, 3, 192, 192)
+    '''
+    transforms = T.Compose([
+        T.Resize((192, 192)),
+        T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
+    ],
+        to_rgb=True)
+    return np.expand_dims(transforms(image_path), axis=0)
+
+
+if __name__ == '__main__':
+    img_path = "../../../../data/messi5.jpg"
+    # load PPSeg Model use cv.dnn
+    net = cv.dnn.readNetFromONNX('humanseg_hrnet18_tiny.onnx')
+    # read and preprocess image file
+    im = preprocess(img_path)
+    # inference
+    net.setInput(im)
+    result = net.forward(['save_infer_model/scale_0.tmp_1'])
+    # post process
+    image = cv.imread(img_path)
+    r, c, _ = image.shape
+    result = np.argmax(result[0], axis=1).astype(np.uint8)
+    result = cv.resize(result[0, :, :],
+                       dsize=(c, r),
+                       interpolation=cv.INTER_NEAREST)
+
+    print("grid_image.shape is: ", result.shape)
+    folder_path = "data"
+    if not os.path.exists(folder_path):
+        os.makedirs(folder_path)
+    file_path = os.path.join(folder_path, '%s.jpg' % "result_test_human")
+    result_color = visualize(img_path, result)
+    cv.imwrite(file_path, result_color)
+    print('%s saved' % file_path)

samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py

@@ -16,15 +16,15 @@ def preprocess(image_path):
                 variable which shape is (1, 3, 224, 224)
     '''
     transforms = T.Compose([
-            T.Resize((256, 256)),
-            T.CenterCrop(224),
-            T.Normalize(mean=[0.485, 0.456, 0.406],
-                        std=[0.229, 0.224, 0.225])],
-            to_rgb=True)
+        T.Resize((256, 256)),
+        T.CenterCrop(224),
+        T.Normalize(mean=[0.485, 0.456, 0.406],
+                    std=[0.229, 0.224, 0.225])],
+        to_rgb=True)
     return np.expand_dims(transforms(image_path), axis=0)
 
 
-def export_onnx_mobilenetv2(save_path):
+def export_onnx_resnet50(save_path):
     ''' export PaddlePaddle model to ONNX format
 
     Args:
@@ -35,7 +35,7 @@ def export_onnx_mobilenetv2(save_path):
     '''
     model = hub.Module(name="resnet50_vd_imagenet_ssld")
     input_spec = paddle.static.InputSpec(
-            [1, 3, 224, 224], "float32", "image")
+        [1, 3, 224, 224], "float32", "image")
     paddle.onnx.export(model, save_path,
                        input_spec=[input_spec],
                        opset_version=10)
@@ -45,9 +45,9 @@ if __name__ == '__main__':
     save_path = './resnet50'
     image_file = './data/cat.jpg'
     labels = open('./data/labels.txt').read().strip().split('\n')
-    model = export_onnx_mobilenetv2(save_path)
+    model = export_onnx_resnet50(save_path)
 
-    # load mobilenetv2 use cv.dnn
+    # load resnet50 use cv.dnn
     net = cv.dnn.readNetFromONNX(save_path + '.onnx')
     # read and preprocess image file
     im = preprocess(image_file)

samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt

@@ -12,3 +12,4 @@ paddlepaddle>=2.0.0
 paddlepaddle-gpu>=2.0.0
 paddlehub>=2.1.0
 paddle2onnx>=0.5.1
+paddleseg>=2.0.0