mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-08-05 14:06:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Consider video meta on GStreamer video capture

Browse Source 
Some GStreamer elements may produce buffers with very non
standard strides, offsets and/or even transport each plane
in different, non-contiguous pointers. This non-standard
layout is communicated via GstVideoMeta structures attached
to the buffers. Given this, when a GstVideoMeta is available,
one should parse the layout from it instead of generating
a generic one from the caps.

The GstVideoFrame utility does precisely this: if the buffer
contains a video meta, it uses that to fill the format and
memory layout. If there is no meta available, the layout is
inferred from the caps.
This commit is contained in:
Michael Gruner 2022-05-22 20:08:38 -06:00
parent 50e8ad285b
commit 6a22c5b2b5
1 changed files with 71 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
modules/videoio/src/cap_gstreamer.cpp
View File

@ -192,6 +192,19 @@ public:
} }
}; };
class ScopeGuardGstVideoFrame
{
GstVideoFrame* frame_;
public:
ScopeGuardGstVideoFrame(GstVideoFrame* frame)
: frame_(frame)
{}
~ScopeGuardGstVideoFrame()
{
gst_video_frame_unmap(frame_);
}
};
} // namespace } // namespace
/*! /*!
@ -651,8 +664,28 @@ bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst)
CV_Error(Error::StsError, "GStreamer: gst_video_info_from_caps() is failed. Can't handle unknown layout"); CV_Error(Error::StsError, "GStreamer: gst_video_info_from_caps() is failed. Can't handle unknown layout");
} }
int frame_width = GST_VIDEO_INFO_WIDTH(&info); // gstreamer expects us to handle the memory at this point
int frame_height = GST_VIDEO_INFO_HEIGHT(&info); // so we can just wrap the raw buffer and be done with it
GstBuffer* buf = gst_sample_get_buffer(sample); // no lifetime transfer
if (!buf)
return false;
// at this point, the gstreamer buffer may contain a video meta with special
// stride and plane locations. We __must__ consider in order to correctly parse
// the data. The gst_video_frame_map will parse the meta for us, or default to
// regular strides/offsets if no meta is present.
GstVideoFrame frame = {};
GstMapFlags flags = static_cast<GstMapFlags>(GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF);
if (!gst_video_frame_map(&frame, &info, buf, flags))
{
CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory");
return false;
}
ScopeGuardGstVideoFrame frame_guard(&frame); // call gst_video_frame_unmap(&frame) on scope leave
int frame_width = GST_VIDEO_FRAME_COMP_WIDTH(&frame, 0);
int frame_height = GST_VIDEO_FRAME_COMP_HEIGHT(&frame, 0);
if (frame_width <= 0 || frame_height <= 0) if (frame_width <= 0 || frame_height <= 0)
{ {
CV_LOG_ERROR(NULL, "GStreamer: Can't query frame size from GStreamer sample"); CV_LOG_ERROR(NULL, "GStreamer: Can't query frame size from GStreamer sample");
@ -674,19 +707,6 @@ bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst)
} }
std::string name = toLowerCase(std::string(name_)); std::string name = toLowerCase(std::string(name_));
// gstreamer expects us to handle the memory at this point
// so we can just wrap the raw buffer and be done with it
GstBuffer* buf = gst_sample_get_buffer(sample); // no lifetime transfer
if (!buf)
return false;
GstMapInfo map_info = {};
if (!gst_buffer_map(buf, &map_info, GST_MAP_READ))
{
CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory");
return false;
}
ScopeGuardGstMapInfo map_guard(buf, &map_info); // call gst_buffer_unmap(buf, &map_info) on scope leave
// we support these types of data: // we support these types of data:
// video/x-raw, format=BGR -> 8bit, 3 channels // video/x-raw, format=BGR -> 8bit, 3 channels
// video/x-raw, format=GRAY8 -> 8bit, 1 channel // video/x-raw, format=GRAY8 -> 8bit, 1 channel
@ -703,10 +723,11 @@ bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst)
// video/x-raw, format={BGRA, RGBA, BGRx, RGBx} -> 8bit, 4 channels // video/x-raw, format={BGRA, RGBA, BGRx, RGBx} -> 8bit, 4 channels
// bayer data is never decoded, the user is responsible for that // bayer data is never decoded, the user is responsible for that
Size sz = Size(frame_width, frame_height); Size sz = Size(frame_width, frame_height);
guint n_planes = GST_VIDEO_INFO_N_PLANES(&info); guint n_planes = GST_VIDEO_FRAME_N_PLANES(&frame);
if (name == "video/x-raw") if (name == "video/x-raw")
{ {
const gchar* format_ = gst_structure_get_string(structure, "format"); const gchar* format_ = frame.info.finfo->name;
if (!format_) if (!format_)
{ {
CV_LOG_ERROR(NULL, "GStreamer: Can't query 'format' of 'video/x-raw'"); CV_LOG_ERROR(NULL, "GStreamer: Can't query 'format' of 'video/x-raw'");
@ -717,97 +738,83 @@ bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst)
if (format == "BGR") if (format == "BGR")
{ {
CV_CheckEQ((int)n_planes, 1, ""); CV_CheckEQ((int)n_planes, 1, "");
size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); size_t step = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0);
CV_CheckGE(step, (size_t)frame_width * 3, ""); CV_CheckGE(step, (size_t)frame_width * 3, "");
Mat src(sz, CV_8UC3, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); Mat src(sz, CV_8UC3, GST_VIDEO_FRAME_PLANE_DATA(&frame, 0), step);
src.copyTo(dst); src.copyTo(dst);
return true; return true;
} }
else if (format == "GRAY8") else if (format == "GRAY8")
{ {
CV_CheckEQ((int)n_planes, 1, ""); CV_CheckEQ((int)n_planes, 1, "");
size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); size_t step = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0);
CV_CheckGE(step, (size_t)frame_width, ""); CV_CheckGE(step, (size_t)frame_width, "");
Mat src(sz, CV_8UC1, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); Mat src(sz, CV_8UC1, GST_VIDEO_FRAME_PLANE_DATA(&frame, 0), step);
src.copyTo(dst); src.copyTo(dst);
return true; return true;
} }
else if (format == "GRAY16_LE" || format == "GRAY16_BE") else if (format == "GRAY16_LE" || format == "GRAY16_BE")
{ {
CV_CheckEQ((int)n_planes, 1, ""); CV_CheckEQ((int)n_planes, 1, "");
size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); size_t step = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0);
CV_CheckGE(step, (size_t)frame_width, ""); CV_CheckGE(step, (size_t)frame_width, "");
Mat src(sz, CV_16UC1, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); Mat src(sz, CV_16UC1, GST_VIDEO_FRAME_PLANE_DATA(&frame, 0), step);
src.copyTo(dst); src.copyTo(dst);
return true; return true;
} }
else if (format == "BGRA" || format == "RGBA" || format == "BGRX" || format == "RGBX") else if (format == "BGRA" || format == "RGBA" || format == "BGRX" || format == "RGBX")
{ {
CV_CheckEQ((int)n_planes, 1, ""); CV_CheckEQ((int)n_planes, 1, "");
size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); size_t step = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0);
CV_CheckGE(step, (size_t)frame_width, ""); CV_CheckGE(step, (size_t)frame_width, "");
Mat src(sz, CV_8UC4, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); Mat src(sz, CV_8UC4, GST_VIDEO_FRAME_PLANE_DATA(&frame, 0), step);
src.copyTo(dst); src.copyTo(dst);
return true; return true;
} }
else if (format == "UYVY" || format == "YUY2" || format == "YVYU") else if (format == "UYVY" || format == "YUY2" || format == "YVYU")
{ {
CV_CheckEQ((int)n_planes, 1, ""); CV_CheckEQ((int)n_planes, 1, "");
size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); size_t step = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0);
CV_CheckGE(step, (size_t)frame_width * 2, ""); CV_CheckGE(step, (size_t)frame_width * 2, "");
Mat src(sz, CV_8UC2, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); Mat src(sz, CV_8UC2, GST_VIDEO_FRAME_PLANE_DATA(&frame, 0), step);
src.copyTo(dst); src.copyTo(dst);
return true; return true;
} }
else if (format == "NV12" || format == "NV21") else if (format == "NV12" || format == "NV21")
{ {
CV_CheckEQ((int)n_planes, 2, ""); CV_CheckEQ((int)n_planes, 2, "");
size_t stepY = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); size_t stepY = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0);
CV_CheckGE(stepY, (size_t)frame_width, ""); CV_CheckGE(stepY, (size_t)frame_width, "");
size_t stepUV = GST_VIDEO_INFO_PLANE_STRIDE(&info, 1); size_t stepUV = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 1);
CV_CheckGE(stepUV, (size_t)frame_width, ""); CV_CheckGE(stepUV, (size_t)frame_width, "");
size_t offsetY = GST_VIDEO_INFO_PLANE_OFFSET(&info, 0);
size_t offsetUV = GST_VIDEO_INFO_PLANE_OFFSET(&info, 1); dst.create(Size(frame_width, frame_height * 3 / 2), CV_8UC1);
if (stepY != stepUV || (offsetUV - offsetY) != (stepY * frame_height)) Mat dst_ = dst.getMat();
{ Mat srcY(sz, CV_8UC1, GST_VIDEO_FRAME_PLANE_DATA(&frame,0), stepY);
dst.create(Size(frame_width, frame_height * 3 / 2), CV_8UC1); Mat srcUV(Size(frame_width, frame_height / 2), CV_8UC1, GST_VIDEO_FRAME_PLANE_DATA(&frame,1), stepUV);
Mat dst_ = dst.getMat(); srcY.copyTo(dst_(Rect(0, 0, frame_width, frame_height)));
Mat srcY(sz, CV_8UC1, map_info.data + offsetY, stepY); srcUV.copyTo(dst_(Rect(0, frame_height, frame_width, frame_height / 2)));
Mat srcUV(Size(frame_width, frame_height / 2), CV_8UC1, map_info.data + offsetUV, stepUV);
srcY.copyTo(dst_(Rect(0, 0, frame_width, frame_height)));
srcUV.copyTo(dst_(Rect(0, frame_height, frame_width, frame_height / 2)));
}
else
{
Mat src(Size(frame_width, frame_height * 3 / 2), CV_8UC1, map_info.data + offsetY, stepY);
src.copyTo(dst);
}
return true; return true;
} }
else if (format == "YV12" || format == "I420") else if (format == "YV12" || format == "I420")
{ {
CV_CheckEQ((int)n_planes, 3, ""); CV_CheckEQ((int)n_planes, 3, "");
size_t step0 = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); size_t step0 = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0);
CV_CheckGE(step0, (size_t)frame_width, ""); CV_CheckGE(step0, (size_t)frame_width, "");
size_t step1 = GST_VIDEO_INFO_PLANE_STRIDE(&info, 1); size_t step1 = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 1);
CV_CheckGE(step1, (size_t)frame_width / 2, ""); CV_CheckGE(step1, (size_t)frame_width / 2, "");
size_t step2 = GST_VIDEO_INFO_PLANE_STRIDE(&info, 2); size_t step2 = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 2);
CV_CheckGE(step2, (size_t)frame_width / 2, ""); CV_CheckGE(step2, (size_t)frame_width / 2, "");
size_t offset0 = GST_VIDEO_INFO_PLANE_OFFSET(&info, 0); dst.create(Size(frame_width, frame_height * 3 / 2), CV_8UC1);
size_t offset1 = GST_VIDEO_INFO_PLANE_OFFSET(&info, 1); Mat dst_ = dst.getMat();
size_t offset2 = GST_VIDEO_INFO_PLANE_OFFSET(&info, 2); Mat srcY(sz, CV_8UC1, GST_VIDEO_FRAME_PLANE_DATA(&frame,0), step0);
{ Size sz2(frame_width / 2, frame_height / 2);
dst.create(Size(frame_width, frame_height * 3 / 2), CV_8UC1); Mat src1(sz2, CV_8UC1, GST_VIDEO_FRAME_PLANE_DATA(&frame,1), step1);
Mat dst_ = dst.getMat(); Mat src2(sz2, CV_8UC1, GST_VIDEO_FRAME_PLANE_DATA(&frame,2), step2);
Mat srcY(sz, CV_8UC1, map_info.data + offset0, step0); srcY.copyTo(dst_(Rect(0, 0, frame_width, frame_height)));
Size sz2(frame_width / 2, frame_height / 2); src1.copyTo(Mat(sz2, CV_8UC1, dst_.ptr<uchar>(frame_height)));
Mat src1(sz2, CV_8UC1, map_info.data + offset1, step1); src2.copyTo(Mat(sz2, CV_8UC1, dst_.ptr<uchar>(frame_height) + src1.total()));
Mat src2(sz2, CV_8UC1, map_info.data + offset2, step2);
srcY.copyTo(dst_(Rect(0, 0, frame_width, frame_height)));
src1.copyTo(Mat(sz2, CV_8UC1, dst_.ptr<uchar>(frame_height)));
src2.copyTo(Mat(sz2, CV_8UC1, dst_.ptr<uchar>(frame_height) + src1.total()));
}
return true; return true;
} }
else else
@ -818,14 +825,14 @@ bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst)
else if (name == "video/x-bayer") else if (name == "video/x-bayer")
{ {
CV_CheckEQ((int)n_planes, 0, ""); CV_CheckEQ((int)n_planes, 0, "");
Mat src = Mat(sz, CV_8UC1, map_info.data); Mat src = Mat(sz, CV_8UC1, frame.map[0].data);
src.copyTo(dst); src.copyTo(dst);
return true; return true;
} }
else if (name == "image/jpeg") else if (name == "image/jpeg")
{ {
CV_CheckEQ((int)n_planes, 0, ""); CV_CheckEQ((int)n_planes, 0, "");
Mat src = Mat(Size(map_info.size, 1), CV_8UC1, map_info.data); Mat src = Mat(Size(frame.map[0].size, 1), CV_8UC1, frame.map[0].data);
src.copyTo(dst); src.copyTo(dst);
return true; return true;
} }