From dfc61fbfaa03781d4905fbbef8aaf95816c5ee8e Mon Sep 17 00:00:00 2001 From: zzuliys <68427285+zzuliys@users.noreply.github.com> Date: Tue, 19 Dec 2023 23:34:21 +0800 Subject: [PATCH] Merge pull request #24666 from zzuliys:4.x Add support for Orbbec Gemini2 and Gemini2 XL camera #24666 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch --- CMakeLists.txt | 2 +- modules/videoio/include/opencv2/videoio.hpp | 2 +- .../obsensor_stream_channel_interface.hpp | 2 + .../obsensor_uvc_stream_channel.cpp | 68 +++++++++++++++++-- modules/videoio/src/cap_obsensor_capture.cpp | 27 ++++++-- samples/cpp/videocapture_obsensor.cpp | 15 +++- 6 files changed, 101 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 29d28cf8d4..cff19439da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,6 @@ # $ cmake # # ---------------------------------------------------------------------------- - # Disable in-source builds to prevent source tree corruption. if(" ${CMAKE_SOURCE_DIR}" STREQUAL " ${CMAKE_BINARY_DIR}") message(FATAL_ERROR " @@ -471,6 +470,7 @@ OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF VISIBLE_IF TRUE VERIFY HAVE_TIMVX) +# attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. OCV_OPTION(WITH_OBSENSOR "Include obsensor support (Orbbec RGB-D modules: Astra+/Femto)" ON VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT AND NOT MINGW) OR ( UNIX AND NOT APPLE AND NOT ANDROID) VERIFY HAVE_OBSENSOR) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 0e4956a552..18a682079f 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -128,7 +128,7 @@ enum VideoCaptureAPIs { CAP_INTEL_MFX = 2300, //!< Intel MediaSDK CAP_XINE = 2400, //!< XINE engine (Linux) CAP_UEYE = 2500, //!< uEye Camera API - CAP_OBSENSOR = 2600, //!< For Orbbec 3D-Sensor device/module (Astra+, Femto) + CAP_OBSENSOR = 2600, //!< For Orbbec 3D-Sensor device/module (Astra+, Femto, Astra2, Gemini2, Gemini2L, Gemini2XL, Femto Mega) attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. }; diff --git a/modules/videoio/src/cap_obsensor/obsensor_stream_channel_interface.hpp b/modules/videoio/src/cap_obsensor/obsensor_stream_channel_interface.hpp index ff78c5a696..7337452359 100644 --- a/modules/videoio/src/cap_obsensor/obsensor_stream_channel_interface.hpp +++ b/modules/videoio/src/cap_obsensor/obsensor_stream_channel_interface.hpp @@ -37,6 +37,8 @@ namespace obsensor { #define OBSENSOR_ASTRA2_PID 0x0660 // pid of Orbbec Astra 2 Camera #define OBSENSOR_GEMINI2_PID 0x0670 // pid of Orbbec Gemini 2 Camera #define OBSENSOR_FEMTO_MEGA_PID 0x0669 // pid of Orbbec Femto Mega Camera +#define OBSENSOR_GEMINI2L_PID 0x0673 // pid of Orbbec Gemini 2 L Camera +#define OBSENSOR_GEMINI2XL_PID 0x0671 // pid of Orbbec Gemini 2 XL Camera enum StreamType { diff --git a/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp b/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp index 6e8a4f653c..9145a010d2 100644 --- a/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp +++ b/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp @@ -46,6 +46,9 @@ const uint8_t OB_EXT_CMD7[16] = { 0x47, 0x4d, 0x04, 0x00, 0x02, 0x00, 0xfe, 0x12 const uint8_t OB_EXT_CMD8[16] = { 0x47, 0x4d, 0x04, 0x00, 0x02, 0x00, 0xfe, 0x13, 0x3f, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; const uint8_t OB_EXT_CMD9[16] = { 0x47, 0x4d, 0x04, 0x00, 0x02, 0x00, 0xfa, 0x13, 0x4b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; const uint8_t OB_EXT_CMD10[16] = { 0x47, 0x4d, 0x04, 0x00, 0x02, 0x00, 0xfa, 0x13, 0x3f, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00 }; +const uint8_t OB_EXT_CMD11[16] = { 0x47, 0x4d, 0x04, 0x00, 0x02, 0x00, 0xfe, 0x13, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +const uint8_t OB_EXT_CMD12[16] = { 0x47, 0x4d, 0x04, 0x00, 0x02, 0x00, 0xfe, 0x13, 0x3f, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 }; +const uint8_t OB_EXT_CMD13[16] = { 0x47, 0x4d, 0x04, 0x00, 0x02, 0x00, 0xfa, 0x13, 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; #if defined(HAVE_OBSENSOR_V4L2) #define fourCc2Int(a, b, c, d) \ @@ -269,14 +272,22 @@ bool IUvcStreamChannel::setProperty(int propId, const uint8_t* /*data*/, uint32_ rst &= getXu(2, &rcvData, &rcvLen); rst &= setXu(2, OB_EXT_CMD6, sizeof(OB_EXT_CMD6)); rst &= getXu(2, &rcvData, &rcvLen); - } - else if(OBSENSOR_ASTRA2_PID == devInfo_.pid ){ - rst &= setXu(2, OB_EXT_CMD10, sizeof(OB_EXT_CMD8)); + }else if(OBSENSOR_ASTRA2_PID == devInfo_.pid ){ + rst &= setXu(2, OB_EXT_CMD12, sizeof(OB_EXT_CMD12)); rst &= getXu(2, &rcvData, &rcvLen); rst &= setXu(2, OB_EXT_CMD6, sizeof(OB_EXT_CMD6)); rst &= getXu(2, &rcvData, &rcvLen); - } - else{ + }else if(OBSENSOR_GEMINI2L_PID == devInfo_.pid){ + rst &= setXu(2, OB_EXT_CMD11, sizeof(OB_EXT_CMD11)); + rst &= getXu(2, &rcvData, &rcvLen); + rst &= setXu(2, OB_EXT_CMD6, sizeof(OB_EXT_CMD6)); + rst &= getXu(2, &rcvData, &rcvLen); + }else if(OBSENSOR_GEMINI2XL_PID == devInfo_.pid){ + rst &= setXu(2, OB_EXT_CMD11, sizeof(OB_EXT_CMD11)); + rst &= getXu(2, &rcvData, &rcvLen); + rst &= setXu(2, OB_EXT_CMD6, sizeof(OB_EXT_CMD6)); + rst &= getXu(2, &rcvData, &rcvLen); + }else{ rst &= setXu(2, OB_EXT_CMD0, sizeof(OB_EXT_CMD0)); rst &= getXu(2, &rcvData, &rcvLen); rst &= setXu(2, OB_EXT_CMD1, sizeof(OB_EXT_CMD1)); @@ -319,6 +330,40 @@ bool IUvcStreamChannel::getProperty(int propId, uint8_t* recvData, uint32_t* rec param.p7[1] = 480; *recvDataSize = sizeof(CameraParam); memcpy(recvData, ¶m, *recvDataSize); + }else if(OBSENSOR_GEMINI2L_PID == devInfo_.pid){ + // return default param + CameraParam param; + param.p0[0] = 688.87f; + param.p0[1] = 688.922f; + param.p0[2] = 644.317f; + param.p0[3] = 354.382f; + param.p1[0] = 688.87f; + param.p1[1] = 688.922f; + param.p1[2] = 644.317f; + param.p1[3] = 354.382f; + param.p6[0] = 1280; + param.p6[1] = 720; + param.p7[0] = 1280; + param.p7[1] = 720; + *recvDataSize = sizeof(CameraParam); + memcpy(recvData, ¶m, *recvDataSize); + }else if(OBSENSOR_GEMINI2XL_PID == devInfo_.pid){ + // return default param + CameraParam param; + param.p0[0] = 610.847f; + param.p0[1] = 610.829f; + param.p0[2] = 640.647f; + param.p0[3] = 401.817f; + param.p1[0] = 610.847f; + param.p1[1] = 610.829f; + param.p1[2] = 640.647f; + param.p1[3] = 401.817f; + param.p6[0] = 640; + param.p6[1] = 480; + param.p7[0] = 640; + param.p7[1] = 480; + *recvDataSize = sizeof(CameraParam); + memcpy(recvData, ¶m, *recvDataSize); } else if(OBSENSOR_ASTRA2_PID == devInfo_.pid){ // return default param @@ -376,7 +421,7 @@ bool IUvcStreamChannel::getProperty(int propId, uint8_t* recvData, uint32_t* rec bool IUvcStreamChannel::initDepthFrameProcessor() { - if(OBSENSOR_GEMINI2_PID == devInfo_.pid || OBSENSOR_ASTRA2_PID == devInfo_.pid){ + if(OBSENSOR_GEMINI2_PID == devInfo_.pid || OBSENSOR_ASTRA2_PID == devInfo_.pid || OBSENSOR_GEMINI2L_PID == devInfo_.pid){ uint8_t* rcvData; uint32_t rcvLen; @@ -387,6 +432,17 @@ bool IUvcStreamChannel::initDepthFrameProcessor() getXu(2, &rcvData, &rcvLen); depthFrameProcessor_ = makePtr(); + return true; + }else if(OBSENSOR_GEMINI2XL_PID == devInfo_.pid){ + uint8_t* rcvData; + uint32_t rcvLen; + + setXu(2, OB_EXT_CMD7, sizeof(OB_EXT_CMD7)); + getXu(2, &rcvData, &rcvLen); + + setXu(2, OB_EXT_CMD13, sizeof(OB_EXT_CMD13)); + getXu(2, &rcvData, &rcvLen); + return true; } else if (streamType_ == OBSENSOR_STREAM_DEPTH && setXu(2, OB_EXT_CMD4, sizeof(OB_EXT_CMD4))) diff --git a/modules/videoio/src/cap_obsensor_capture.cpp b/modules/videoio/src/cap_obsensor_capture.cpp index 8138f09333..21db152fc3 100644 --- a/modules/videoio/src/cap_obsensor_capture.cpp +++ b/modules/videoio/src/cap_obsensor_capture.cpp @@ -35,9 +35,14 @@ VideoCapture_obsensor::VideoCapture_obsensor(int index) : isOpened_(false) static const obsensor::StreamProfile colorProfile = { 640, 480, 30, obsensor::FRAME_FORMAT_MJPG }; static const obsensor::StreamProfile depthProfile = {640, 480, 30, obsensor::FRAME_FORMAT_Y16}; static const obsensor::StreamProfile gemini2DepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y14}; - static const obsensor::StreamProfile astra2DepthProfile = {640, 480, 30, obsensor::FRAME_FORMAT_Y14}; + static const obsensor::StreamProfile astra2ColorProfile = {800, 600, 30, obsensor::FRAME_FORMAT_MJPG}; + static const obsensor::StreamProfile astra2DepthProfile = {800, 600, 30, obsensor::FRAME_FORMAT_Y14}; static const obsensor::StreamProfile megaColorProfile = {1280, 720, 30, obsensor::FRAME_FORMAT_MJPG}; static const obsensor::StreamProfile megaDepthProfile = {640, 576, 30, obsensor::FRAME_FORMAT_Y16}; + static const obsensor::StreamProfile gemini2lColorProfile = { 1280, 720, 30, obsensor::FRAME_FORMAT_MJPG}; + static const obsensor::StreamProfile gemini2lDepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y14}; + static const obsensor::StreamProfile gemini2XlColorProfile = { 1280, 800, 10, obsensor::FRAME_FORMAT_MJPG}; + static const obsensor::StreamProfile gemini2XlDepthProfile = {1280, 800, 10, obsensor::FRAME_FORMAT_Y16}; streamChannelGroup_ = obsensor::getStreamChannelGroup(index); if (!streamChannelGroup_.empty()) @@ -52,6 +57,12 @@ VideoCapture_obsensor::VideoCapture_obsensor(int index) : isOpened_(false) auto profile = colorProfile; if(OBSENSOR_FEMTO_MEGA_PID == channel->getPid()){ profile = megaColorProfile; + }else if(OBSENSOR_GEMINI2L_PID == channel->getPid()){ + profile = gemini2lColorProfile; + }else if(OBSENSOR_ASTRA2_PID == channel->getPid()){ + profile = astra2ColorProfile; + }else if(OBSENSOR_GEMINI2XL_PID == channel->getPid()){ + profile = gemini2XlColorProfile; } channel->start(profile, [&](obsensor::Frame* frame) { std::unique_lock lk(frameMutex_); @@ -74,8 +85,11 @@ VideoCapture_obsensor::VideoCapture_obsensor(int index) : isOpened_(false) } else if(OBSENSOR_FEMTO_MEGA_PID == channel->getPid()){ profile = megaDepthProfile; + }else if(OBSENSOR_GEMINI2L_PID == channel->getPid()){ + profile = gemini2lDepthProfile; + }else if(OBSENSOR_GEMINI2XL_PID == channel->getPid()){ + profile = gemini2XlDepthProfile; } - channel->start(profile, [&](obsensor::Frame* frame) { std::unique_lock lk(frameMutex_); depthFrame_ = Mat(frame->height, frame->width, CV_16UC1, frame->data, frame->width * 2).clone(); @@ -140,8 +154,13 @@ bool VideoCapture_obsensor::retrieveFrame(int outputType, OutputArray frame) else if(OBSENSOR_FEMTO_MEGA_PID == streamChannelGroup_.front()->getPid()){ Rect rect(0, 0, 640, 360); grabbedDepthFrame_(rect).copyTo(frame); - } - else{ + }else if(OBSENSOR_GEMINI2L_PID == streamChannelGroup_.front()->getPid()){ + grabbedDepthFrame_ = grabbedDepthFrame_*0.8; + Rect rect(0, 40, 1280, 720); + grabbedDepthFrame_(rect).copyTo(frame); + }else if(OBSENSOR_GEMINI2XL_PID == streamChannelGroup_.front()->getPid()){ + grabbedDepthFrame_.copyTo(frame); + }else{ grabbedDepthFrame_.copyTo(frame); } grabbedDepthFrame_.release(); diff --git a/samples/cpp/videocapture_obsensor.cpp b/samples/cpp/videocapture_obsensor.cpp index 98eb9f479f..7dda3386ce 100644 --- a/samples/cpp/videocapture_obsensor.cpp +++ b/samples/cpp/videocapture_obsensor.cpp @@ -1,3 +1,7 @@ +/** + * attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. +*/ + #include #include #include @@ -21,6 +25,11 @@ int main() Mat image; Mat depthMap; Mat adjDepthMap; + + // Minimum depth value + const double minVal = 300; + // Maximum depth value + const double maxVal = 5000; while (true) { // Grab depth map like this: @@ -36,7 +45,7 @@ int main() if (obsensorCapture.retrieve(depthMap, CAP_OBSENSOR_DEPTH_MAP)) { - normalize(depthMap, adjDepthMap, 0, 255, NORM_MINMAX, CV_8UC1); + depthMap.convertTo(adjDepthMap, CV_8U, 255.0 / (maxVal - minVal), -minVal * 255.0 / (maxVal - minVal)); applyColorMap(adjDepthMap, adjDepthMap, COLORMAP_JET); imshow("DEPTH", adjDepthMap); } @@ -45,7 +54,7 @@ int main() static const float alpha = 0.6f; if (!image.empty() && !depthMap.empty()) { - normalize(depthMap, adjDepthMap, 0, 255, NORM_MINMAX, CV_8UC1); + depthMap.convertTo(adjDepthMap, CV_8U, 255.0 / (maxVal - minVal), -minVal * 255.0 / (maxVal - minVal)); cv::resize(adjDepthMap, adjDepthMap, cv::Size(image.cols, image.rows)); for (int i = 0; i < image.rows; i++) { @@ -71,4 +80,4 @@ int main() break; } return 0; -} \ No newline at end of file +}