Merge pull request #24877 from zzuliys:feature/mac

Orbbec Camera supports MacOS,Gemini2 and Gemini2L support Y16 format #24877

note:
1.Gemini2 and Gemini2L must use the latest firmware -- https://github.com/orbbec/OrbbecFirmware;
2.Administrator privileges are necessary to run on MacOS.
This commit is contained in:
zzuliys 2024-03-29 16:23:41 +08:00 committed by GitHub
parent 01dc010436
commit 2b9d1a2ff8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 357 additions and 60 deletions

18
3rdparty/orbbecsdk/orbbecsdk.cmake vendored Normal file
View File

@ -0,0 +1,18 @@
function(download_orbbec_sdk root_var)
set(ORBBECSDK_DOWNLOAD_DIR "${OpenCV_BINARY_DIR}/3rdparty/orbbecsdk")
set(ORBBECSDK_FILE_HASH_CMAKE "2624c84837d3416fd8b3e95750e6e725")
ocv_download(FILENAME "v1.9.4.tar.gz"
HASH ${ORBBECSDK_FILE_HASH_CMAKE}
URL "https://github.com/orbbec/OrbbecSDK/archive/refs/tags/v1.9.4/"
DESTINATION_DIR ${ORBBECSDK_DOWNLOAD_DIR}
ID OrbbecSDK
STATUS res
UNPACK RELATIVE_URL
)
if(${res})
message(STATUS "orbbec sdk downloaded to: ${ORBBECSDK_DOWNLOAD_DIR}")
set(${root_var} "${ORBBECSDK_DOWNLOAD_DIR}/OrbbecSDK-1.9.4" PARENT_SCOPE)
else()
message(FATAL_ERROR "Failed to download orbbec sdk")
endif()
endfunction()

View File

@ -478,9 +478,11 @@ OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF
OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF
VISIBLE_IF TRUE
VERIFY HAVE_TIMVX)
# attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
OCV_OPTION(WITH_OBSENSOR "Include obsensor support (Orbbec RGB-D modules: Astra+/Femto)" ON
VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT AND NOT MINGW) OR ( UNIX AND NOT APPLE AND NOT ANDROID)
# Attention when OBSENSOR_USE_ORBBEC_SDK set to off:
# Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
OCV_OPTION(OBSENSOR_USE_ORBBEC_SDK "Use Orbbec SDK as backend to support more camera models and platforms (force to ON on MacOS)" OFF)
OCV_OPTION(WITH_OBSENSOR "Include obsensor support (Orbbec 3D Cameras)" ON
VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT AND NOT MINGW) OR ( UNIX AND NOT APPLE AND NOT ANDROID) OR (APPLE AND AARCH64 AND NOT IOS)
VERIFY HAVE_OBSENSOR)
OCV_OPTION(WITH_CANN "Include CANN support" OFF
VISIBLE_IF TRUE

View File

@ -236,6 +236,11 @@ if(TARGET ocv.3rdparty.android_native_camera)
endif()
if(TARGET ocv.3rdparty.obsensor)
if(OBSENSOR_USE_ORBBEC_SDK)
list(APPEND videoio_srcs
${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_liborbbec.hpp
${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_liborbbec.cpp)
else()
list(APPEND videoio_srcs
${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_capture.cpp)
list(APPEND videoio_hdrs
@ -256,6 +261,7 @@ if(TARGET ocv.3rdparty.obsensor)
${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.hpp
${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_v4l2.hpp)
endif()
endif()
list(APPEND tgts ocv.3rdparty.obsensor)
endif()

View File

@ -1,5 +1,30 @@
# --- obsensor ---
if(NOT HAVE_OBSENSOR)
if(APPLE)
# force to use orbbec sdk on mac
set(OBSENSOR_USE_ORBBEC_SDK ON)
endif()
if(OBSENSOR_USE_ORBBEC_SDK)
include(${CMAKE_SOURCE_DIR}/3rdparty/orbbecsdk/orbbecsdk.cmake)
download_orbbec_sdk(ORBBEC_SDK_ROOT_DIR)
message(STATUS "ORBBEC_SDK_ROOT_DIR: ${ORBBEC_SDK_ROOT_DIR}")
if(ORBBEC_SDK_ROOT_DIR)
set(OrbbecSDK_DIR "${ORBBEC_SDK_ROOT_DIR}")
find_package(OrbbecSDK REQUIRED)
message(STATUS "OrbbecSDK_FOUND: ${OrbbecSDK_FOUND}")
message(STATUS "OrbbecSDK_INCLUDE_DIRS: ${OrbbecSDK_INCLUDE_DIRS}")
if(OrbbecSDK_FOUND)
set(HAVE_OBSENSOR TRUE)
set(HAVE_OBSENSOR_ORBBEC_SDK TRUE)
ocv_add_external_target(obsensor "${OrbbecSDK_INCLUDE_DIRS}" "${OrbbecSDK_LIBRARY}" "HAVE_OBSENSOR;HAVE_OBSENSOR_ORBBEC_SDK")
file(COPY ${OrbbecSDK_DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/bin)
file(COPY ${OrbbecSDK_DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/lib)
install(FILES ${OrbbecSDK_DLL_FILES} DESTINATION ${OPENCV_LIB_INSTALL_PATH})
ocv_install_3rdparty_licenses(OrbbecSDK ${OrbbecSDK_DIR}/LICENSE.txt)
endif()
endif()
else()
if(WIN32)
check_include_file(mfapi.h HAVE_MFAPI)
check_include_file(vidcap.h HAVE_VIDCAP)
@ -25,4 +50,5 @@ if(NOT HAVE_OBSENSOR)
ocv_add_external_target(obsensor "" "" "HAVE_OBSENSOR;HAVE_OBSENSOR_V4L2")
endif()
endif()
endif()
endif()

View File

@ -128,7 +128,7 @@ enum VideoCaptureAPIs {
CAP_INTEL_MFX = 2300, //!< Intel MediaSDK
CAP_XINE = 2400, //!< XINE engine (Linux)
CAP_UEYE = 2500, //!< uEye Camera API
CAP_OBSENSOR = 2600, //!< For Orbbec 3D-Sensor device/module (Astra+, Femto, Astra2, Gemini2, Gemini2L, Gemini2XL, Femto Mega) attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
CAP_OBSENSOR = 2600, //!< For Orbbec 3D-Sensor device/module (Astra+, Femto, Astra2, Gemini2, Gemini2L, Gemini2XL, Femto Mega) attention: Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
};

View File

@ -316,14 +316,14 @@ bool IUvcStreamChannel::getProperty(int propId, uint8_t* recvData, uint32_t* rec
if(OBSENSOR_GEMINI2_PID == devInfo_.pid){
// return default param
CameraParam param;
param.p0[0] = 516.652f;
param.p0[1] = 516.692f;
param.p0[2] = 322.988f;
param.p0[3] = 235.787f;
param.p1[0] = 516.652f;
param.p1[1] = 516.692f;
param.p1[2] = 322.988f;
param.p1[3] = 235.787f;
param.p0[0] = 519.342f;
param.p0[1] = 519.043f;
param.p0[2] = 319.41f;
param.p0[3] = 240.839f;
param.p1[0] = 519.342f;
param.p1[1] = 519.043f;
param.p1[2] = 319.41f;
param.p1[3] = 240.839f;
param.p6[0] = 640;
param.p6[1] = 480;
param.p7[0] = 640;
@ -421,7 +421,7 @@ bool IUvcStreamChannel::getProperty(int propId, uint8_t* recvData, uint32_t* rec
bool IUvcStreamChannel::initDepthFrameProcessor()
{
if(OBSENSOR_GEMINI2_PID == devInfo_.pid || OBSENSOR_ASTRA2_PID == devInfo_.pid || OBSENSOR_GEMINI2L_PID == devInfo_.pid){
if( OBSENSOR_ASTRA2_PID == devInfo_.pid){
uint8_t* rcvData;
uint32_t rcvLen;
@ -433,7 +433,19 @@ bool IUvcStreamChannel::initDepthFrameProcessor()
depthFrameProcessor_ = makePtr<DepthFrameUnpacker>();
return true;
}else if(OBSENSOR_GEMINI2XL_PID == devInfo_.pid){
}
else if(OBSENSOR_GEMINI2_PID == devInfo_.pid || OBSENSOR_GEMINI2L_PID == devInfo_.pid){
uint8_t* rcvData;
uint32_t rcvLen;
setXu(2, OB_EXT_CMD7, sizeof(OB_EXT_CMD7));
getXu(2, &rcvData, &rcvLen);
setXu(2, OB_EXT_CMD9, sizeof(OB_EXT_CMD9));
getXu(2, &rcvData, &rcvLen);
return true;
}
else if(OBSENSOR_GEMINI2XL_PID == devInfo_.pid){
uint8_t* rcvData;
uint32_t rcvLen;
@ -445,7 +457,7 @@ bool IUvcStreamChannel::initDepthFrameProcessor()
return true;
}
else if (streamType_ == OBSENSOR_STREAM_DEPTH && setXu(2, OB_EXT_CMD4, sizeof(OB_EXT_CMD4)))
else if(streamType_ == OBSENSOR_STREAM_DEPTH && setXu(2, OB_EXT_CMD4, sizeof(OB_EXT_CMD4)))
{
uint8_t* rcvData;
uint32_t rcvLen;

View File

@ -23,7 +23,8 @@
#include "cap_obsensor_capture.hpp"
#include "cap_obsensor/obsensor_stream_channel_interface.hpp"
#ifdef HAVE_OBSENSOR
#if defined(HAVE_OBSENSOR) && !defined(HAVE_OBSENSOR_ORBBEC_SDK)
namespace cv {
Ptr<IVideoCapture> create_obsensor_capture(int index)
{
@ -34,13 +35,13 @@ VideoCapture_obsensor::VideoCapture_obsensor(int index) : isOpened_(false)
{
static const obsensor::StreamProfile colorProfile = { 640, 480, 30, obsensor::FRAME_FORMAT_MJPG };
static const obsensor::StreamProfile depthProfile = {640, 480, 30, obsensor::FRAME_FORMAT_Y16};
static const obsensor::StreamProfile gemini2DepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y14};
static const obsensor::StreamProfile gemini2DepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y16};
static const obsensor::StreamProfile astra2ColorProfile = {800, 600, 30, obsensor::FRAME_FORMAT_MJPG};
static const obsensor::StreamProfile astra2DepthProfile = {800, 600, 30, obsensor::FRAME_FORMAT_Y14};
static const obsensor::StreamProfile megaColorProfile = {1280, 720, 30, obsensor::FRAME_FORMAT_MJPG};
static const obsensor::StreamProfile megaDepthProfile = {640, 576, 30, obsensor::FRAME_FORMAT_Y16};
static const obsensor::StreamProfile gemini2lColorProfile = { 1280, 720, 30, obsensor::FRAME_FORMAT_MJPG};
static const obsensor::StreamProfile gemini2lDepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y14};
static const obsensor::StreamProfile gemini2lDepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y16};
static const obsensor::StreamProfile gemini2XlColorProfile = { 1280, 800, 10, obsensor::FRAME_FORMAT_MJPG};
static const obsensor::StreamProfile gemini2XlDepthProfile = {1280, 800, 10, obsensor::FRAME_FORMAT_Y16};
@ -143,19 +144,22 @@ bool VideoCapture_obsensor::retrieveFrame(int outputType, OutputArray frame)
if (!grabbedDepthFrame_.empty())
{
if(OBSENSOR_GEMINI2_PID == streamChannelGroup_.front()->getPid()){
grabbedDepthFrame_ = grabbedDepthFrame_*0.8;
const double DepthValueScaleGemini2 = 0.2;
grabbedDepthFrame_ = grabbedDepthFrame_*DepthValueScaleGemini2;
Rect rect(320, 160, 640, 480);
grabbedDepthFrame_(rect).copyTo(frame);
}
else if(OBSENSOR_ASTRA2_PID == streamChannelGroup_.front()->getPid()){
grabbedDepthFrame_ = grabbedDepthFrame_*0.8;
const double DepthValueScaleAstra2 = 0.8;
grabbedDepthFrame_ = grabbedDepthFrame_*DepthValueScaleAstra2;
grabbedDepthFrame_.copyTo(frame);
}
else if(OBSENSOR_FEMTO_MEGA_PID == streamChannelGroup_.front()->getPid()){
Rect rect(0, 0, 640, 360);
grabbedDepthFrame_(rect).copyTo(frame);
}else if(OBSENSOR_GEMINI2L_PID == streamChannelGroup_.front()->getPid()){
grabbedDepthFrame_ = grabbedDepthFrame_*0.8;
const double DepthValueScaleGemini2L = 0.2;
grabbedDepthFrame_ = grabbedDepthFrame_*DepthValueScaleGemini2L;
Rect rect(0, 40, 1280, 720);
grabbedDepthFrame_(rect).copyTo(frame);
}else if(OBSENSOR_GEMINI2XL_PID == streamChannelGroup_.front()->getPid()){

View File

@ -28,7 +28,8 @@
#include "cap_obsensor/obsensor_stream_channel_interface.hpp"
#ifdef HAVE_OBSENSOR
#if defined(HAVE_OBSENSOR) && !defined(HAVE_OBSENSOR_ORBBEC_SDK)
namespace cv {
class VideoCapture_obsensor : public IVideoCapture
{

View File

@ -0,0 +1,154 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
/*
* Copyright(C) 2024 by ORBBEC Technology., Inc.
* Authors:
* Huang Zhenchang <yufeng@orbbec.com>
* Yu Shuai <daiyin@orbbec.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "precomp.hpp"
#if defined(HAVE_OBSENSOR) && defined(HAVE_OBSENSOR_ORBBEC_SDK)
#include "libobsensor/ObSensor.hpp"
#include "cap_obsensor_liborbbec.hpp"
namespace cv
{
Ptr<IVideoCapture> create_obsensor_capture(int index)
{
return makePtr<VideoCapture_obsensor>(index);
}
VideoCapture_obsensor::VideoCapture_obsensor(int)
{
ob::Context::setLoggerToFile(OB_LOG_SEVERITY_OFF, "");
config = std::make_shared<ob::Config>();
pipe = std::make_shared<ob::Pipeline>();
auto colorProfiles = pipe->getStreamProfileList(OB_SENSOR_COLOR);
auto colorProfile = colorProfiles->getProfile(OB_PROFILE_DEFAULT);
config->enableStream(colorProfile->as<ob::VideoStreamProfile>());
auto depthProfiles = pipe->getStreamProfileList(OB_SENSOR_DEPTH);
auto depthProfile = depthProfiles->getProfile(OB_PROFILE_DEFAULT);
config->enableStream(depthProfile->as<ob::VideoStreamProfile>());
config->setAlignMode(ALIGN_D2C_SW_MODE);
pipe->start(config, [&](std::shared_ptr<ob::FrameSet> frameset) {
std::unique_lock<std::mutex> lk(videoFrameMutex);
colorFrame = frameset->colorFrame();
depthFrame = frameset->depthFrame();
});
auto param = pipe->getCameraParam();
camParam.p1[0] = param.rgbIntrinsic.fx;
camParam.p1[1] = param.rgbIntrinsic.fy;
camParam.p1[2] = param.rgbIntrinsic.cx;
camParam.p1[3] = param.rgbIntrinsic.cy;
}
VideoCapture_obsensor::~VideoCapture_obsensor(){
pipe->stop();
}
double VideoCapture_obsensor::getProperty(int propIdx) const
{
double rst = 0.0;
propIdx = propIdx & (~CAP_OBSENSOR_GENERATORS_MASK);
switch (propIdx)
{
case CAP_PROP_OBSENSOR_INTRINSIC_FX:
rst = camParam.p1[0];
break;
case CAP_PROP_OBSENSOR_INTRINSIC_FY:
rst = camParam.p1[1];
break;
case CAP_PROP_OBSENSOR_INTRINSIC_CX:
rst = camParam.p1[2];
break;
case CAP_PROP_OBSENSOR_INTRINSIC_CY:
rst = camParam.p1[3];
break;
}
return rst;
}
bool VideoCapture_obsensor::setProperty(int, double)
{
return false;
}
bool VideoCapture_obsensor::grabFrame()
{
std::unique_lock<std::mutex> lk(videoFrameMutex);
grabbedColorFrame = colorFrame;
grabbedDepthFrame = depthFrame;
return grabbedColorFrame || grabbedDepthFrame;
}
bool VideoCapture_obsensor::retrieveFrame(int outputType, cv::OutputArray frame)
{
switch (outputType)
{
case CAP_OBSENSOR_BGR_IMAGE:
if(grabbedColorFrame != nullptr){
auto format = grabbedColorFrame->format();
if(format != OB_FORMAT_MJPEG){
CV_LOG_WARNING(NULL, "Unsupported color frame format");
return false;
}
auto mjpgMat = Mat(1, grabbedColorFrame->dataSize() , CV_8UC1, grabbedColorFrame->data()).clone();
auto bgrMat = imdecode(mjpgMat, IMREAD_COLOR);
if(bgrMat.empty()){
CV_LOG_WARNING(NULL, "Failed to decode color frame");
return false;
}
bgrMat.copyTo(frame);
return true;
}
break;
case CAP_OBSENSOR_DEPTH_MAP:
if(grabbedDepthFrame != nullptr){
auto format = grabbedDepthFrame->format();
if(format != OB_FORMAT_Y16){
CV_LOG_WARNING(NULL, "Unsupported depth frame format");
return false;
}
Mat(grabbedDepthFrame->height(), grabbedDepthFrame->width(), CV_16UC1, grabbedDepthFrame->data()).copyTo(frame);
return true;
}
break;
default:
return false;
}
return false;
}
int VideoCapture_obsensor::getCaptureDomain()
{
return CAP_OBSENSOR;
}
bool VideoCapture_obsensor::isOpened() const
{
return true;
}
}
#endif

View File

@ -0,0 +1,74 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
/*
* Copyright(C) 2024 by ORBBEC Technology., Inc.
* Authors:
* Huang Zhenchang <yufeng@orbbec.com>
* Yu Shuai <daiyin@orbbec.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef _CAP_LIBORBBEC_HPP_
#define _CAP_LIBORBBEC_HPP_
#if defined(HAVE_OBSENSOR) && defined(HAVE_OBSENSOR_ORBBEC_SDK)
#include <libobsensor/ObSensor.hpp>
#include <mutex>
namespace cv
{
struct CameraParam
{
float p0[4];
float p1[4];
float p2[9];
float p3[3];
float p4[5];
float p5[5];
uint32_t p6[2];
uint32_t p7[2];
};
class VideoCapture_obsensor : public IVideoCapture
{
public:
VideoCapture_obsensor(int index);
virtual ~VideoCapture_obsensor();
virtual double getProperty(int propIdx) const CV_OVERRIDE;
virtual bool setProperty(int propIdx, double propVal) CV_OVERRIDE;
virtual bool grabFrame() CV_OVERRIDE;
virtual bool retrieveFrame(int outputType, OutputArray frame) CV_OVERRIDE;
virtual int getCaptureDomain() CV_OVERRIDE;
virtual bool isOpened() const CV_OVERRIDE;
protected:
std::mutex videoFrameMutex;
std::shared_ptr<ob::VideoFrame> colorFrame;
std::shared_ptr<ob::VideoFrame> depthFrame;
std::shared_ptr<ob::VideoFrame> grabbedColorFrame;
std::shared_ptr<ob::VideoFrame> grabbedDepthFrame;
std::shared_ptr<ob::Pipeline> pipe;
std::shared_ptr<ob::Config> config;
CameraParam camParam;
};
}
#endif
#endif

View File

@ -1,5 +1,5 @@
/**
* attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
* attention: Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
*/
#include <opencv2/videoio.hpp>