Merge 59b2c07fdb into 9e18169959

CMakeLists.txt

@@ -268,6 +268,9 @@ OCV_OPTION(WITH_FFMPEG "Include FFMPEG support" (NOT ANDROID)
 OCV_OPTION(WITH_GSTREAMER "Include Gstreamer support" ON
   VISIBLE_IF NOT ANDROID AND NOT IOS AND NOT XROS AND NOT WINRT
   VERIFY HAVE_GSTREAMER AND GSTREAMER_VERSION VERSION_GREATER "0.99")
+OCV_OPTION(WITH_LIBCAMERA "Include Libcamera support" ON
+  VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID
+  VERIFY HAVE_LIBCAMERA OR HAVE_VIDEOIO)
 OCV_OPTION(WITH_GTK "Include GTK support" ON
   VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID
   VERIFY HAVE_GTK)
@@ -1720,6 +1723,13 @@ if(ANDROID)
   status("   NDK Camera:"       HAVE_ANDROID_NATIVE_CAMERA THEN "YES"                         ELSE NO)
 endif()
 
+if(WITH_LIBCAMERA OR HAVE_LIBCAMERA)
+  ocv_build_features_string(libcamera_status
+    IF HAVE_LIBCAMERA THEN "libcamera/libcamera/libcamera.h"
+    ELSE "NO")
+  status("    libcamera:" HAVE_LIBCAMERA THEN YES ELSE NO)
+endif()
+
 # Order is similar to CV_PARALLEL_FRAMEWORK in core/src/parallel.cpp
 ocv_build_features_string(parallel_status EXCLUSIVE
   IF HAVE_TBB THEN "TBB (ver ${TBB_VERSION_MAJOR}.${TBB_VERSION_MINOR} interface ${TBB_INTERFACE_VERSION})"

modules/videoio/CMakeLists.txt

@@ -108,6 +108,11 @@ if(TARGET ocv.3rdparty.dshow)
   list(APPEND tgts ocv.3rdparty.dshow)
 endif()
 
+if(TARGET ocv.3rdparty.libcamera)
+  list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_libcamera.cpp)
+  list(APPEND tgts ocv.3rdparty.libcamera)
+endif()
+
 if(TARGET ocv.3rdparty.msmf)
   if("msmf" IN_LIST VIDEOIO_PLUGIN_LIST OR VIDEOIO_PLUGIN_LIST STREQUAL "all")
     ocv_create_builtin_videoio_plugin("opencv_videoio_msmf" ocv.3rdparty.msmf "cap_msmf.cpp")

modules/videoio/cmake/detect_libcamera.cmake

@@ -0,0 +1,19 @@
+# --- Libcamera ---
+
+if(NOT HAVE_LIBCAMERA AND PKG_CONFIG_FOUND)
+  ocv_check_modules(LIBCAMERA libcamera)
+  if(LIBCAMERA_FOUND)
+    set(HAVE_LIBCAMERA TRUE)
+  endif()
+endif()
+
+if(HAVE_LIBCAMERA)
+  if((CMAKE_CXX_STANDARD EQUAL 98) OR (CMAKE_CXX_STANDARD LESS 17))
+    message(STATUS "CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} is too old to support libcamera. Use C++17 or later. Turning HAVE_LIBCAMERA off")
+    set(HAVE_LIBCAMERA FALSE)
+  endif()
+endif()
+
+if(HAVE_LIBCAMERA)
+  ocv_add_external_target(libcamera "${LIBCAMERA_INCLUDE_DIRS}" "${LIBCAMERA_LINK_LIBRARIES}" "HAVE_LIBCAMERA")
+endif()

modules/videoio/cmake/init.cmake

@@ -11,6 +11,7 @@ endmacro()
 add_backend("ffmpeg" WITH_FFMPEG)
 add_backend("gstreamer" WITH_GSTREAMER)
 add_backend("v4l" WITH_V4L)
+add_backend("libcamera" WITH_LIBCAMERA)
 
 add_backend("aravis" WITH_ARAVIS)
 add_backend("dc1394" WITH_1394)

modules/videoio/include/opencv2/videoio.hpp

@@ -128,7 +128,9 @@ enum VideoCaptureAPIs {
        CAP_INTEL_MFX    = 2300,         //!< Intel MediaSDK
        CAP_XINE         = 2400,         //!< XINE engine (Linux)
        CAP_UEYE         = 2500,         //!< uEye Camera API
-       CAP_OBSENSOR     = 2600,         //!< For Orbbec 3D-Sensor device/module (Astra+, Femto, Astra2, Gemini2, Gemini2L, Gemini2XL, Gemini330, Femto Mega) attention: Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
+       CAP_OBSENSOR     = 2600,         //!< For Orbbec 3D-Sensor device/module (Astra+, Femto, Astra2, Gemini2, Gemini2L, Gemini2XL, Femto Mega) attention: Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions.
+       CAP_LIBCAMERA    = 2700,         //!< Libcamera API
+
      };
 
 

modules/videoio/src/cap_interface.hpp

@@ -402,6 +402,10 @@ Ptr<IVideoWriter> createAndroidVideoWriter(const std::string& filename, int four
 
 Ptr<IVideoCapture> create_obsensor_capture(int index);
 
+Ptr<IVideoCapture> createLibcameraCapture_cam(int index);
+Ptr<IVideoCapture> createLibcameraCapture_file(const std::string &filename);
+
+
 bool VideoCapture_V4L_waitAny(
         const std::vector<VideoCapture>& streams,
         CV_OUT std::vector<int>& ready,

modules/videoio/src/cap_libcamera.hpp

@@ -0,0 +1,396 @@
+#pragma once
+
+#include <fstream>
+#include <iostream>
+
+#include <libcamera/camera.h>
+#include <libcamera/camera_manager.h>
+#include <libcamera/control_ids.h>
+#include <libcamera/property_ids.h>
+#include <libcamera/transform.h>
+// #include <libcamera/libcamera.hpp>
+#include <mutex>
+// #include <queue>
+
+namespace cv
+{
+
+    enum Exposure_Modes
+    {
+        EXPOSURE_NORMAL = libcamera::controls::ExposureNormal,
+        EXPOSURE_SHORT = libcamera::controls::ExposureShort,
+        EXPOSURE_CUSTOM = libcamera::controls::ExposureCustom
+    };
+
+    enum Metering_Modes
+    {
+        METERING_CENTRE = libcamera::controls::MeteringCentreWeighted,
+        METERING_SPOT = libcamera::controls::MeteringSpot,
+        METERING_MATRIX = libcamera::controls::MeteringMatrix,
+        METERING_CUSTOM = libcamera::controls::MeteringCustom
+    };
+
+    enum WhiteBalance_Modes
+    {
+        WB_AUTO = libcamera::controls::AwbAuto,
+        WB_NORMAL = libcamera::controls::AwbAuto,
+        WB_INCANDESCENT = libcamera::controls::AwbIncandescent,
+        WB_TUNGSTEN = libcamera::controls::AwbTungsten,
+        WB_FLUORESCENT = libcamera::controls::AwbFluorescent,
+        WB_INDOOR = libcamera::controls::AwbIndoor,
+        WB_DAYLIGHT = libcamera::controls::AwbDaylight,
+        WB_CLOUDY = libcamera::controls::AwbCloudy,
+        WB_CUSTOM = libcamera::controls::AwbAuto
+    };
+
+    class Options
+    {
+    public:
+        Options()
+        {
+            timeout = 1000;
+            metering_index = Metering_Modes::METERING_CENTRE;
+            exposure_index = Exposure_Modes::EXPOSURE_NORMAL;
+            awb_index = WhiteBalance_Modes::WB_AUTO;
+            saturation = 1.0f;
+            contrast = 1.0f;
+            sharpness = 1.0f;
+            brightness = 0.0f;
+            shutter = 0.0f;
+            gain = 0.0f;
+            ev = 0.0f;
+            roi_x = roi_y = roi_width = roi_height = 0;
+            awb_gain_r = awb_gain_b = 0;
+            denoise = "auto";
+            verbose = false;
+            transform = libcamera::Transform::Identity;
+            camera = 0;
+        }
+
+        ~Options() {}
+
+        void setMetering(Metering_Modes meteringmode) { metering_index = meteringmode; }
+        void setWhiteBalance(WhiteBalance_Modes wb) { awb_index = wb; }
+        void setExposureMode(Exposure_Modes exp) { exposure_index = exp; }
+
+        int getExposureMode() { return exposure_index; }
+        int getMeteringMode() { return metering_index; }
+        int getWhiteBalance() { return awb_index; }
+
+        bool help;
+        bool version;
+        bool list_cameras;
+        bool verbose;
+        uint64_t timeout; // in ms
+        unsigned int photo_width, photo_height;
+        unsigned int video_width, video_height;
+        bool rawfull;
+        libcamera::Transform transform;
+        float roi_x, roi_y, roi_width, roi_height;
+        float shutter;
+        float gain;
+        float ev;
+        float awb_gain_r;
+        float awb_gain_b;
+        float brightness;
+        float contrast;
+        float saturation;
+        float sharpness;
+        float framerate;
+        std::string denoise;
+        std::string info_text;
+        unsigned int camera;
+
+    protected:
+        int metering_index;
+        int exposure_index;
+        int awb_index;
+
+    private:
+    };
+    struct CompletedRequest;
+    using CompletedRequestPtr = std::shared_ptr<CompletedRequest>;
+
+    namespace controls = libcamera::controls;
+    namespace properties = libcamera::properties;
+
+    class LibcameraApp
+    {
+    public:
+        using Stream = libcamera::Stream;
+        using FrameBuffer = libcamera::FrameBuffer;
+        using ControlList = libcamera::ControlList;
+        using Request = libcamera::Request;
+        using CameraManager = libcamera::CameraManager;
+        using Camera = libcamera::Camera;
+        using CameraConfiguration = libcamera::CameraConfiguration;
+        using FrameBufferAllocator = libcamera::FrameBufferAllocator;
+        using StreamRole = libcamera::StreamRole;
+        using StreamRoles = std::vector<libcamera::StreamRole>;
+        using PixelFormat = libcamera::PixelFormat;
+        using StreamConfiguration = libcamera::StreamConfiguration;
+        using BufferMap = Request::BufferMap;
+        using Size = libcamera::Size;
+        using Rectangle = libcamera::Rectangle;
+        enum class MsgType
+        {
+            RequestComplete,
+            Quit
+        };
+        typedef std::variant<CompletedRequestPtr> MsgPayload;
+        struct Msg
+        {
+            Msg(MsgType const &t) : type(t) {}
+            template <typename T>
+            Msg(MsgType const &t, T p) : type(t), payload(std::forward<T>(p))
+            {
+            }
+            MsgType type;
+            MsgPayload payload;
+        };
+
+        // Some flags that can be used to give hints to the camera configuration.
+        static constexpr unsigned int FLAG_STILL_NONE = 0;
+        static constexpr unsigned int FLAG_STILL_BGR = 1;            // supply BGR images, not YUV
+        static constexpr unsigned int FLAG_STILL_RGB = 2;            // supply RGB images, not YUV
+        static constexpr unsigned int FLAG_STILL_RAW = 4;            // request raw image stream
+        static constexpr unsigned int FLAG_STILL_DOUBLE_BUFFER = 8;  // double-buffer stream
+        static constexpr unsigned int FLAG_STILL_TRIPLE_BUFFER = 16; // triple-buffer stream
+        static constexpr unsigned int FLAG_STILL_BUFFER_MASK = 24;   // mask for buffer flags
+
+        static constexpr unsigned int FLAG_VIDEO_NONE = 0;
+        static constexpr unsigned int FLAG_VIDEO_RAW = 1;              // request raw image stream
+        static constexpr unsigned int FLAG_VIDEO_JPEG_COLOURSPACE = 2; // force JPEG colour space
+
+        LibcameraApp(std::unique_ptr<Options> const opts = nullptr);
+        virtual ~LibcameraApp();
+
+        Options *GetOptions() const { return options_.get(); }
+
+        std::string const &CameraId() const;
+        void OpenCamera();
+        void CloseCamera();
+
+        void ConfigureStill(unsigned int flags = FLAG_STILL_NONE);
+        void ConfigureViewfinder();
+
+        void Teardown();
+        void StartCamera();
+        void StopCamera();
+
+        void ApplyRoiSettings();
+
+        Msg Wait();
+        void PostMessage(MsgType &t, MsgPayload &p);
+
+        Stream *GetStream(std::string const &name, unsigned int *w = nullptr, unsigned int *h = nullptr,
+                          unsigned int *stride = nullptr) const;
+        Stream *ViewfinderStream(unsigned int *w = nullptr, unsigned int *h = nullptr,
+                                 unsigned int *stride = nullptr) const;
+        Stream *StillStream(unsigned int *w = nullptr, unsigned int *h = nullptr, unsigned int *stride = nullptr) const;
+        Stream *RawStream(unsigned int *w = nullptr, unsigned int *h = nullptr, unsigned int *stride = nullptr) const;
+        Stream *VideoStream(unsigned int *w = nullptr, unsigned int *h = nullptr, unsigned int *stride = nullptr) const;
+        Stream *LoresStream(unsigned int *w = nullptr, unsigned int *h = nullptr, unsigned int *stride = nullptr) const;
+        Stream *GetMainStream() const;
+
+        std::vector<libcamera::Span<uint8_t>> Mmap(FrameBuffer *buffer) const;
+
+        void SetControls(ControlList &controls);
+        void StreamDimensions(Stream const *stream, unsigned int *w, unsigned int *h, unsigned int *stride) const;
+
+    protected:
+        std::unique_ptr<Options> options_;
+
+    private:
+        static std::shared_ptr<CameraManager> getCameraManager()
+        {
+            static std::shared_ptr<CameraManager> camera_manager_;
+            if (!camera_manager_)
+            {
+                std::cerr << "creating manager" << std::endl;
+                camera_manager_ = std::make_shared<CameraManager>();
+                int ret = camera_manager_->start();
+                if (ret)
+                    throw std::runtime_error("camera manager failed to start,"
+                                             "code " +
+                                             std::to_string(-ret));
+            }
+
+            return camera_manager_;
+        }
+
+        template <typename T>
+        class MessageQueue
+        {
+        public:
+            template <typename U>
+            void Post(U &&msg)
+            {
+                std::unique_lock<std::mutex> lock(mutex_);
+                queue_.push(std::forward<U>(msg));
+                cond_.notify_one();
+            }
+            T Wait()
+            {
+                std::unique_lock<std::mutex> lock(mutex_);
+                cond_.wait(lock, [this]
+                           { return !queue_.empty(); });
+                T msg = std::move(queue_.front());
+                queue_.pop();
+                return msg;
+            }
+            void Clear()
+            {
+                std::unique_lock<std::mutex> lock(mutex_);
+                queue_ = {};
+            }
+
+        private:
+            std::queue<T> queue_;
+            std::mutex mutex_;
+            std::condition_variable cond_;
+        };
+
+        void setupCapture();
+        void makeRequests();
+        void queueRequest(CompletedRequest *completed_request);
+        void requestComplete(Request *request);
+        void configureDenoise(const std::string &denoise_mode);
+
+        // std::unique_ptr<CameraManager> camera_manager_;
+        std::shared_ptr<Camera> camera_;
+        bool camera_acquired_ = false;
+        std::unique_ptr<CameraConfiguration> configuration_;
+        std::map<FrameBuffer *, std::vector<libcamera::Span<uint8_t>>> mapped_buffers_;
+        std::map<std::string, Stream *> streams_;
+        FrameBufferAllocator *allocator_ = nullptr;
+        std::map<Stream *, std::queue<FrameBuffer *>> frame_buffers_;
+        std::queue<Request *> free_requests_;
+        std::vector<std::unique_ptr<Request>> requests_;
+        std::mutex completed_requests_mutex_;
+        std::set<CompletedRequest *> completed_requests_;
+        bool camera_started_ = false;
+        std::mutex camera_stop_mutex_;
+        MessageQueue<Msg> msg_queue_;
+        // For setting camera controls.
+        std::mutex control_mutex_;
+        ControlList controls_;
+        // Other:
+        uint64_t last_timestamp_;
+        uint64_t sequence_ = 0;
+    };
+
+    class Metadata
+    {
+    public:
+        Metadata() = default;
+
+        Metadata(Metadata const &other)
+        {
+            std::scoped_lock other_lock(other.mutex_);
+            data_ = other.data_;
+        }
+
+        Metadata(Metadata &&other)
+        {
+            std::scoped_lock other_lock(other.mutex_);
+            data_ = std::move(other.data_);
+            other.data_.clear();
+        }
+
+        template <typename T>
+        void Set(std::string const &tag, T &&value)
+        {
+            std::scoped_lock lock(mutex_);
+            data_.insert_or_assign(tag, std::forward<T>(value));
+        }
+
+        template <typename T>
+        int Get(std::string const &tag, T &value) const
+        {
+            std::scoped_lock lock(mutex_);
+            auto it = data_.find(tag);
+            if (it == data_.end())
+                return -1;
+            value = std::any_cast<T>(it->second);
+            return 0;
+        }
+
+        void Clear()
+        {
+            std::scoped_lock lock(mutex_);
+            data_.clear();
+        }
+
+        Metadata &operator=(Metadata const &other)
+        {
+            std::scoped_lock lock(mutex_, other.mutex_);
+            data_ = other.data_;
+            return *this;
+        }
+
+        Metadata &operator=(Metadata &&other)
+        {
+            std::scoped_lock lock(mutex_, other.mutex_);
+            data_ = std::move(other.data_);
+            other.data_.clear();
+            return *this;
+        }
+
+        void Merge(Metadata &other)
+        {
+            std::scoped_lock lock(mutex_, other.mutex_);
+            data_.merge(other.data_);
+        }
+
+        template <typename T>
+        T *GetLocked(std::string const &tag)
+        {
+            // This allows in-place access to the Metadata contents,
+            // for which you should be holding the lock.
+            auto it = data_.find(tag);
+            if (it == data_.end())
+                return nullptr;
+            return std::any_cast<T>(&it->second);
+        }
+
+        template <typename T>
+        void SetLocked(std::string const &tag, T &&value)
+        {
+            // Use this only if you're holding the lock yourself.
+            data_.insert_or_assign(tag, std::forward<T>(value));
+        }
+
+        // Note: use of (lowercase) lock and unlock means you can create scoped
+        // locks with the standard lock classes.
+        // e.g. std::lock_guard<RPiController::Metadata> lock(metadata)
+        void lock() { mutex_.lock(); }
+        void unlock() { mutex_.unlock(); }
+
+    private:
+        mutable std::mutex mutex_;
+        std::map<std::string, std::any> data_;
+    };
+
+    struct CompletedRequest
+    {
+        using BufferMap = libcamera::Request::BufferMap;
+        using ControlList = libcamera::ControlList;
+        using Request = libcamera::Request;
+
+        CompletedRequest(unsigned int seq, Request *r)
+            : sequence(seq), buffers(r->buffers()), metadata(r->metadata()), request(r)
+        {
+            r->reuse();
+        }
+        unsigned int sequence;
+        BufferMap buffers;
+        ControlList metadata;
+        Request *request;
+        float framerate;
+        Metadata post_process_metadata;
+    };
+
+    class LibcameraCapture;
+
+};

modules/videoio/src/videoio_registry.cpp

@@ -105,6 +105,10 @@ static const struct VideoBackendInfo builtin_backends[] =
     DECLARE_STATIC_BACKEND(CAP_DSHOW, "DSHOW", MODE_CAPTURE_BY_INDEX, 0, create_DShow_capture, 0)
 #endif
 
+#ifdef HAVE_LIBCAMERA
+    DECLARE_STATIC_BACKEND(CAP_LIBCAMERA, "LIBCAMERA", MODE_CAPTURE_ALL, createLibcameraCapture_file, createLibcameraCapture_cam, 0)
+#endif
+
     // Linux, some Unix
 #if defined HAVE_CAMV4L2
     DECLARE_STATIC_BACKEND(CAP_V4L2, "V4L2", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0)