From 767780a4b28d2c75f67f00e5cce9a3a4e13d36df Mon Sep 17 00:00:00 2001 From: yoffy Date: Mon, 9 May 2016 03:50:16 +0900 Subject: [PATCH 1/4] add VideoCapture / VideoWriter AVFoundation implementation for Mac --- CMakeLists.txt | 13 +- cmake/OpenCVFindLibsVideo.cmake | 13 +- cmake/templates/cvconfig.h.in | 3 + modules/videoio/CMakeLists.txt | 4 + modules/videoio/include/opencv2/videoio.hpp | 57 +- .../include/opencv2/videoio/videoio_c.h | 1 + modules/videoio/src/cap.cpp | 16 + modules/videoio/src/cap_avfoundation_mac.mm | 1170 +++++++++++++++++ modules/videoio/src/precomp.hpp | 4 + 9 files changed, 1243 insertions(+), 38 deletions(-) create mode 100644 modules/videoio/src/cap_avfoundation_mac.mm diff --git a/CMakeLists.txt b/CMakeLists.txt index c9568546f7..9681a28810 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,7 +169,8 @@ OCV_OPTION(OPENCV_ENABLE_NONFREE "Enable non-free algorithms" OFF) # Optional 3rd party components # =================================================== OCV_OPTION(WITH_1394 "Include IEEE1394 support" ON IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) -OCV_OPTION(WITH_AVFOUNDATION "Use AVFoundation for Video I/O" ON IF IOS) +OCV_OPTION(WITH_AVFOUNDATION "Use AVFoundation for Video I/O (iOS)" ON IF IOS) +OCV_OPTION(WITH_AVFOUNDATION_MAC "Use AVFoundation for Video I/O (Mac)" ON IF (NOT IOS AND APPLE) ) OCV_OPTION(WITH_CARBON "Use Carbon for UI instead of Cocoa" OFF IF APPLE ) OCV_OPTION(WITH_CAROTENE "Use NVidia carotene acceleration library for ARM platform" ON IF (ARM OR AARCH64) AND NOT IOS AND NOT (CMAKE_VERSION VERSION_LESS "2.8.11")) OCV_OPTION(WITH_VTK "Include VTK library support (and build opencv_viz module eiher)" ON IF (NOT ANDROID AND NOT IOS AND NOT WINRT AND NOT CMAKE_CROSSCOMPILING) ) @@ -1063,10 +1064,6 @@ if(DEFINED WITH_1394) status(" DC1394 2.x:" HAVE_DC1394_2 THEN "YES (ver ${ALIASOF_libdc1394-2_VERSION})" ELSE NO) endif(DEFINED WITH_1394) -if(DEFINED WITH_AVFOUNDATION) - status(" AVFoundation:" WITH_AVFOUNDATION THEN YES ELSE NO) -endif(DEFINED WITH_AVFOUNDATION) - if(DEFINED WITH_FFMPEG) if(WIN32) status(" FFMPEG:" WITH_FFMPEG THEN "YES (prebuilt binaries)" ELSE NO) @@ -1112,10 +1109,12 @@ if(DEFINED WITH_GIGEAPI) status(" GigEVisionSDK:" HAVE_GIGE_API THEN YES ELSE NO) endif(DEFINED WITH_GIGEAPI) -if(DEFINED WITH_QUICKTIME) +if(DEFINED APPLE) status(" QuickTime:" HAVE_QUICKTIME THEN YES ELSE NO) status(" QTKit:" HAVE_QTKIT THEN YES ELSE NO) -endif(DEFINED WITH_QUICKTIME) + status(" AVFoundation:" WITH_AVFOUNDATION THEN YES ELSE NO) + status(" AVFoundationMac:" WITH_AVFOUNDATION_MAC THEN YES ELSE NO) +endif(DEFINED APPLE) if(DEFINED WITH_UNICAP) status(" UniCap:" HAVE_UNICAP THEN "YES (ver ${ALIASOF_libunicap_VERSION})" ELSE NO) diff --git a/cmake/OpenCVFindLibsVideo.cmake b/cmake/OpenCVFindLibsVideo.cmake index 20e31518a4..c23ee06d87 100644 --- a/cmake/OpenCVFindLibsVideo.cmake +++ b/cmake/OpenCVFindLibsVideo.cmake @@ -302,17 +302,24 @@ if(WIN32) endif() endif(WIN32) -# --- Apple AV Foundation --- +# --- Apple AV Foundation (iOS) --- if(WITH_AVFOUNDATION) set(HAVE_AVFOUNDATION YES) endif() -# --- QuickTime --- +# --- Apple AV Foundation (Mac) --- +if(WITH_AVFOUNDATION_MAC) + set(HAVE_AVFOUNDATION_MAC YES) +endif() + +# --- QuickTime, Apple AV Foundation (Mac) --- if (NOT IOS) if(WITH_QUICKTIME) set(HAVE_QUICKTIME YES) - elseif(APPLE AND CMAKE_COMPILER_IS_CLANGCXX) + elseif(WITH_QTKIT) set(HAVE_QTKIT YES) + elseif(APPLE AND CMAKE_COMPILER_IS_CLANGCXX) + set(HAVE_AVFOUNDATION_MAC YES) endif() endif() diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index 56d415b400..50b4185e2c 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -16,6 +16,9 @@ /* AVFoundation video libraries */ #cmakedefine HAVE_AVFOUNDATION +/* AVFoundation (Mac) video libraries */ +#cmakedefine HAVE_AVFOUNDATION_MAC + /* V4L capturing support */ #cmakedefine HAVE_CAMV4L diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 41b0691b75..d75c2d80bb 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -171,6 +171,10 @@ if(HAVE_AVFOUNDATION) list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_avfoundation.mm) list(APPEND VIDEOIO_LIBRARIES "-framework AVFoundation" "-framework QuartzCore") endif() +if(HAVE_AVFOUNDATION_MAC) + list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_avfoundation_mac.mm) + list(APPEND VIDEOIO_LIBRARIES "-framework Cocoa" "-framework Accelerate" "-framework AVFoundation" "-framework CoreGraphics" "-framework CoreImage" "-framework CoreMedia" "-framework CoreVideo" "-framework QuartzCore") +endif() if(HAVE_QUICKTIME) list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_qt.cpp) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index b769163cf3..c41d4246d2 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -82,34 +82,35 @@ To enable/disable APIs, you have to: 2. rebuild OpenCV itself */ enum VideoCaptureAPIs { - CAP_ANY = 0, //!< Auto detect - CAP_VFW = 200, //!< Video For Windows (platform native) - CAP_V4L = 200, //!< V4L/V4L2 capturing support via libv4l - CAP_V4L2 = CAP_V4L, //!< Same as CAP_V4L - CAP_FIREWIRE = 300, //!< IEEE 1394 drivers - CAP_FIREWARE = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_IEEE1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_DC1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_CMU1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_QT = 500, //!< QuickTime - CAP_UNICAP = 600, //!< Unicap drivers - CAP_DSHOW = 700, //!< DirectShow (via videoInput) - CAP_PVAPI = 800, //!< PvAPI, Prosilica GigE SDK - CAP_OPENNI = 900, //!< OpenNI (for Kinect) - CAP_OPENNI_ASUS = 910, //!< OpenNI (for Asus Xtion) - CAP_ANDROID = 1000, //!< Android - not used - CAP_XIAPI = 1100, //!< XIMEA Camera API - CAP_AVFOUNDATION = 1200, //!< AVFoundation framework for iOS (OS X Lion will have the same API) - CAP_GIGANETIX = 1300, //!< Smartek Giganetix GigEVisionSDK - CAP_MSMF = 1400, //!< Microsoft Media Foundation (via videoInput) - CAP_WINRT = 1410, //!< Microsoft Windows Runtime using Media Foundation - CAP_INTELPERC = 1500, //!< Intel Perceptual Computing SDK - CAP_OPENNI2 = 1600, //!< OpenNI2 (for Kinect) - CAP_OPENNI2_ASUS = 1610, //!< OpenNI2 (for Asus Xtion and Occipital Structure sensors) - CAP_GPHOTO2 = 1700, //!< gPhoto2 connection - CAP_GSTREAMER = 1800, //!< GStreamer - CAP_FFMPEG = 1900, //!< FFMPEG - CAP_IMAGES = 2000 //!< OpenCV Image Sequence (e.g. img_%02d.jpg) + CAP_ANY = 0, //!< Auto detect + CAP_VFW = 200, //!< Video For Windows (platform native) + CAP_V4L = 200, //!< V4L/V4L2 capturing support via libv4l + CAP_V4L2 = CAP_V4L, //!< Same as CAP_V4L + CAP_FIREWIRE = 300, //!< IEEE 1394 drivers + CAP_FIREWARE = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_IEEE1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_DC1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_CMU1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_QT = 500, //!< QuickTime + CAP_UNICAP = 600, //!< Unicap drivers + CAP_DSHOW = 700, //!< DirectShow (via videoInput) + CAP_PVAPI = 800, //!< PvAPI, Prosilica GigE SDK + CAP_OPENNI = 900, //!< OpenNI (for Kinect) + CAP_OPENNI_ASUS = 910, //!< OpenNI (for Asus Xtion) + CAP_ANDROID = 1000, //!< Android - not used + CAP_XIAPI = 1100, //!< XIMEA Camera API + CAP_AVFOUNDATION = 1200, //!< AVFoundation framework for iOS + CAP_AVFOUNDATION_MAC = 1210, //!< AVFoundation framework for macOS + CAP_GIGANETIX = 1300, //!< Smartek Giganetix GigEVisionSDK + CAP_MSMF = 1400, //!< Microsoft Media Foundation (via videoInput) + CAP_WINRT = 1410, //!< Microsoft Windows Runtime using Media Foundation + CAP_INTELPERC = 1500, //!< Intel Perceptual Computing SDK + CAP_OPENNI2 = 1600, //!< OpenNI2 (for Kinect) + CAP_OPENNI2_ASUS = 1610, //!< OpenNI2 (for Asus Xtion and Occipital Structure sensors) + CAP_GPHOTO2 = 1700, //!< gPhoto2 connection + CAP_GSTREAMER = 1800, //!< GStreamer + CAP_FFMPEG = 1900, //!< FFMPEG + CAP_IMAGES = 2000 //!< OpenCV Image Sequence (e.g. img_%02d.jpg) }; //! generic properties (based on DC1394 properties) diff --git a/modules/videoio/include/opencv2/videoio/videoio_c.h b/modules/videoio/include/opencv2/videoio/videoio_c.h index e502c7be83..1351c4e73a 100644 --- a/modules/videoio/include/opencv2/videoio/videoio_c.h +++ b/modules/videoio/include/opencv2/videoio/videoio_c.h @@ -108,6 +108,7 @@ enum CV_CAP_XIAPI =1100, // XIMEA Camera API CV_CAP_AVFOUNDATION = 1200, // AVFoundation framework for iOS (OS X Lion will have the same API) + CV_CAP_AVFOUNDATION_MAC = 1210, // AVFoundation framework for Mac CV_CAP_GIGANETIX = 1300, // Smartek Giganetix GigEVisionSDK diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index 55c20bef52..8e12b62da0 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -266,6 +266,12 @@ CV_IMPL CvCapture * cvCreateCameraCapture (int index) if (pref) break; #endif +#ifdef HAVE_AVFOUNDATION_MAC + case CV_CAP_AVFOUNDATION_MAC: + TRY_OPEN(capture, cvCreateCameraCapture_AVFoundation_Mac(index)) + if (pref) break; +#endif + #ifdef HAVE_GIGE_API case CV_CAP_GIGANETIX: TRY_OPEN(capture, cvCreateCameraCapture_Giganetix(index)) @@ -334,6 +340,12 @@ CV_IMPL CvCapture * cvCreateFileCaptureWithPreference (const char * filename, in if (apiPreference) break; #endif +#ifdef HAVE_AVFOUNDATION_MAC + case CV_CAP_AVFOUNDATION_MAC: + TRY_OPEN(result, cvCreateFileCapture_AVFoundation_Mac(filename)) + if (apiPreference) break; +#endif + #ifdef HAVE_OPENNI case CV_CAP_OPENNI: TRY_OPEN(result, cvCreateFileCapture_OpenNI (filename)) @@ -391,6 +403,10 @@ CV_IMPL CvVideoWriter* cvCreateVideoWriter( const char* filename, int fourcc, TRY_OPEN(result, cvCreateVideoWriter_AVFoundation(filename, fourcc, fps, frameSize, is_color)) #endif +#ifdef HAVE_AVFOUNDATION_MAC + TRY_OPEN(result, cvCreateVideoWriter_AVFoundation_Mac(filename, fourcc, fps, frameSize, is_color)) +#endif + #if defined(HAVE_QUICKTIME) || defined(HAVE_QTKIT) TRY_OPEN(result, cvCreateVideoWriter_QT(filename, fourcc, fps, frameSize, is_color)) #endif diff --git a/modules/videoio/src/cap_avfoundation_mac.mm b/modules/videoio/src/cap_avfoundation_mac.mm new file mode 100644 index 0000000000..814e151ccc --- /dev/null +++ b/modules/videoio/src/cap_avfoundation_mac.mm @@ -0,0 +1,1170 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the contributor be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*//////////////////////////////////////////////////////////////////////////////////////// + + +#include "precomp.hpp" +#include "opencv2/imgproc.hpp" +#include +#import + +/********************** Declaration of class headers ************************/ + +/***************************************************************************** + * + * CaptureDelegate Declaration. + * + * CaptureDelegate is notified on a separate thread by the OS whenever there + * is a new frame. When "updateImage" is called from the main thread, it + * copies this new frame into an IplImage, but only if this frame has not + * been copied before. When "getOutput" is called from the main thread, + * it gives the last copied IplImage. + * + *****************************************************************************/ + + +@interface CaptureDelegate : NSObject +{ + NSCondition *mHasNewFrame; + CVPixelBufferRef mGrabbedPixels; + CVImageBufferRef mCurrentImageBuffer; + IplImage *mDeviceImage; + uint8_t *mOutImagedata; + IplImage *mOutImage; + size_t currSize; +} + +- (void)captureOutput:(AVCaptureOutput *)captureOutput +didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection; + +- (BOOL)grabImageUntilDate: (NSDate *)limit; +- (int)updateImage; +- (IplImage*)getOutput; + +@end + +/***************************************************************************** + * + * CvCaptureCAM Declaration. + * + * CvCaptureCAM is the instantiation of a capture source for cameras. + * + *****************************************************************************/ + +class CvCaptureCAM : public CvCapture { +public: + CvCaptureCAM(int cameraNum = -1) ; + ~CvCaptureCAM(); + virtual bool grabFrame(); + virtual IplImage* retrieveFrame(int); + virtual double getProperty(int property_id) const; + virtual bool setProperty(int property_id, double value); + virtual int didStart(); + + +private: + AVCaptureSession *mCaptureSession; + AVCaptureDeviceInput *mCaptureDeviceInput; + AVCaptureVideoDataOutput *mCaptureVideoDataOutput; + AVCaptureDevice *mCaptureDevice; + CaptureDelegate *mCapture; + + int startCaptureDevice(int cameraNum); + void stopCaptureDevice(); + + void setWidthHeight(); + bool grabFrame(double timeOut); + + int camNum; + int width; + int height; + int settingWidth; + int settingHeight; + OSType mInputPixelFormat; + + int started; +}; + + +/***************************************************************************** + * + * CvCaptureFile Declaration. + * + * CvCaptureFile is the instantiation of a capture source for video files. + * + *****************************************************************************/ + +class CvCaptureFile : public CvCapture { +public: + CvCaptureFile(const char* filename) ; + ~CvCaptureFile(); + virtual bool grabFrame(); + virtual IplImage* retrieveFrame(int); + virtual double getProperty(int property_id) const; + virtual bool setProperty(int property_id, double value); + virtual int didStart(); + + +private: + AVAsset *mAsset; + AVAssetTrack *mAssetTrack; + AVAssetReader *mAssetReader; + AVAssetReaderTrackOutput *mTrackOutput; + + CMSampleBufferRef mCurrentSampleBuffer; + CVImageBufferRef mGrabbedPixels; + IplImage *mDeviceImage; + uint8_t *mOutImagedata; + IplImage *mOutImage; + size_t currSize; + + bool setupReadingAt(CMTime position); + IplImage* retrieveFramePixelBuffer(); + + CMTime mFrameTimestamp; + size_t mFrameNum; + OSType mInputPixelFormat; + + int started; +}; + + +/***************************************************************************** + * + * CvCaptureFile Declaration. + * + * CvCaptureFile is the instantiation of a capture source for video files. + * + *****************************************************************************/ + +class CvVideoWriter_AVFoundation_Mac : public CvVideoWriter{ + public: + CvVideoWriter_AVFoundation_Mac(const char* filename, int fourcc, + double fps, CvSize frame_size, + int is_color=1); + ~CvVideoWriter_AVFoundation_Mac(); + bool writeFrame(const IplImage* image); + private: + IplImage* argbimage; + + AVAssetWriter *mMovieWriter; + AVAssetWriterInput* mMovieWriterInput; + AVAssetWriterInputPixelBufferAdaptor* mMovieWriterAdaptor; + + NSString* path; + NSString* codec; + NSString* fileType; + double mMovieFPS; + CvSize movieSize; + int movieColor; + unsigned long mFrameNum; +}; + +/****************** Implementation of interface functions ********************/ + + +CvCapture* cvCreateFileCapture_AVFoundation_Mac(const char* filename) { + CvCaptureFile *retval = new CvCaptureFile(filename); + + if(retval->didStart()) + return retval; + delete retval; + return NULL; +} + +CvCapture* cvCreateCameraCapture_AVFoundation_Mac(int index ) { + CvCapture* retval = new CvCaptureCAM(index); + if (!((CvCaptureCAM *)retval)->didStart()) + cvReleaseCapture(&retval); + return retval; +} + +CvVideoWriter* cvCreateVideoWriter_AVFoundation_Mac(const char* filename, int fourcc, + double fps, CvSize frame_size, + int is_color) { + return new CvVideoWriter_AVFoundation_Mac(filename, fourcc, fps, frame_size,is_color); +} + +/********************** Implementation of Classes ****************************/ + +/***************************************************************************** + * + * CvCaptureCAM Implementation. + * + * CvCaptureCAM is the instantiation of a capture source for cameras. + * + *****************************************************************************/ + +CvCaptureCAM::CvCaptureCAM(int cameraNum) { + mCaptureSession = nil; + mCaptureDeviceInput = nil; + mCaptureVideoDataOutput = nil; + mCaptureDevice = nil; + mCapture = nil; + + width = 0; + height = 0; + settingWidth = 0; + settingHeight = 0; + + camNum = cameraNum; + + if ( ! startCaptureDevice(camNum) ) { + fprintf(stderr, "OpenCV: camera failed to properly initialize!\n"); + started = 0; + } else { + started = 1; + } +} + +CvCaptureCAM::~CvCaptureCAM() { + stopCaptureDevice(); +} + +int CvCaptureCAM::didStart() { + return started; +} + + +bool CvCaptureCAM::grabFrame() { + return grabFrame(1); +} + +bool CvCaptureCAM::grabFrame(double timeOut) { + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + bool isGrabbed = false; + NSDate *limit = [NSDate dateWithTimeIntervalSinceNow: timeOut]; + if ( [mCapture grabImageUntilDate: limit] ) { + [mCapture updateImage]; + isGrabbed = true; + } + + [localpool drain]; + return isGrabbed; +} + +IplImage* CvCaptureCAM::retrieveFrame(int) { + return [mCapture getOutput]; +} + +void CvCaptureCAM::stopCaptureDevice() { + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + [mCaptureSession stopRunning]; + + [mCaptureSession release]; + [mCaptureDeviceInput release]; + [mCaptureDevice release]; + + [mCaptureVideoDataOutput release]; + [mCapture release]; + + [localpool drain]; +} + +int CvCaptureCAM::startCaptureDevice(int cameraNum) { + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + // get capture device + NSArray *devices = [AVCaptureDevice devicesWithMediaType: AVMediaTypeVideo]; + + if ( devices.count == 0 ) { + fprintf(stderr, "OpenCV: AVFoundation didn't find any attached Video Input Devices!\n"); + [localpool drain]; + return 0; + } + + if ( cameraNum < 0 || devices.count <= NSUInteger(cameraNum) ) { + fprintf(stderr, "OpenCV: out device of bound (0-%ld): %d\n", devices.count-1, cameraNum); + [localpool drain]; + return 0; + } + + mCaptureDevice = devices[cameraNum]; + + if ( ! mCaptureDevice ) { + fprintf(stderr, "OpenCV: device %d not able to use.\n", cameraNum); + [localpool drain]; + return 0; + } + + // get input device + NSError *error = nil; + mCaptureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice: mCaptureDevice + error: &error]; + if ( error ) { + fprintf(stderr, "OpenCV: error in [AVCaptureDeviceInput initWithDevice:error:]\n"); + NSLog(@"OpenCV: %@", error.localizedDescription); + [localpool drain]; + return 0; + } + + // create output + mCapture = [[CaptureDelegate alloc] init]; + mCaptureVideoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; + dispatch_queue_t queue = dispatch_queue_create("cameraQueue", DISPATCH_QUEUE_SERIAL); + [mCaptureVideoDataOutput setSampleBufferDelegate: mCapture queue: queue]; + dispatch_release(queue); + + OSType pixelFormat = kCVPixelFormatType_32BGRA; + //OSType pixelFormat = kCVPixelFormatType_422YpCbCr8; + NSDictionary *pixelBufferOptions; + if (width > 0 && height > 0) { + pixelBufferOptions = + @{ + (id)kCVPixelBufferWidthKey: @(1.0*width), + (id)kCVPixelBufferHeightKey: @(1.0*height), + (id)kCVPixelBufferPixelFormatTypeKey: @(pixelFormat) + }; + } else { + pixelBufferOptions = + @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(pixelFormat) + }; + } + mCaptureVideoDataOutput.videoSettings = pixelBufferOptions; + mCaptureVideoDataOutput.alwaysDiscardsLateVideoFrames = YES; + + // create session + mCaptureSession = [[AVCaptureSession alloc] init]; + mCaptureSession.sessionPreset = AVCaptureSessionPresetMedium; + [mCaptureSession addInput: mCaptureDeviceInput]; + [mCaptureSession addOutput: mCaptureVideoDataOutput]; + + [mCaptureSession startRunning]; + + // flush old position image + grabFrame(1); + + [localpool drain]; + return 1; +} + +void CvCaptureCAM::setWidthHeight() { + NSMutableDictionary *pixelBufferOptions = [mCaptureVideoDataOutput.videoSettings mutableCopy]; + + while ( true ) { + // auto matching + pixelBufferOptions[(id)kCVPixelBufferWidthKey] = @(1.0*width); + pixelBufferOptions[(id)kCVPixelBufferHeightKey] = @(1.0*height); + mCaptureVideoDataOutput.videoSettings = pixelBufferOptions; + + // compare matched size and my options + CMFormatDescriptionRef format = mCaptureDevice.activeFormat.formatDescription; + CMVideoDimensions deviceSize = CMVideoFormatDescriptionGetDimensions(format); + if ( deviceSize.width == width && deviceSize.height == height ) { + break; + } + + // fit my options to matched size + width = deviceSize.width; + height = deviceSize.height; + } + + // flush old size image + grabFrame(1); + + [pixelBufferOptions release]; +} + + +double CvCaptureCAM::getProperty(int property_id) const{ + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + CMFormatDescriptionRef format = mCaptureDevice.activeFormat.formatDescription; + CMVideoDimensions s1 = CMVideoFormatDescriptionGetDimensions(format); + double retval = 0; + + switch (property_id) { + case CV_CAP_PROP_FRAME_WIDTH: + retval = s1.width; + break; + case CV_CAP_PROP_FRAME_HEIGHT: + retval = s1.height; + break; + case CV_CAP_PROP_FPS: + { + CMTime frameDuration = mCaptureDevice.activeVideoMaxFrameDuration; + retval = frameDuration.timescale / double(frameDuration.value); + } + break; + case CV_CAP_PROP_FORMAT: + retval = CV_8UC3; + break; + default: + break; + } + + [localpool drain]; + return retval; +} + +bool CvCaptureCAM::setProperty(int property_id, double value) { + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + bool isSucceeded = false; + + switch (property_id) { + case CV_CAP_PROP_FRAME_WIDTH: + width = value; + settingWidth = 1; + if (settingWidth && settingHeight) { + setWidthHeight(); + settingWidth = 0; + settingHeight = 0; + } + isSucceeded = true; + break; + case CV_CAP_PROP_FRAME_HEIGHT: + height = value; + settingHeight = 1; + if (settingWidth && settingHeight) { + setWidthHeight(); + settingWidth = 0; + settingHeight = 0; + } + isSucceeded = true; + break; + case CV_CAP_PROP_FPS: + if ( [mCaptureDevice lockForConfiguration: NULL] ) { + NSArray * ranges = mCaptureDevice.activeFormat.videoSupportedFrameRateRanges; + AVFrameRateRange *matchedRange = ranges[0]; + double minDiff = fabs(matchedRange.maxFrameRate - value); + for ( AVFrameRateRange *range in ranges ) { + double diff = fabs(range.maxFrameRate - value); + if ( diff < minDiff ) { + minDiff = diff; + matchedRange = range; + } + } + mCaptureDevice.activeVideoMinFrameDuration = matchedRange.minFrameDuration; + mCaptureDevice.activeVideoMaxFrameDuration = matchedRange.minFrameDuration; + isSucceeded = true; + [mCaptureDevice unlockForConfiguration]; + } + break; + default: + break; + } + + [localpool drain]; + return isSucceeded; +} + + +/***************************************************************************** + * + * CaptureDelegate Implementation. + * + * CaptureDelegate is notified on a separate thread by the OS whenever there + * is a new frame. When "updateImage" is called from the main thread, it + * copies this new frame into an IplImage, but only if this frame has not + * been copied before. When "getOutput" is called from the main thread, + * it gives the last copied IplImage. + * + *****************************************************************************/ + + +@implementation CaptureDelegate + +- (id)init { + [super init]; + mHasNewFrame = [[NSCondition alloc] init]; + mCurrentImageBuffer = NULL; + mGrabbedPixels = NULL; + mDeviceImage = NULL; + mOutImagedata = NULL; + mOutImage = NULL; + currSize = 0; + return self; +} + +-(void)dealloc { + free(mOutImagedata); + cvReleaseImage(&mOutImage); + cvReleaseImage(&mDeviceImage); + CVBufferRelease(mCurrentImageBuffer); + CVBufferRelease(mGrabbedPixels); + [mHasNewFrame release]; + [super dealloc]; +} + +- (void)captureOutput:(AVCaptureOutput *)captureOutput +didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { + (void)captureOutput; + (void)sampleBuffer; + (void)connection; + + CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + CVBufferRetain(imageBuffer); + + [mHasNewFrame lock]; + + CVBufferRelease(mCurrentImageBuffer); + mCurrentImageBuffer = imageBuffer; + [mHasNewFrame signal]; + + [mHasNewFrame unlock]; + +} + +-(IplImage*) getOutput { + return mOutImage; +} + +-(BOOL) grabImageUntilDate: (NSDate *)limit { + BOOL isGrabbed = NO; + [mHasNewFrame lock]; + + if ( mGrabbedPixels ) { + CVBufferRelease(mGrabbedPixels); + } + if ( [mHasNewFrame waitUntilDate: limit] ) { + isGrabbed = YES; + mGrabbedPixels = CVBufferRetain(mCurrentImageBuffer); + } + + [mHasNewFrame unlock]; + return isGrabbed; +} + +-(int) updateImage { + if ( ! mGrabbedPixels ) { + return 0; + } + + CVPixelBufferLockBaseAddress(mGrabbedPixels, 0); + void *baseaddress = CVPixelBufferGetBaseAddress(mGrabbedPixels); + + size_t width = CVPixelBufferGetWidth(mGrabbedPixels); + size_t height = CVPixelBufferGetHeight(mGrabbedPixels); + size_t rowBytes = CVPixelBufferGetBytesPerRow(mGrabbedPixels); + OSType pixelFormat = CVPixelBufferGetPixelFormatType(mGrabbedPixels); + + if ( rowBytes == 0 ) { + fprintf(stderr, "OpenCV: error: rowBytes == 0\n"); + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + return 0; + } + + if ( currSize != width*3*height ) { + currSize = width*3*height; + free(mOutImagedata); + mOutImagedata = reinterpret_cast(malloc(currSize)); + } + + if (mOutImage == NULL) { + mOutImage = cvCreateImageHeader(cvSize((int)width,(int)height), IPL_DEPTH_8U, 3); + } + mOutImage->width = int(width); + mOutImage->height = int(height); + mOutImage->nChannels = 3; + mOutImage->depth = IPL_DEPTH_8U; + mOutImage->widthStep = int(width*3); + mOutImage->imageData = reinterpret_cast(mOutImagedata); + mOutImage->imageSize = int(currSize); + + if ( pixelFormat == kCVPixelFormatType_32BGRA ) { + if (mDeviceImage == NULL) { + mDeviceImage = cvCreateImageHeader(cvSize(int(width),int(height)), IPL_DEPTH_8U, 4); + } + mDeviceImage->width = int(width); + mDeviceImage->height = int(height); + mDeviceImage->nChannels = 4; + mDeviceImage->depth = IPL_DEPTH_8U; + mDeviceImage->widthStep = int(rowBytes); + mDeviceImage->imageData = reinterpret_cast(baseaddress); + mDeviceImage->imageSize = int(rowBytes*height); + + cvCvtColor(mDeviceImage, mOutImage, CV_BGRA2BGR); + } else if ( pixelFormat == kCVPixelFormatType_422YpCbCr8 ) { + if ( currSize != width*3*height ) { + currSize = width*3*height; + free(mOutImagedata); + mOutImagedata = reinterpret_cast(malloc(currSize)); + } + + if (mDeviceImage == NULL) { + mDeviceImage = cvCreateImageHeader(cvSize(int(width),int(height)), IPL_DEPTH_8U, 2); + } + mDeviceImage->width = int(width); + mDeviceImage->height = int(height); + mDeviceImage->nChannels = 2; + mDeviceImage->depth = IPL_DEPTH_8U; + mDeviceImage->widthStep = int(rowBytes); + mDeviceImage->imageData = reinterpret_cast(baseaddress); + mDeviceImage->imageSize = int(rowBytes*height); + + cvCvtColor(mDeviceImage, mOutImage, CV_YUV2BGR_UYVY); + } else { + fprintf(stderr, "OpenCV: unknown pixel format 0x%08X\n", pixelFormat); + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + return 0; + } + + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + + return 1; +} + +@end + + +/***************************************************************************** + * + * CvCaptureFile Implementation. + * + * CvCaptureFile is the instantiation of a capture source for video files. + * + *****************************************************************************/ + +CvCaptureFile::CvCaptureFile(const char* filename) { + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + mAsset = nil; + mAssetTrack = nil; + mAssetReader = nil; + mTrackOutput = nil; + mDeviceImage = NULL; + mOutImage = NULL; + mOutImagedata = NULL; + currSize = 0; + mCurrentSampleBuffer = NULL; + mGrabbedPixels = NULL; + mFrameTimestamp = kCMTimeZero; + mFrameNum = 0; + + started = 0; + + mAsset = [[AVAsset assetWithURL:[NSURL fileURLWithPath: @(filename)]] retain]; + + if ( mAsset == nil ) { + fprintf(stderr, "OpenCV: Couldn't read movie file \"%s\"\n", filename); + [localpool drain]; + started = 0; + return; + } + + mAssetTrack = [[mAsset tracksWithMediaType: AVMediaTypeVideo][0] retain]; + + if ( ! setupReadingAt(kCMTimeZero) ) { + fprintf(stderr, "OpenCV: Couldn't read movie file \"%s\"\n", filename); + [localpool drain]; + started = 0; + return; + } + + started = 1; + [localpool drain]; +} + +CvCaptureFile::~CvCaptureFile() { + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + free(mOutImagedata); + cvReleaseImage(&mOutImage); + cvReleaseImage(&mDeviceImage); + [mAssetReader release]; + [mTrackOutput release]; + [mAssetTrack release]; + [mAsset release]; + CVBufferRelease(mGrabbedPixels); + if ( mCurrentSampleBuffer ) { + CFRelease(mCurrentSampleBuffer); + } + + [localpool drain]; +} + +bool CvCaptureFile::setupReadingAt(CMTime position) { + if (mAssetReader) { + if (mAssetReader.status == AVAssetReaderStatusReading) { + [mAssetReader cancelReading]; + } + [mAssetReader release]; + mAssetReader = nil; + } + if (mTrackOutput) { + [mTrackOutput release]; + mTrackOutput = nil; + } + + OSType pixelFormat = kCVPixelFormatType_32BGRA; + //OSType pixelFormat = kCVPixelFormatType_422YpCbCr8; + NSDictionary *settings = + @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(pixelFormat) + }; + mTrackOutput = [[AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack: mAssetTrack + outputSettings: settings] retain]; + + NSError *error = nil; + mAssetReader = [[AVAssetReader assetReaderWithAsset: mAsset + error: &error] retain]; + if ( error ) { + fprintf(stderr, "OpenCV: error in [AVAssetReader assetReaderWithAsset:error:]\n"); + NSLog(@"OpenCV: %@", error.localizedDescription); + return false; + } + + mAssetReader.timeRange = CMTimeRangeMake(position, kCMTimePositiveInfinity); + mFrameTimestamp = position; + mFrameNum = round((mFrameTimestamp.value * mAssetTrack.nominalFrameRate) / double(mFrameTimestamp.timescale)); + [mAssetReader addOutput: mTrackOutput]; + [mAssetReader startReading]; + + return true; +} + +int CvCaptureFile::didStart() { + return started; +} + +bool CvCaptureFile::grabFrame() { + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + CVBufferRelease(mGrabbedPixels); + if ( mCurrentSampleBuffer ) { + CFRelease(mCurrentSampleBuffer); + } + mCurrentSampleBuffer = [mTrackOutput copyNextSampleBuffer]; + mGrabbedPixels = CMSampleBufferGetImageBuffer(mCurrentSampleBuffer); + CVBufferRetain(mGrabbedPixels); + mFrameTimestamp = CMSampleBufferGetOutputPresentationTimeStamp(mCurrentSampleBuffer); + mFrameNum++; + + bool isReading = (mAssetReader.status == AVAssetReaderStatusReading); + [localpool drain]; + return isReading; +} + + +IplImage* CvCaptureFile::retrieveFramePixelBuffer() { + if ( ! mGrabbedPixels ) { + return 0; + } + + NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; + + CVPixelBufferLockBaseAddress(mGrabbedPixels, 0); + void *baseaddress = CVPixelBufferGetBaseAddress(mGrabbedPixels); + + size_t width = CVPixelBufferGetWidth(mGrabbedPixels); + size_t height = CVPixelBufferGetHeight(mGrabbedPixels); + size_t rowBytes = CVPixelBufferGetBytesPerRow(mGrabbedPixels); + OSType pixelFormat = CVPixelBufferGetPixelFormatType(mGrabbedPixels); + + if ( rowBytes == 0 ) { + fprintf(stderr, "OpenCV: error: rowBytes == 0\n"); + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + return 0; + } + + if ( currSize != width*3*height ) { + currSize = width*3*height; + free(mOutImagedata); + mOutImagedata = reinterpret_cast(malloc(currSize)); + } + + if (mOutImage == NULL) { + mOutImage = cvCreateImageHeader(cvSize((int)width,(int)height), IPL_DEPTH_8U, 3); + } + mOutImage->width = int(width); + mOutImage->height = int(height); + mOutImage->nChannels = 3; + mOutImage->depth = IPL_DEPTH_8U; + mOutImage->widthStep = int(width*3); + mOutImage->imageData = reinterpret_cast(mOutImagedata); + mOutImage->imageSize = int(currSize); + + if ( pixelFormat == kCVPixelFormatType_32BGRA ) { + if (mDeviceImage == NULL) { + mDeviceImage = cvCreateImageHeader(cvSize(int(width),int(height)), IPL_DEPTH_8U, 4); + } + mDeviceImage->width = int(width); + mDeviceImage->height = int(height); + mDeviceImage->nChannels = 4; + mDeviceImage->depth = IPL_DEPTH_8U; + mDeviceImage->widthStep = int(rowBytes); + mDeviceImage->imageData = reinterpret_cast(baseaddress); + mDeviceImage->imageSize = int(rowBytes*height); + + cvCvtColor(mDeviceImage, mOutImage, CV_BGRA2BGR); + } else if ( pixelFormat == kCVPixelFormatType_422YpCbCr8 ) { + if ( currSize != width*3*height ) { + currSize = width*3*height; + free(mOutImagedata); + mOutImagedata = reinterpret_cast(malloc(currSize)); + } + + if (mDeviceImage == NULL) { + mDeviceImage = cvCreateImageHeader(cvSize(int(width),int(height)), IPL_DEPTH_8U, 2); + } + mDeviceImage->width = int(width); + mDeviceImage->height = int(height); + mDeviceImage->nChannels = 2; + mDeviceImage->depth = IPL_DEPTH_8U; + mDeviceImage->widthStep = int(rowBytes); + mDeviceImage->imageData = reinterpret_cast(baseaddress); + mDeviceImage->imageSize = int(rowBytes*height); + + cvCvtColor(mDeviceImage, mOutImage, CV_YUV2BGR_UYVY); + } else { + fprintf(stderr, "OpenCV: unknown pixel format 0x%08X\n", pixelFormat); + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + return 0; + } + + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + + [localpool drain]; + + return mOutImage; +} + + +IplImage* CvCaptureFile::retrieveFrame(int) { + return retrieveFramePixelBuffer(); +} + +double CvCaptureFile::getProperty(int property_id) const{ + if (mAsset == nil) return 0; + + CMTime t; + + switch (property_id) { + case CV_CAP_PROP_POS_MSEC: + return mFrameTimestamp.value * 1000.0 / mFrameTimestamp.timescale; + case CV_CAP_PROP_POS_FRAMES: + return mFrameNum; + case CV_CAP_PROP_POS_AVI_RATIO: + t = [mAsset duration]; + return (mFrameTimestamp.value * t.timescale) / double(mFrameTimestamp.timescale * t.value); + case CV_CAP_PROP_FRAME_WIDTH: + return mAssetTrack.naturalSize.width; + case CV_CAP_PROP_FRAME_HEIGHT: + return mAssetTrack.naturalSize.height; + case CV_CAP_PROP_FPS: + return mAssetTrack.nominalFrameRate; + case CV_CAP_PROP_FRAME_COUNT: + t = [mAsset duration]; + return round((t.value * mAssetTrack.nominalFrameRate) / double(t.timescale)); + case CV_CAP_PROP_FORMAT: + return CV_8UC3; + default: + break; + } + + return 0; +} + +bool CvCaptureFile::setProperty(int property_id, double value) { + if (mAsset == nil) return false; + + NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; + + bool retval = false; + CMTime t; + + switch (property_id) { + case CV_CAP_PROP_POS_MSEC: + t = mAsset.duration; + t.value = value * t.timescale / 1000; + setupReadingAt(t); + retval = true; + break; + case CV_CAP_PROP_POS_FRAMES: + setupReadingAt(CMTimeMake(value, mAssetTrack.nominalFrameRate)); + retval = true; + break; + case CV_CAP_PROP_POS_AVI_RATIO: + t = mAsset.duration; + t.value = round(t.value * value); + setupReadingAt(t); + retval = true; + break; + default: + break; + } + + [localpool drain]; + return retval; +} + + +/***************************************************************************** + * + * CvVideoWriter Implementation. + * + * CvVideoWriter is the instantiation of a video output class + * + *****************************************************************************/ + + +CvVideoWriter_AVFoundation_Mac::CvVideoWriter_AVFoundation_Mac(const char* filename, int fourcc, + double fps, CvSize frame_size, + int is_color) { + + NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; + + + mFrameNum = 0; + mMovieFPS = fps; + movieSize = frame_size; + movieColor = is_color; + argbimage = cvCreateImage(movieSize, IPL_DEPTH_8U, 4); + path = [[[NSString stringWithCString:filename encoding:NSASCIIStringEncoding] stringByExpandingTildeInPath] retain]; + + + /* + AVFileTypeQuickTimeMovie + UTI for the QuickTime movie file format. + The value of this UTI is com.apple.quicktime-movie. Files are identified with the .mov and .qt extensions. + + AVFileTypeMPEG4 + UTI for the MPEG-4 file format. + The value of this UTI is public.mpeg-4. Files are identified with the .mp4 extension. + + AVFileTypeAppleM4V + UTI for the iTunes video file format. + The value of this UTI is com.apple.mpeg-4-video. Files are identified with the .m4v extension. + + AVFileType3GPP + UTI for the 3GPP file format. + The value of this UTI is public.3gpp. Files are identified with the .3gp, .3gpp, and .sdv extensions. + */ + + NSString *fileExt =[[[path pathExtension] lowercaseString] copy]; + if ([fileExt isEqualToString:@"mov"] || [fileExt isEqualToString:@"qt"]){ + fileType = [AVFileTypeQuickTimeMovie copy]; + }else if ([fileExt isEqualToString:@"mp4"]){ + fileType = [AVFileTypeMPEG4 copy]; + }else if ([fileExt isEqualToString:@"m4v"]){ + fileType = [AVFileTypeAppleM4V copy]; +#if TARGET_OS_IPHONE || TARGET_IPHONE_SIMULATOR + }else if ([fileExt isEqualToString:@"3gp"] || [fileExt isEqualToString:@"3gpp"] || [fileExt isEqualToString:@"sdv"] ){ + fileType = [AVFileType3GPP copy]; +#endif + } else{ + fileType = [AVFileTypeMPEG4 copy]; //default mp4 + } + [fileExt release]; + + char cc[5]; + cc[0] = fourcc & 255; + cc[1] = (fourcc >> 8) & 255; + cc[2] = (fourcc >> 16) & 255; + cc[3] = (fourcc >> 24) & 255; + cc[4] = 0; + int cc2 = CV_FOURCC(cc[0], cc[1], cc[2], cc[3]); + if (cc2!=fourcc) { + fprintf(stderr, "OpenCV: Didn't properly encode FourCC. Expected 0x%08X but got 0x%08X.\n", fourcc, cc2); + //exception; + } + + // Two codec supported AVVideoCodecH264 AVVideoCodecJPEG + // On iPhone 3G H264 is not supported. + if (fourcc == CV_FOURCC('J','P','E','G') || fourcc == CV_FOURCC('j','p','e','g') || + fourcc == CV_FOURCC('M','J','P','G') || fourcc == CV_FOURCC('m','j','p','g') ){ + codec = [AVVideoCodecJPEG copy]; // Use JPEG codec if specified, otherwise H264 + }else if(fourcc == CV_FOURCC('H','2','6','4') || fourcc == CV_FOURCC('a','v','c','1')){ + codec = [AVVideoCodecH264 copy]; + }else{ + codec = [AVVideoCodecH264 copy]; // default canonical H264. + + } + + //NSLog(@"Path: %@", path); + + NSError *error = nil; + + + // Make sure the file does not already exist. Necessary to overwirte?? + /* + NSFileManager *fileManager = [NSFileManager defaultManager]; + if ([fileManager fileExistsAtPath:path]){ + [fileManager removeItemAtPath:path error:&error]; + } + */ + + // Wire the writer: + // Supported file types: + // AVFileTypeQuickTimeMovie AVFileTypeMPEG4 AVFileTypeAppleM4V AVFileType3GPP + + mMovieWriter = [[AVAssetWriter alloc] initWithURL:[NSURL fileURLWithPath:path] + fileType:fileType + error:&error]; + //NSParameterAssert(mMovieWriter); + + NSDictionary *videoSettings = [NSDictionary dictionaryWithObjectsAndKeys: + codec, AVVideoCodecKey, + [NSNumber numberWithInt:movieSize.width], AVVideoWidthKey, + [NSNumber numberWithInt:movieSize.height], AVVideoHeightKey, + nil]; + + mMovieWriterInput = [[AVAssetWriterInput + assetWriterInputWithMediaType:AVMediaTypeVideo + outputSettings:videoSettings] retain]; + + //NSParameterAssert(mMovieWriterInput); + //NSParameterAssert([mMovieWriter canAddInput:mMovieWriterInput]); + + [mMovieWriter addInput:mMovieWriterInput]; + + mMovieWriterAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc] initWithAssetWriterInput:mMovieWriterInput sourcePixelBufferAttributes:nil]; + + + //Start a session: + [mMovieWriter startWriting]; + [mMovieWriter startSessionAtSourceTime:kCMTimeZero]; + + + if(mMovieWriter.status == AVAssetWriterStatusFailed){ + NSLog(@"%@", [mMovieWriter.error localizedDescription]); + // TODO: error handling, cleanup. Throw execption? + // return; + } + + [localpool drain]; +} + + +CvVideoWriter_AVFoundation_Mac::~CvVideoWriter_AVFoundation_Mac() { + NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; + + [mMovieWriterInput markAsFinished]; + [mMovieWriter finishWriting]; + [mMovieWriter release]; + [mMovieWriterInput release]; + [mMovieWriterAdaptor release]; + [path release]; + [codec release]; + [fileType release]; + cvReleaseImage(&argbimage); + + [localpool drain]; + +} + +bool CvVideoWriter_AVFoundation_Mac::writeFrame(const IplImage* iplimage) { + NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; + + // writer status check + if (![mMovieWriterInput isReadyForMoreMediaData] || mMovieWriter.status != AVAssetWriterStatusWriting ) { + NSLog(@"[mMovieWriterInput isReadyForMoreMediaData] Not ready for media data or ..."); + NSLog(@"mMovieWriter.status: %d. Error: %@", (int)mMovieWriter.status, [mMovieWriter.error localizedDescription]); + [localpool drain]; + return false; + } + + BOOL success = FALSE; + + if (iplimage->height!=movieSize.height || iplimage->width!=movieSize.width){ + fprintf(stderr, "OpenCV: Frame size does not match video size.\n"); + [localpool drain]; + return false; + } + + if (movieColor) { + //assert(iplimage->nChannels == 3); + cvCvtColor(iplimage, argbimage, CV_BGR2BGRA); + }else{ + //assert(iplimage->nChannels == 1); + cvCvtColor(iplimage, argbimage, CV_GRAY2BGRA); + } + //IplImage -> CGImage conversion + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + NSData *nsData = [NSData dataWithBytes:argbimage->imageData length:argbimage->imageSize]; + CGDataProviderRef provider = CGDataProviderCreateWithCFData((CFDataRef)nsData); + CGImageRef cgImage = CGImageCreate(argbimage->width, argbimage->height, + argbimage->depth, argbimage->depth * argbimage->nChannels, argbimage->widthStep, + colorSpace, kCGImageAlphaLast|kCGBitmapByteOrderDefault, + provider, NULL, false, kCGRenderingIntentDefault); + + //CGImage -> CVPixelBufferRef coversion + CVPixelBufferRef pixelBuffer = NULL; + CFDataRef cfData = CGDataProviderCopyData(CGImageGetDataProvider(cgImage)); + int status = CVPixelBufferCreateWithBytes(NULL, + movieSize.width, + movieSize.height, + kCVPixelFormatType_32BGRA, + (void*)CFDataGetBytePtr(cfData), + CGImageGetBytesPerRow(cgImage), + NULL, + 0, + NULL, + &pixelBuffer); + if(status == kCVReturnSuccess){ + success = [mMovieWriterAdaptor appendPixelBuffer:pixelBuffer + withPresentationTime:CMTimeMake(mFrameNum, mMovieFPS)]; + } + + //cleanup + CFRelease(cfData); + CVPixelBufferRelease(pixelBuffer); + CGImageRelease(cgImage); + CGDataProviderRelease(provider); + CGColorSpaceRelease(colorSpace); + + [localpool drain]; + + if (success) { + mFrameNum ++; + //NSLog(@"Frame #%d", mFrameNum); + return true; + }else{ + NSLog(@"Frame appendPixelBuffer failed."); + return false; + } + +} diff --git a/modules/videoio/src/precomp.hpp b/modules/videoio/src/precomp.hpp index 45ce1a1f60..50853d0bb9 100644 --- a/modules/videoio/src/precomp.hpp +++ b/modules/videoio/src/precomp.hpp @@ -127,6 +127,7 @@ CvCapture* cvCreateFileCapture_OpenNI( const char* filename ); CvCapture* cvCreateCameraCapture_Android( int index ); CvCapture* cvCreateCameraCapture_XIMEA( int index ); CvCapture* cvCreateCameraCapture_AVFoundation(int index); +CvCapture* cvCreateCameraCapture_AVFoundation_Mac(int index); CvCapture* cvCreateFileCapture_Images(const char* filename); CvVideoWriter* cvCreateVideoWriter_Images(const char* filename); @@ -155,6 +156,9 @@ CvVideoWriter* cvCreateVideoWriter_QT ( const char* filename, int fourcc, CvCapture* cvCreateFileCapture_AVFoundation (const char * filename); CvVideoWriter* cvCreateVideoWriter_AVFoundation( const char* filename, int fourcc, double fps, CvSize frameSize, int is_color ); +CvCapture* cvCreateFileCapture_AVFoundation_Mac(const char * filename); +CvVideoWriter* cvCreateVideoWriter_AVFoundation_Mac( const char* filename, int fourcc, + double fps, CvSize frameSize, int is_color ); CvCapture * cvCreateCameraCapture_Unicap (const int index); From a92da54e79493e50e61527bd2b6ea2ec3e1bd288 Mon Sep 17 00:00:00 2001 From: Matthew Self Date: Tue, 23 Aug 2016 11:11:00 -0700 Subject: [PATCH 2/4] Add support for CAP_PROP_MODE Support setting CAP_PROP_MODE to capture grayscale or YUV frames much faster from CV_CAP_AVFOUNDATION_MAC. --- modules/videoio/src/cap_avfoundation_mac.mm | 231 ++++++++++++++++---- 1 file changed, 187 insertions(+), 44 deletions(-) diff --git a/modules/videoio/src/cap_avfoundation_mac.mm b/modules/videoio/src/cap_avfoundation_mac.mm index 814e151ccc..aa888188c0 100644 --- a/modules/videoio/src/cap_avfoundation_mac.mm +++ b/modules/videoio/src/cap_avfoundation_mac.mm @@ -155,6 +155,8 @@ private: uint8_t *mOutImagedata; IplImage *mOutImage; size_t currSize; + int mMode; + int mFormat; bool setupReadingAt(CMTime position); IplImage* retrieveFramePixelBuffer(); @@ -169,13 +171,13 @@ private: /***************************************************************************** * - * CvCaptureFile Declaration. + * CvVideoWriter_AVFoundation_Mac Declaration. * - * CvCaptureFile is the instantiation of a capture source for video files. + * CvVideoWriter_AVFoundation_Mac is the instantiation of a video output class. * *****************************************************************************/ -class CvVideoWriter_AVFoundation_Mac : public CvVideoWriter{ +class CvVideoWriter_AVFoundation_Mac : public CvVideoWriter { public: CvVideoWriter_AVFoundation_Mac(const char* filename, int fourcc, double fps, CvSize frame_size, @@ -675,6 +677,8 @@ CvCaptureFile::CvCaptureFile(const char* filename) { mOutImage = NULL; mOutImagedata = NULL; currSize = 0; + mMode = CV_CAP_MODE_BGR; + mFormat = CV_8UC3; mCurrentSampleBuffer = NULL; mGrabbedPixels = NULL; mFrameTimestamp = kCMTimeZero; @@ -735,8 +739,27 @@ bool CvCaptureFile::setupReadingAt(CMTime position) { mTrackOutput = nil; } - OSType pixelFormat = kCVPixelFormatType_32BGRA; - //OSType pixelFormat = kCVPixelFormatType_422YpCbCr8; + // Capture in a pixel format that can be converted efficiently to the output mode. + OSType pixelFormat; + if (mMode == CV_CAP_MODE_BGR || mMode == CV_CAP_MODE_RGB) { + // For CV_CAP_MODE_BGR, read frames as BGRA (AV Foundation's YUV->RGB conversion is slightly faster than OpenCV's CV_YUV2BGR_YV12) + // kCVPixelFormatType_32ABGR is reportedly faster on OS X, but OpenCV doesn't have a CV_ABGR2BGR conversion. + // kCVPixelFormatType_24RGB is significanly slower than kCVPixelFormatType_32BGRA. + pixelFormat = kCVPixelFormatType_32BGRA; + mFormat = CV_8UC3; + } else if (mMode == CV_CAP_MODE_GRAY) { + // For CV_CAP_MODE_GRAY, read frames as 420v (faster than 420f or 422 -- at least for H.264 files) + pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + mFormat = CV_8UC1; + } else if (mMode == CV_CAP_MODE_YUYV) { + // For CV_CAP_MODE_YUYV, read frames directly as 422. + pixelFormat = kCVPixelFormatType_422YpCbCr8; + mFormat = CV_8UC2; + } else { + fprintf(stderr, "VIDEOIO ERROR: AVF Mac: Unsupported mode: %d\n", mMode); + return false; + } + NSDictionary *settings = @{ (id)kCVPixelBufferPixelFormatTypeKey: @(pixelFormat) @@ -744,6 +767,11 @@ bool CvCaptureFile::setupReadingAt(CMTime position) { mTrackOutput = [[AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack: mAssetTrack outputSettings: settings] retain]; + if ( !mTrackOutput ) { + fprintf(stderr, "OpenCV: error in [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:outputSettings:]\n"); + return false; + } + NSError *error = nil; mAssetReader = [[AVAssetReader assetReaderWithAsset: mAsset error: &error] retain]; @@ -793,13 +821,23 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() { NSAutoreleasePool *localpool = [[NSAutoreleasePool alloc] init]; CVPixelBufferLockBaseAddress(mGrabbedPixels, 0); - void *baseaddress = CVPixelBufferGetBaseAddress(mGrabbedPixels); + void *baseaddress; + size_t width, height, rowBytes; - size_t width = CVPixelBufferGetWidth(mGrabbedPixels); - size_t height = CVPixelBufferGetHeight(mGrabbedPixels); - size_t rowBytes = CVPixelBufferGetBytesPerRow(mGrabbedPixels); OSType pixelFormat = CVPixelBufferGetPixelFormatType(mGrabbedPixels); + if (CVPixelBufferIsPlanar(mGrabbedPixels)) { + baseaddress = CVPixelBufferGetBaseAddressOfPlane(mGrabbedPixels, 0); + width = CVPixelBufferGetWidthOfPlane(mGrabbedPixels, 0); + height = CVPixelBufferGetHeightOfPlane(mGrabbedPixels, 0); + rowBytes = CVPixelBufferGetBytesPerRowOfPlane(mGrabbedPixels, 0); + } else { + baseaddress = CVPixelBufferGetBaseAddress(mGrabbedPixels); + width = CVPixelBufferGetWidth(mGrabbedPixels); + height = CVPixelBufferGetHeight(mGrabbedPixels); + rowBytes = CVPixelBufferGetBytesPerRow(mGrabbedPixels); + } + if ( rowBytes == 0 ) { fprintf(stderr, "OpenCV: error: rowBytes == 0\n"); CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); @@ -808,63 +846,144 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() { return 0; } - if ( currSize != width*3*height ) { - currSize = width*3*height; + // Output image paramaters. + int outChannels; + if (mMode == CV_CAP_MODE_BGR || mMode == CV_CAP_MODE_RGB) { + outChannels = 3; + } else if (mMode == CV_CAP_MODE_GRAY) { + outChannels = 1; + } else if (mMode == CV_CAP_MODE_YUYV) { + outChannels = 2; + } else { + fprintf(stderr, "VIDEOIO ERROR: AVF Mac: Unsupported mode: %d\n", mMode); + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + return 0; + } + + if ( currSize != width*outChannels*height ) { + currSize = width*outChannels*height; free(mOutImagedata); mOutImagedata = reinterpret_cast(malloc(currSize)); } + // Build the header for the output image. if (mOutImage == NULL) { - mOutImage = cvCreateImageHeader(cvSize((int)width,(int)height), IPL_DEPTH_8U, 3); + mOutImage = cvCreateImageHeader(cvSize((int)width,(int)height), IPL_DEPTH_8U, outChannels); } mOutImage->width = int(width); mOutImage->height = int(height); - mOutImage->nChannels = 3; + mOutImage->nChannels = outChannels; mOutImage->depth = IPL_DEPTH_8U; - mOutImage->widthStep = int(width*3); + mOutImage->widthStep = int(width*outChannels); mOutImage->imageData = reinterpret_cast(mOutImagedata); mOutImage->imageSize = int(currSize); + // Device image paramaters and conversion code. + // (Not all of these conversions are used in production, but they were all tested to find the fastest options.) + int deviceChannels; + int cvtCode; + if ( pixelFormat == kCVPixelFormatType_32BGRA ) { - if (mDeviceImage == NULL) { - mDeviceImage = cvCreateImageHeader(cvSize(int(width),int(height)), IPL_DEPTH_8U, 4); - } - mDeviceImage->width = int(width); - mDeviceImage->height = int(height); - mDeviceImage->nChannels = 4; - mDeviceImage->depth = IPL_DEPTH_8U; - mDeviceImage->widthStep = int(rowBytes); - mDeviceImage->imageData = reinterpret_cast(baseaddress); - mDeviceImage->imageSize = int(rowBytes*height); + deviceChannels = 4; - cvCvtColor(mDeviceImage, mOutImage, CV_BGRA2BGR); - } else if ( pixelFormat == kCVPixelFormatType_422YpCbCr8 ) { - if ( currSize != width*3*height ) { - currSize = width*3*height; - free(mOutImagedata); - mOutImagedata = reinterpret_cast(malloc(currSize)); + if (mMode == CV_CAP_MODE_BGR) { + cvtCode = CV_BGRA2BGR; + } else if (mMode == CV_CAP_MODE_RGB) { + cvtCode = CV_BGRA2RGB; + } else if (mMode == CV_CAP_MODE_GRAY) { + cvtCode = CV_BGRA2GRAY; + } else { + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + fprintf(stderr, "OpenCV: unsupported pixel conversion mode\n"); + return 0; } + } else if ( pixelFormat == kCVPixelFormatType_24RGB ) { + deviceChannels = 3; - if (mDeviceImage == NULL) { - mDeviceImage = cvCreateImageHeader(cvSize(int(width),int(height)), IPL_DEPTH_8U, 2); + if (mMode == CV_CAP_MODE_BGR) { + cvtCode = CV_RGB2BGR; + } else if (mMode == CV_CAP_MODE_RGB) { + cvtCode = 0; + } else if (mMode == CV_CAP_MODE_GRAY) { + cvtCode = CV_RGB2GRAY; + } else { + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + fprintf(stderr, "OpenCV: unsupported pixel conversion mode\n"); + return 0; } - mDeviceImage->width = int(width); - mDeviceImage->height = int(height); - mDeviceImage->nChannels = 2; - mDeviceImage->depth = IPL_DEPTH_8U; - mDeviceImage->widthStep = int(rowBytes); - mDeviceImage->imageData = reinterpret_cast(baseaddress); - mDeviceImage->imageSize = int(rowBytes*height); + } else if ( pixelFormat == kCVPixelFormatType_422YpCbCr8 ) { // 422 (2vuy, UYVY) + deviceChannels = 2; - cvCvtColor(mDeviceImage, mOutImage, CV_YUV2BGR_UYVY); + if (mMode == CV_CAP_MODE_BGR) { + cvtCode = CV_YUV2BGR_UYVY; + } else if (mMode == CV_CAP_MODE_RGB) { + cvtCode = CV_YUV2RGB_UYVY; + } else if (mMode == CV_CAP_MODE_GRAY) { + cvtCode = CV_YUV2GRAY_UYVY; + } else if (mMode == CV_CAP_MODE_YUYV) { + cvtCode = 0; // Copy + } else { + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + fprintf(stderr, "OpenCV: unsupported pixel conversion mode\n"); + return 0; + } + } else if ( pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || // 420v + pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ) { // 420f + // cvCvtColor(CV_YUV2GRAY_420) is expecting a single buffer with both the Y plane and the CrCb planes. + // So, lie about the height of the buffer. cvCvtColor(CV_YUV2GRAY_420) will only read the first 2/3 of it. + height = height * 3 / 2; + deviceChannels = 1; + + if (mMode == CV_CAP_MODE_BGR) { + cvtCode = CV_YUV2BGR_YV12; + } else if (mMode == CV_CAP_MODE_RGB) { + cvtCode = CV_YUV2RGB_YV12; + } else if (mMode == CV_CAP_MODE_GRAY) { + cvtCode = CV_YUV2GRAY_420; + } else { + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); + CVBufferRelease(mGrabbedPixels); + mGrabbedPixels = NULL; + fprintf(stderr, "OpenCV: unsupported pixel conversion mode\n"); + return 0; + } } else { - fprintf(stderr, "OpenCV: unknown pixel format 0x%08X\n", pixelFormat); + fprintf(stderr, "OpenCV: unsupported pixel format 0x%08X\n", pixelFormat); CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); CVBufferRelease(mGrabbedPixels); mGrabbedPixels = NULL; return 0; } + // Build the header for the device image. + if (mDeviceImage == NULL) { + mDeviceImage = cvCreateImageHeader(cvSize(int(width),int(height)), IPL_DEPTH_8U, deviceChannels); + } + mDeviceImage->width = int(width); + mDeviceImage->height = int(height); + mDeviceImage->nChannels = deviceChannels; + mDeviceImage->depth = IPL_DEPTH_8U; + mDeviceImage->widthStep = int(rowBytes); + mDeviceImage->imageData = reinterpret_cast(baseaddress); + mDeviceImage->imageSize = int(rowBytes*height); + + // Convert the device image into the output image. + if (cvtCode == 0) { + // Copy. + cv::cvarrToMat(mDeviceImage).copyTo(cv::cvarrToMat(mOutImage)); + } else { + cvCvtColor(mDeviceImage, mOutImage, cvtCode); + } + + CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); [localpool drain]; @@ -900,7 +1019,9 @@ double CvCaptureFile::getProperty(int property_id) const{ t = [mAsset duration]; return round((t.value * mAssetTrack.nominalFrameRate) / double(t.timescale)); case CV_CAP_PROP_FORMAT: - return CV_8UC3; + return mFormat; + case CV_CAP_PROP_MODE: + return mMode; default: break; } @@ -933,6 +1054,28 @@ bool CvCaptureFile::setProperty(int property_id, double value) { setupReadingAt(t); retval = true; break; + case CV_CAP_PROP_MODE: + int mode; + mode = cvRound(value); + if (mMode == mode) { + retval = true; + } else { + switch (mode) { + case CV_CAP_MODE_BGR: + case CV_CAP_MODE_RGB: + case CV_CAP_MODE_GRAY: + case CV_CAP_MODE_YUYV: + mMode = mode; + setupReadingAt(mFrameTimestamp); + retval = true; + break; + default: + fprintf(stderr, "VIDEOIO ERROR: AVF Mac: Unsupported mode: %d\n", mode); + retval=false; + break; + } + } + break; default: break; } @@ -944,9 +1087,9 @@ bool CvCaptureFile::setProperty(int property_id, double value) { /***************************************************************************** * - * CvVideoWriter Implementation. + * CvVideoWriter_AVFoundation_Mac Implementation. * - * CvVideoWriter is the instantiation of a video output class + * CvVideoWriter_AVFoundation_Mac is the instantiation of a video output class. * *****************************************************************************/ From 0882936707c019a8f12a58519a431e042f03e9be Mon Sep 17 00:00:00 2001 From: Matthew Self Date: Tue, 23 Aug 2016 11:15:24 -0700 Subject: [PATCH 3/4] Fix buffer release issue CvVideoWriter_AVFoundation_Mac had a serious buffer release bug. Also made writeFrame() block until isReadyForMoreMediaData rather than return an error. --- modules/videoio/src/cap_avfoundation_mac.mm | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/modules/videoio/src/cap_avfoundation_mac.mm b/modules/videoio/src/cap_avfoundation_mac.mm index aa888188c0..4228bc3735 100644 --- a/modules/videoio/src/cap_avfoundation_mac.mm +++ b/modules/videoio/src/cap_avfoundation_mac.mm @@ -927,7 +927,7 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() { } else if (mMode == CV_CAP_MODE_GRAY) { cvtCode = CV_YUV2GRAY_UYVY; } else if (mMode == CV_CAP_MODE_YUYV) { - cvtCode = 0; // Copy + cvtCode = -1; // Copy } else { CVPixelBufferUnlockBaseAddress(mGrabbedPixels, 0); CVBufferRelease(mGrabbedPixels); @@ -976,7 +976,7 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() { mDeviceImage->imageSize = int(rowBytes*height); // Convert the device image into the output image. - if (cvtCode == 0) { + if (cvtCode == -1) { // Copy. cv::cvarrToMat(mDeviceImage).copyTo(cv::cvarrToMat(mOutImage)); } else { @@ -1213,7 +1213,7 @@ CvVideoWriter_AVFoundation_Mac::CvVideoWriter_AVFoundation_Mac(const char* filen if(mMovieWriter.status == AVAssetWriterStatusFailed){ - NSLog(@"%@", [mMovieWriter.error localizedDescription]); + NSLog(@"AVF: AVAssetWriter status: %@", [mMovieWriter.error localizedDescription]); // TODO: error handling, cleanup. Throw execption? // return; } @@ -1239,17 +1239,27 @@ CvVideoWriter_AVFoundation_Mac::~CvVideoWriter_AVFoundation_Mac() { } +static void releaseCallback( void *releaseRefCon, const void * ) { + CFRelease((CFDataRef)releaseRefCon); +} + bool CvVideoWriter_AVFoundation_Mac::writeFrame(const IplImage* iplimage) { NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; // writer status check - if (![mMovieWriterInput isReadyForMoreMediaData] || mMovieWriter.status != AVAssetWriterStatusWriting ) { - NSLog(@"[mMovieWriterInput isReadyForMoreMediaData] Not ready for media data or ..."); + if (mMovieWriter.status != AVAssetWriterStatusWriting ) { NSLog(@"mMovieWriter.status: %d. Error: %@", (int)mMovieWriter.status, [mMovieWriter.error localizedDescription]); [localpool drain]; return false; } + // Make writeFrame() a blocking call. + while (![mMovieWriterInput isReadyForMoreMediaData]) { + fprintf(stderr, "OpenCV: AVF: waiting to write video data.\n"); + // Sleep 1 msec. + usleep(1000); + } + BOOL success = FALSE; if (iplimage->height!=movieSize.height || iplimage->width!=movieSize.width){ @@ -1283,8 +1293,8 @@ bool CvVideoWriter_AVFoundation_Mac::writeFrame(const IplImage* iplimage) { kCVPixelFormatType_32BGRA, (void*)CFDataGetBytePtr(cfData), CGImageGetBytesPerRow(cgImage), - NULL, - 0, + &releaseCallback, + (void *)cfData, NULL, &pixelBuffer); if(status == kCVReturnSuccess){ @@ -1293,7 +1303,6 @@ bool CvVideoWriter_AVFoundation_Mac::writeFrame(const IplImage* iplimage) { } //cleanup - CFRelease(cfData); CVPixelBufferRelease(pixelBuffer); CGImageRelease(cgImage); CGDataProviderRelease(provider); From f85e33f4af15638cdff6ffb50e7669370cdd7f5b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 9 Sep 2016 15:38:00 +0300 Subject: [PATCH 4/4] videoio: refactor AVFoundation code integration --- CMakeLists.txt | 15 +++-- cmake/OpenCVFindLibsVideo.cmake | 30 ++++------ cmake/templates/cvconfig.h.in | 3 - modules/videoio/CMakeLists.txt | 13 +++-- modules/videoio/include/opencv2/videoio.hpp | 57 +++++++++---------- .../include/opencv2/videoio/videoio_c.h | 1 - modules/videoio/src/cap.cpp | 16 ------ modules/videoio/src/cap_avfoundation_mac.mm | 28 ++++----- modules/videoio/src/precomp.hpp | 4 -- 9 files changed, 69 insertions(+), 98 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9681a28810..96cb85bcdf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,8 +169,7 @@ OCV_OPTION(OPENCV_ENABLE_NONFREE "Enable non-free algorithms" OFF) # Optional 3rd party components # =================================================== OCV_OPTION(WITH_1394 "Include IEEE1394 support" ON IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) -OCV_OPTION(WITH_AVFOUNDATION "Use AVFoundation for Video I/O (iOS)" ON IF IOS) -OCV_OPTION(WITH_AVFOUNDATION_MAC "Use AVFoundation for Video I/O (Mac)" ON IF (NOT IOS AND APPLE) ) +OCV_OPTION(WITH_AVFOUNDATION "Use AVFoundation for Video I/O (iOS/Mac)" ON IF APPLE) OCV_OPTION(WITH_CARBON "Use Carbon for UI instead of Cocoa" OFF IF APPLE ) OCV_OPTION(WITH_CAROTENE "Use NVidia carotene acceleration library for ARM platform" ON IF (ARM OR AARCH64) AND NOT IOS AND NOT (CMAKE_VERSION VERSION_LESS "2.8.11")) OCV_OPTION(WITH_VTK "Include VTK library support (and build opencv_viz module eiher)" ON IF (NOT ANDROID AND NOT IOS AND NOT WINRT AND NOT CMAKE_CROSSCOMPILING) ) @@ -199,7 +198,8 @@ OCV_OPTION(WITH_PVAPI "Include Prosilica GigE support" OFF OCV_OPTION(WITH_GIGEAPI "Include Smartek GigE support" OFF IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_QT "Build with Qt Backend support" OFF IF (NOT ANDROID AND NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_WIN32UI "Build with Win32 UI Backend support" ON IF WIN32 AND NOT WINRT) -OCV_OPTION(WITH_QUICKTIME "Use QuickTime for Video I/O insted of QTKit" OFF IF APPLE ) +OCV_OPTION(WITH_QUICKTIME "Use QuickTime for Video I/O" OFF IF APPLE ) +OCV_OPTION(WITH_QTKIT "Use QTKit Video I/O backend" OFF IF APPLE ) OCV_OPTION(WITH_TBB "Include Intel TBB support" OFF IF (NOT IOS AND NOT WINRT) ) OCV_OPTION(WITH_OPENMP "Include OpenMP support" OFF) OCV_OPTION(WITH_CSTRIPES "Include C= support" OFF IF (WIN32 AND NOT WINRT) ) @@ -1110,10 +1110,13 @@ if(DEFINED WITH_GIGEAPI) endif(DEFINED WITH_GIGEAPI) if(DEFINED APPLE) + status(" AVFoundation:" HAVE_AVFOUNDATION THEN YES ELSE NO) + if(WITH_QUICKTIME OR HAVE_QUICKTIME) status(" QuickTime:" HAVE_QUICKTIME THEN YES ELSE NO) - status(" QTKit:" HAVE_QTKIT THEN YES ELSE NO) - status(" AVFoundation:" WITH_AVFOUNDATION THEN YES ELSE NO) - status(" AVFoundationMac:" WITH_AVFOUNDATION_MAC THEN YES ELSE NO) + endif() + if(WITH_QTKIT OR HAVE_QTKIT) + status(" QTKit:" HAVE_QTKIT THEN "YES (deprecated)" ELSE NO) + endif() endif(DEFINED APPLE) if(DEFINED WITH_UNICAP) diff --git a/cmake/OpenCVFindLibsVideo.cmake b/cmake/OpenCVFindLibsVideo.cmake index c23ee06d87..0c4072cb3b 100644 --- a/cmake/OpenCVFindLibsVideo.cmake +++ b/cmake/OpenCVFindLibsVideo.cmake @@ -302,26 +302,18 @@ if(WIN32) endif() endif(WIN32) -# --- Apple AV Foundation (iOS) --- -if(WITH_AVFOUNDATION) - set(HAVE_AVFOUNDATION YES) -endif() - -# --- Apple AV Foundation (Mac) --- -if(WITH_AVFOUNDATION_MAC) - set(HAVE_AVFOUNDATION_MAC YES) -endif() - -# --- QuickTime, Apple AV Foundation (Mac) --- -if (NOT IOS) - if(WITH_QUICKTIME) - set(HAVE_QUICKTIME YES) - elseif(WITH_QTKIT) - set(HAVE_QTKIT YES) - elseif(APPLE AND CMAKE_COMPILER_IS_CLANGCXX) - set(HAVE_AVFOUNDATION_MAC YES) +if(APPLE) + if(WITH_AVFOUNDATION) + set(HAVE_AVFOUNDATION YES) endif() -endif() + if(NOT IOS) + if(WITH_QUICKTIME) + set(HAVE_QUICKTIME YES) + elseif(WITH_QTKIT) + set(HAVE_QTKIT YES) + endif() + endif() +endif(APPLE) # --- Intel Perceptual Computing SDK --- if(WITH_INTELPERC) diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index 50b4185e2c..56d415b400 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -16,9 +16,6 @@ /* AVFoundation video libraries */ #cmakedefine HAVE_AVFOUNDATION -/* AVFoundation (Mac) video libraries */ -#cmakedefine HAVE_AVFOUNDATION_MAC - /* V4L capturing support */ #cmakedefine HAVE_CAMV4L diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index d75c2d80bb..413b3cdea0 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -168,12 +168,13 @@ if(HAVE_GIGE_API) endif(HAVE_GIGE_API) if(HAVE_AVFOUNDATION) - list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_avfoundation.mm) - list(APPEND VIDEOIO_LIBRARIES "-framework AVFoundation" "-framework QuartzCore") -endif() -if(HAVE_AVFOUNDATION_MAC) - list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_avfoundation_mac.mm) - list(APPEND VIDEOIO_LIBRARIES "-framework Cocoa" "-framework Accelerate" "-framework AVFoundation" "-framework CoreGraphics" "-framework CoreImage" "-framework CoreMedia" "-framework CoreVideo" "-framework QuartzCore") + if(IOS) + list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_avfoundation.mm) + list(APPEND VIDEOIO_LIBRARIES "-framework AVFoundation" "-framework QuartzCore") + else() + list(APPEND videoio_srcs ${CMAKE_CURRENT_LIST_DIR}/src/cap_avfoundation_mac.mm) + list(APPEND VIDEOIO_LIBRARIES "-framework Cocoa" "-framework Accelerate" "-framework AVFoundation" "-framework CoreGraphics" "-framework CoreImage" "-framework CoreMedia" "-framework CoreVideo" "-framework QuartzCore") + endif() endif() if(HAVE_QUICKTIME) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index c41d4246d2..b769163cf3 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -82,35 +82,34 @@ To enable/disable APIs, you have to: 2. rebuild OpenCV itself */ enum VideoCaptureAPIs { - CAP_ANY = 0, //!< Auto detect - CAP_VFW = 200, //!< Video For Windows (platform native) - CAP_V4L = 200, //!< V4L/V4L2 capturing support via libv4l - CAP_V4L2 = CAP_V4L, //!< Same as CAP_V4L - CAP_FIREWIRE = 300, //!< IEEE 1394 drivers - CAP_FIREWARE = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_IEEE1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_DC1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_CMU1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE - CAP_QT = 500, //!< QuickTime - CAP_UNICAP = 600, //!< Unicap drivers - CAP_DSHOW = 700, //!< DirectShow (via videoInput) - CAP_PVAPI = 800, //!< PvAPI, Prosilica GigE SDK - CAP_OPENNI = 900, //!< OpenNI (for Kinect) - CAP_OPENNI_ASUS = 910, //!< OpenNI (for Asus Xtion) - CAP_ANDROID = 1000, //!< Android - not used - CAP_XIAPI = 1100, //!< XIMEA Camera API - CAP_AVFOUNDATION = 1200, //!< AVFoundation framework for iOS - CAP_AVFOUNDATION_MAC = 1210, //!< AVFoundation framework for macOS - CAP_GIGANETIX = 1300, //!< Smartek Giganetix GigEVisionSDK - CAP_MSMF = 1400, //!< Microsoft Media Foundation (via videoInput) - CAP_WINRT = 1410, //!< Microsoft Windows Runtime using Media Foundation - CAP_INTELPERC = 1500, //!< Intel Perceptual Computing SDK - CAP_OPENNI2 = 1600, //!< OpenNI2 (for Kinect) - CAP_OPENNI2_ASUS = 1610, //!< OpenNI2 (for Asus Xtion and Occipital Structure sensors) - CAP_GPHOTO2 = 1700, //!< gPhoto2 connection - CAP_GSTREAMER = 1800, //!< GStreamer - CAP_FFMPEG = 1900, //!< FFMPEG - CAP_IMAGES = 2000 //!< OpenCV Image Sequence (e.g. img_%02d.jpg) + CAP_ANY = 0, //!< Auto detect + CAP_VFW = 200, //!< Video For Windows (platform native) + CAP_V4L = 200, //!< V4L/V4L2 capturing support via libv4l + CAP_V4L2 = CAP_V4L, //!< Same as CAP_V4L + CAP_FIREWIRE = 300, //!< IEEE 1394 drivers + CAP_FIREWARE = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_IEEE1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_DC1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_CMU1394 = CAP_FIREWIRE, //!< Same as CAP_FIREWIRE + CAP_QT = 500, //!< QuickTime + CAP_UNICAP = 600, //!< Unicap drivers + CAP_DSHOW = 700, //!< DirectShow (via videoInput) + CAP_PVAPI = 800, //!< PvAPI, Prosilica GigE SDK + CAP_OPENNI = 900, //!< OpenNI (for Kinect) + CAP_OPENNI_ASUS = 910, //!< OpenNI (for Asus Xtion) + CAP_ANDROID = 1000, //!< Android - not used + CAP_XIAPI = 1100, //!< XIMEA Camera API + CAP_AVFOUNDATION = 1200, //!< AVFoundation framework for iOS (OS X Lion will have the same API) + CAP_GIGANETIX = 1300, //!< Smartek Giganetix GigEVisionSDK + CAP_MSMF = 1400, //!< Microsoft Media Foundation (via videoInput) + CAP_WINRT = 1410, //!< Microsoft Windows Runtime using Media Foundation + CAP_INTELPERC = 1500, //!< Intel Perceptual Computing SDK + CAP_OPENNI2 = 1600, //!< OpenNI2 (for Kinect) + CAP_OPENNI2_ASUS = 1610, //!< OpenNI2 (for Asus Xtion and Occipital Structure sensors) + CAP_GPHOTO2 = 1700, //!< gPhoto2 connection + CAP_GSTREAMER = 1800, //!< GStreamer + CAP_FFMPEG = 1900, //!< FFMPEG + CAP_IMAGES = 2000 //!< OpenCV Image Sequence (e.g. img_%02d.jpg) }; //! generic properties (based on DC1394 properties) diff --git a/modules/videoio/include/opencv2/videoio/videoio_c.h b/modules/videoio/include/opencv2/videoio/videoio_c.h index 1351c4e73a..e502c7be83 100644 --- a/modules/videoio/include/opencv2/videoio/videoio_c.h +++ b/modules/videoio/include/opencv2/videoio/videoio_c.h @@ -108,7 +108,6 @@ enum CV_CAP_XIAPI =1100, // XIMEA Camera API CV_CAP_AVFOUNDATION = 1200, // AVFoundation framework for iOS (OS X Lion will have the same API) - CV_CAP_AVFOUNDATION_MAC = 1210, // AVFoundation framework for Mac CV_CAP_GIGANETIX = 1300, // Smartek Giganetix GigEVisionSDK diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index 8e12b62da0..55c20bef52 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -266,12 +266,6 @@ CV_IMPL CvCapture * cvCreateCameraCapture (int index) if (pref) break; #endif -#ifdef HAVE_AVFOUNDATION_MAC - case CV_CAP_AVFOUNDATION_MAC: - TRY_OPEN(capture, cvCreateCameraCapture_AVFoundation_Mac(index)) - if (pref) break; -#endif - #ifdef HAVE_GIGE_API case CV_CAP_GIGANETIX: TRY_OPEN(capture, cvCreateCameraCapture_Giganetix(index)) @@ -340,12 +334,6 @@ CV_IMPL CvCapture * cvCreateFileCaptureWithPreference (const char * filename, in if (apiPreference) break; #endif -#ifdef HAVE_AVFOUNDATION_MAC - case CV_CAP_AVFOUNDATION_MAC: - TRY_OPEN(result, cvCreateFileCapture_AVFoundation_Mac(filename)) - if (apiPreference) break; -#endif - #ifdef HAVE_OPENNI case CV_CAP_OPENNI: TRY_OPEN(result, cvCreateFileCapture_OpenNI (filename)) @@ -403,10 +391,6 @@ CV_IMPL CvVideoWriter* cvCreateVideoWriter( const char* filename, int fourcc, TRY_OPEN(result, cvCreateVideoWriter_AVFoundation(filename, fourcc, fps, frameSize, is_color)) #endif -#ifdef HAVE_AVFOUNDATION_MAC - TRY_OPEN(result, cvCreateVideoWriter_AVFoundation_Mac(filename, fourcc, fps, frameSize, is_color)) -#endif - #if defined(HAVE_QUICKTIME) || defined(HAVE_QTKIT) TRY_OPEN(result, cvCreateVideoWriter_QT(filename, fourcc, fps, frameSize, is_color)) #endif diff --git a/modules/videoio/src/cap_avfoundation_mac.mm b/modules/videoio/src/cap_avfoundation_mac.mm index 4228bc3735..8c30506400 100644 --- a/modules/videoio/src/cap_avfoundation_mac.mm +++ b/modules/videoio/src/cap_avfoundation_mac.mm @@ -171,18 +171,18 @@ private: /***************************************************************************** * - * CvVideoWriter_AVFoundation_Mac Declaration. + * CvVideoWriter_AVFoundation Declaration. * - * CvVideoWriter_AVFoundation_Mac is the instantiation of a video output class. + * CvVideoWriter_AVFoundation is the instantiation of a video output class. * *****************************************************************************/ -class CvVideoWriter_AVFoundation_Mac : public CvVideoWriter { +class CvVideoWriter_AVFoundation : public CvVideoWriter { public: - CvVideoWriter_AVFoundation_Mac(const char* filename, int fourcc, + CvVideoWriter_AVFoundation(const char* filename, int fourcc, double fps, CvSize frame_size, int is_color=1); - ~CvVideoWriter_AVFoundation_Mac(); + ~CvVideoWriter_AVFoundation(); bool writeFrame(const IplImage* image); private: IplImage* argbimage; @@ -203,7 +203,7 @@ class CvVideoWriter_AVFoundation_Mac : public CvVideoWriter { /****************** Implementation of interface functions ********************/ -CvCapture* cvCreateFileCapture_AVFoundation_Mac(const char* filename) { +CvCapture* cvCreateFileCapture_AVFoundation(const char* filename) { CvCaptureFile *retval = new CvCaptureFile(filename); if(retval->didStart()) @@ -212,17 +212,17 @@ CvCapture* cvCreateFileCapture_AVFoundation_Mac(const char* filename) { return NULL; } -CvCapture* cvCreateCameraCapture_AVFoundation_Mac(int index ) { +CvCapture* cvCreateCameraCapture_AVFoundation(int index ) { CvCapture* retval = new CvCaptureCAM(index); if (!((CvCaptureCAM *)retval)->didStart()) cvReleaseCapture(&retval); return retval; } -CvVideoWriter* cvCreateVideoWriter_AVFoundation_Mac(const char* filename, int fourcc, +CvVideoWriter* cvCreateVideoWriter_AVFoundation(const char* filename, int fourcc, double fps, CvSize frame_size, int is_color) { - return new CvVideoWriter_AVFoundation_Mac(filename, fourcc, fps, frame_size,is_color); + return new CvVideoWriter_AVFoundation(filename, fourcc, fps, frame_size,is_color); } /********************** Implementation of Classes ****************************/ @@ -1087,14 +1087,14 @@ bool CvCaptureFile::setProperty(int property_id, double value) { /***************************************************************************** * - * CvVideoWriter_AVFoundation_Mac Implementation. + * CvVideoWriter_AVFoundation Implementation. * - * CvVideoWriter_AVFoundation_Mac is the instantiation of a video output class. + * CvVideoWriter_AVFoundation is the instantiation of a video output class. * *****************************************************************************/ -CvVideoWriter_AVFoundation_Mac::CvVideoWriter_AVFoundation_Mac(const char* filename, int fourcc, +CvVideoWriter_AVFoundation::CvVideoWriter_AVFoundation(const char* filename, int fourcc, double fps, CvSize frame_size, int is_color) { @@ -1222,7 +1222,7 @@ CvVideoWriter_AVFoundation_Mac::CvVideoWriter_AVFoundation_Mac(const char* filen } -CvVideoWriter_AVFoundation_Mac::~CvVideoWriter_AVFoundation_Mac() { +CvVideoWriter_AVFoundation::~CvVideoWriter_AVFoundation() { NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; [mMovieWriterInput markAsFinished]; @@ -1243,7 +1243,7 @@ static void releaseCallback( void *releaseRefCon, const void * ) { CFRelease((CFDataRef)releaseRefCon); } -bool CvVideoWriter_AVFoundation_Mac::writeFrame(const IplImage* iplimage) { +bool CvVideoWriter_AVFoundation::writeFrame(const IplImage* iplimage) { NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; // writer status check diff --git a/modules/videoio/src/precomp.hpp b/modules/videoio/src/precomp.hpp index 50853d0bb9..45ce1a1f60 100644 --- a/modules/videoio/src/precomp.hpp +++ b/modules/videoio/src/precomp.hpp @@ -127,7 +127,6 @@ CvCapture* cvCreateFileCapture_OpenNI( const char* filename ); CvCapture* cvCreateCameraCapture_Android( int index ); CvCapture* cvCreateCameraCapture_XIMEA( int index ); CvCapture* cvCreateCameraCapture_AVFoundation(int index); -CvCapture* cvCreateCameraCapture_AVFoundation_Mac(int index); CvCapture* cvCreateFileCapture_Images(const char* filename); CvVideoWriter* cvCreateVideoWriter_Images(const char* filename); @@ -156,9 +155,6 @@ CvVideoWriter* cvCreateVideoWriter_QT ( const char* filename, int fourcc, CvCapture* cvCreateFileCapture_AVFoundation (const char * filename); CvVideoWriter* cvCreateVideoWriter_AVFoundation( const char* filename, int fourcc, double fps, CvSize frameSize, int is_color ); -CvCapture* cvCreateFileCapture_AVFoundation_Mac(const char * filename); -CvVideoWriter* cvCreateVideoWriter_AVFoundation_Mac( const char* filename, int fourcc, - double fps, CvSize frameSize, int is_color ); CvCapture * cvCreateCameraCapture_Unicap (const int index);