mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-24 03:00:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #14005 from komakai:android-video-cap

Browse Source 
* Add Android Media NDK video i/o file capture back-end

* Fix failing test

* Improve error handling/prevent resource leaks

* Add license text

* Modify default for WITH_ANDROID_MEDIANDK option

* Fix spelling of deleter_AMediaExtractor
This commit is contained in:
Giles Payne 2019-03-20 03:28:45 +09:00 committed by Alexander Alekhin
parent 9a202c1522
commit 0d1a0b126b
8 changed files with 279 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -410,6 +410,9 @@ OCV_OPTION(WITH_IMGCODEC_PFM "Include PFM formats support" ON
OCV_OPTION(WITH_QUIRC "Include library QR-code decoding" ON
VISIBLE_IF TRUE
VERIFY HAVE_QUIRC)
OCV_OPTION(WITH_ANDROID_MEDIANDK "Use Android Media NDK for Video I/O (Android)" (ANDROID_NATIVE_API_LEVEL GREATER 20)
VISIBLE_IF ANDROID
VERIFY HAVE_ANDROID_MEDIANDK)
# OpenCV build components
# ===================================================

6
modules/videoio/CMakeLists.txt
View File

@ -173,6 +173,12 @@ if(TARGET ocv.3rdparty.cap_ios)
list(APPEND tgts ocv.3rdparty.cap_ios)
endif()
if(TARGET ocv.3rdparty.android_mediandk)
list(APPEND videoio_srcs
${CMAKE_CURRENT_LIST_DIR}/src/cap_android_mediandk.cpp)
list(APPEND tgts ocv.3rdparty.android_mediandk)
endif()
ocv_set_module_sources(HEADERS ${videoio_ext_hdrs} ${videoio_hdrs} SOURCES ${videoio_srcs})
ocv_module_include_directories()
ocv_create_module()

8
modules/videoio/cmake/detect_android_mediandk.cmake Normal file
View File

@ -0,0 +1,8 @@
# if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER_EQUAL 21) <-- would be nicer but requires CMake 3.7 or later
if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 20)
set(HAVE_ANDROID_MEDIANDK TRUE)
set(libs "-landroid -llog -lmediandk")
ocv_add_external_target(android_mediandk "" "${libs}" "HAVE_ANDROID_MEDIANDK")
endif()
set(HAVE_ANDROID_MEDIANDK ${HAVE_ANDROID_MEDIANDK} PARENT_SCOPE)

2
modules/videoio/cmake/init.cmake
View File

@ -40,3 +40,5 @@ add_backend("ios" WITH_CAP_IOS)
add_backend("dshow" WITH_DSHOW)
add_backend("msmf" WITH_MSMF)
add_backend("android_mediandk" WITH_ANDROID_MEDIANDK)

248
modules/videoio/src/cap_android_mediandk.cpp Normal file
View File

@ -0,0 +1,248 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html
#include "precomp.hpp"
#include <stdio.h>
#include <string.h>
#include <fstream>
#include <iostream>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <android/log.h>
#include "media/NdkMediaCodec.h"
#include "media/NdkMediaExtractor.h"
#define INPUT_TIMEOUT_MS 2000
#define COLOR_FormatYUV420Planar 19
#define COLOR_FormatYUV420SemiPlanar 21
using namespace cv;
#define TAG "NativeCodec"
#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
static inline void deleter_AMediaExtractor(AMediaExtractor *extractor) {
AMediaExtractor_delete(extractor);
}
static inline void deleter_AMediaCodec(AMediaCodec *codec) {
AMediaCodec_stop(codec);
AMediaCodec_delete(codec);
}
static inline void deleter_AMediaFormat(AMediaFormat *format) {
AMediaFormat_delete(format);
}
class AndroidMediaNdkCapture : public IVideoCapture
{
public:
AndroidMediaNdkCapture():
sawInputEOS(false), sawOutputEOS(false),
frameWidth(0), frameHeight(0), colorFormat(0) {}
std::shared_ptr<AMediaExtractor> mediaExtractor;
std::shared_ptr<AMediaCodec> mediaCodec;
bool sawInputEOS;
bool sawOutputEOS;
int32_t frameWidth;
int32_t frameHeight;
int32_t colorFormat;
std::vector<uint8_t> buffer;
~AndroidMediaNdkCapture() { cleanUp(); }
bool decodeFrame() {
while (!sawInputEOS || !sawOutputEOS) {
if (!sawInputEOS) {
auto bufferIndex = AMediaCodec_dequeueInputBuffer(mediaCodec.get(), INPUT_TIMEOUT_MS);
LOGV("input buffer %zd", bufferIndex);
if (bufferIndex >= 0) {
size_t bufferSize;
auto inputBuffer = AMediaCodec_getInputBuffer(mediaCodec.get(), bufferIndex, &bufferSize);
auto sampleSize = AMediaExtractor_readSampleData(mediaExtractor.get(), inputBuffer, bufferSize);
if (sampleSize < 0) {
sampleSize = 0;
sawInputEOS = true;
LOGV("EOS");
}
auto presentationTimeUs = AMediaExtractor_getSampleTime(mediaExtractor.get());
AMediaCodec_queueInputBuffer(mediaCodec.get(), bufferIndex, 0, sampleSize,
presentationTimeUs, sawInputEOS ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0);
AMediaExtractor_advance(mediaExtractor.get());
}
}
if (!sawOutputEOS) {
AMediaCodecBufferInfo info;
auto bufferIndex = AMediaCodec_dequeueOutputBuffer(mediaCodec.get(), &info, 0);
if (bufferIndex >= 0) {
size_t bufferSize = 0;
auto mediaFormat = std::shared_ptr<AMediaFormat>(AMediaCodec_getOutputFormat(mediaCodec.get()), deleter_AMediaFormat);
AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_WIDTH, &frameWidth);
AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_HEIGHT, &frameHeight);
AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat);
uint8_t* codecBuffer = AMediaCodec_getOutputBuffer(mediaCodec.get(), bufferIndex, &bufferSize);
buffer = std::vector<uint8_t>(codecBuffer + info.offset, codecBuffer + bufferSize);
LOGV("colorFormat: %d", colorFormat);
LOGV("buffer size: %zu", bufferSize);
LOGV("width (frame): %d", frameWidth);
LOGV("height (frame): %d", frameHeight);
if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
LOGV("output EOS");
sawOutputEOS = true;
}
AMediaCodec_releaseOutputBuffer(mediaCodec.get(), bufferIndex, info.size != 0);
return true;
} else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
LOGV("output buffers changed");
} else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
auto format = AMediaCodec_getOutputFormat(mediaCodec.get());
LOGV("format changed to: %s", AMediaFormat_toString(format));
AMediaFormat_delete(format);
} else if (bufferIndex == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
LOGV("no output buffer right now");
} else {
LOGV("unexpected info code: %zd", bufferIndex);
}
}
}
return false;
}
bool isOpened() const CV_OVERRIDE { return mediaCodec.get() != nullptr; }
int getCaptureDomain() CV_OVERRIDE { return CAP_ANDROID; }
bool grabFrame() CV_OVERRIDE
{
// clear the previous frame
buffer.clear();
return decodeFrame();
}
bool retrieveFrame(int, OutputArray out) CV_OVERRIDE
{
if (buffer.empty()) {
return false;
}
Mat yuv(frameHeight + frameHeight/2, frameWidth, CV_8UC1, buffer.data());
if (colorFormat == COLOR_FormatYUV420Planar) {
cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_YV12);
} else if (colorFormat == COLOR_FormatYUV420SemiPlanar) {
cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_NV21);
} else {
LOGE("Unsupported video format: %d", colorFormat);
return false;
}
return true;
}
double getProperty(int property_id) const CV_OVERRIDE
{
switch (property_id)
{
case CV_CAP_PROP_FRAME_WIDTH: return frameWidth;
case CV_CAP_PROP_FRAME_HEIGHT: return frameHeight;
}
return 0;
}
bool setProperty(int /* property_id */, double /* value */) CV_OVERRIDE
{
return false;
}
bool initCapture(const char * filename)
{
struct stat statBuffer;
if (stat(filename, &statBuffer) != 0) {
LOGE("failed to stat file: %s (%s)", filename, strerror(errno));
return false;
}
int fd = open(filename, O_RDONLY);
if (fd < 0) {
LOGE("failed to open file: %s %d (%s)", filename, fd, strerror(errno));
return false;
}
mediaExtractor = std::shared_ptr<AMediaExtractor>(AMediaExtractor_new(), deleter_AMediaExtractor);
if (!mediaExtractor) {
return false;
}
media_status_t err = AMediaExtractor_setDataSourceFd(mediaExtractor.get(), fd, 0, statBuffer.st_size);
close(fd);
if (err != AMEDIA_OK) {
LOGV("setDataSource error: %d", err);
return false;
}
int numtracks = AMediaExtractor_getTrackCount(mediaExtractor.get());
LOGV("input has %d tracks", numtracks);
for (int i = 0; i < numtracks; i++) {
auto format = std::shared_ptr<AMediaFormat>(AMediaExtractor_getTrackFormat(mediaExtractor.get(), i), deleter_AMediaFormat);
if (!format) {
continue;
}
const char *s = AMediaFormat_toString(format.get());
LOGV("track %d format: %s", i, s);
const char *mime;
if (!AMediaFormat_getString(format.get(), AMEDIAFORMAT_KEY_MIME, &mime)) {
LOGV("no mime type");
} else if (!strncmp(mime, "video/", 6)) {
int32_t trackWidth, trackHeight;
AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_WIDTH, &trackWidth);
AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_HEIGHT, &trackHeight);
LOGV("width (track): %d", trackWidth);
LOGV("height (track): %d", trackHeight);
if (AMediaExtractor_selectTrack(mediaExtractor.get(), i) != AMEDIA_OK) {
continue;
}
mediaCodec = std::shared_ptr<AMediaCodec>(AMediaCodec_createDecoderByType(mime), deleter_AMediaCodec);
if (!mediaCodec) {
continue;
}
if (AMediaCodec_configure(mediaCodec.get(), format.get(), NULL, NULL, 0) != AMEDIA_OK) {
continue;
}
sawInputEOS = false;
sawOutputEOS = false;
if (AMediaCodec_start(mediaCodec.get()) != AMEDIA_OK) {
continue;
}
return true;
}
}
return false;
}
void cleanUp() {
sawInputEOS = true;
sawOutputEOS = true;
frameWidth = 0;
frameHeight = 0;
colorFormat = 0;
}
};
/****************** Implementation of interface functions ********************/
Ptr<IVideoCapture> cv::createAndroidCapture_file(const std::string &filename) {
Ptr<AndroidMediaNdkCapture> res = makePtr<AndroidMediaNdkCapture>();
if (res && res->initCapture(filename.c_str()))
return res;
return Ptr<IVideoCapture>();
}

2
modules/videoio/src/cap_interface.hpp
View File

@ -206,6 +206,8 @@ Ptr<IVideoCapture> createGPhoto2Capture(const std::string& deviceName);
Ptr<IVideoCapture> createXINECapture(const std::string &filename);
Ptr<IVideoCapture> createAndroidCapture_file(const std::string &filename);
} // cv::
#endif // CAP_INTERFACE_HPP

6
modules/videoio/src/videoio_registry.cpp
View File

@ -130,8 +130,10 @@ static const struct VideoBackendInfo builtin_backends[] =
#ifdef HAVE_XINE
DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0),
#endif
// dropped backends: MIL, TYZX, Android
#ifdef HAVE_ANDROID_MEDIANDK
DECLARE_STATIC_BACKEND(CAP_ANDROID, "ANDROID_MEDIANDK", MODE_CAPTURE_BY_FILENAME, createAndroidCapture_file, 0, 0),
#endif
// dropped backends: MIL, TYZX
};
bool sortByPriority(const VideoBackendInfo &lhs, const VideoBackendInfo &rhs)

6
platforms/android/ndk-18-api-level-21.config.py Normal file
View File

@ -0,0 +1,6 @@
ABIs = [
ABI("2", "armeabi-v7a", None, 21, cmake_vars=dict(ANDROID_ABI='armeabi-v7a with NEON')),
ABI("3", "arm64-v8a", None, 21),
ABI("5", "x86_64", None, 21),
ABI("4", "x86", None, 21),
]