mirror of
https://github.com/opencv/opencv.git
synced 2025-06-06 00:43:52 +08:00
Merge pull request #10081 from floe:java-camera2-view
* add (untested) JavaCamera2View class * initial fixes * minor cleanup * exclude JavaCamera2View from build for older SDK version * fix method name typo * add asserts + sanity checks * fix preview format checks * fix the memory leak * export cvtTwoPlaneYUVtoBGR for Java usage * add handling for two-plane YUV frames (C wrapper still missing) * add two-plane cvtColor helper function * fix warnings * actually use the new cvtColorTwoPlane helper func * fix wrong output matrix size * fix wrong conversion type * simplify method signature, add error condition * minor fixes to Mat types * remove leftover semaphore from camera api 1 * android: JavaCamera2View minor refactoring - re-apply Java code style - imports cleanup - dump exceptions information * android: JavaCamera2View: pause/resume fixes * android: JavaCamera2View: fix mScale
This commit is contained in:
Florian Echtler
Alexander Alekhin
parent
eb94dfb442
commit
6a8f57e5d7
@ -3694,6 +3694,8 @@ channels is derived automatically from src and code.
|
||||
*/
|
||||
CV_EXPORTS_W void cvtColor( InputArray src, OutputArray dst, int code, int dstCn = 0 );
|
||||
|
||||
CV_EXPORTS_W void cvtColorTwoPlane( InputArray src1, InputArray src2, OutputArray dst, int code );
|
||||
|
||||
//! @} imgproc_misc
|
||||
|
||||
// main function for all demosaicing processes
|
||||
|
||||
@ -11050,6 +11050,42 @@ inline bool isFullRange(int code)
|
||||
|
||||
} // namespace::
|
||||
|
||||
// helper function for dual-plane modes
|
||||
void cv::cvtColorTwoPlane( InputArray _ysrc, InputArray _uvsrc, OutputArray _dst, int code )
|
||||
{
|
||||
// only YUV420 is currently supported
|
||||
switch (code) {
|
||||
case COLOR_YUV2BGR_NV21: case COLOR_YUV2RGB_NV21: case COLOR_YUV2BGR_NV12: case COLOR_YUV2RGB_NV12:
|
||||
case COLOR_YUV2BGRA_NV21: case COLOR_YUV2RGBA_NV21: case COLOR_YUV2BGRA_NV12: case COLOR_YUV2RGBA_NV12:
|
||||
break;
|
||||
default:
|
||||
CV_Error( CV_StsBadFlag, "Unknown/unsupported color conversion code" );
|
||||
return;
|
||||
}
|
||||
|
||||
int stype = _ysrc.type();
|
||||
int depth = CV_MAT_DEPTH(stype), uidx, dcn;
|
||||
|
||||
Mat ysrc, uvsrc, dst;
|
||||
ysrc = _ysrc.getMat();
|
||||
uvsrc = _uvsrc.getMat();
|
||||
Size ysz = _ysrc.size();
|
||||
Size uvs = _uvsrc.size();
|
||||
|
||||
// http://www.fourcc.org/yuv.php#NV21 == yuv420sp -> a plane of 8 bit Y samples followed by an interleaved V/U plane containing 8 bit 2x2 subsampled chroma samples
|
||||
// http://www.fourcc.org/yuv.php#NV12 -> a plane of 8 bit Y samples followed by an interleaved U/V plane containing 8 bit 2x2 subsampled colour difference samples
|
||||
dcn = (code==COLOR_YUV420sp2BGRA || code==COLOR_YUV420sp2RGBA || code==COLOR_YUV2BGRA_NV12 || code==COLOR_YUV2RGBA_NV12) ? 4 : 3;
|
||||
uidx = (code==COLOR_YUV2BGR_NV21 || code==COLOR_YUV2BGRA_NV21 || code==COLOR_YUV2RGB_NV21 || code==COLOR_YUV2RGBA_NV21) ? 1 : 0;
|
||||
CV_Assert( dcn == 3 || dcn == 4 );
|
||||
CV_Assert( ysz.width == uvs.width * 2 );
|
||||
CV_Assert( ysz.width % 2 == 0 && depth == CV_8U );
|
||||
CV_Assert( ysz.height == uvs.height * 2 );
|
||||
_dst.create( ysz, CV_MAKETYPE(depth, dcn));
|
||||
dst = _dst.getMat();
|
||||
hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, dst.data, dst.step, dst.cols, dst.rows, dcn, swapBlue(code), uidx);
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// The main function //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@ -0,0 +1,374 @@
|
||||
package org.opencv.android;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Arrays;
|
||||
|
||||
import android.annotation.TargetApi;
|
||||
import android.content.Context;
|
||||
import android.graphics.ImageFormat;
|
||||
import android.hardware.camera2.CameraAccessException;
|
||||
import android.hardware.camera2.CameraCaptureSession;
|
||||
import android.hardware.camera2.CameraCharacteristics;
|
||||
import android.hardware.camera2.CameraDevice;
|
||||
import android.hardware.camera2.CameraManager;
|
||||
import android.hardware.camera2.CaptureRequest;
|
||||
import android.hardware.camera2.params.StreamConfigurationMap;
|
||||
import android.media.Image;
|
||||
import android.media.ImageReader;
|
||||
import android.os.Handler;
|
||||
import android.os.HandlerThread;
|
||||
import android.util.AttributeSet;
|
||||
import android.util.Log;
|
||||
import android.view.Surface;
|
||||
import android.view.ViewGroup.LayoutParams;
|
||||
|
||||
import org.opencv.core.CvType;
|
||||
import org.opencv.core.Mat;
|
||||
import org.opencv.imgproc.Imgproc;
|
||||
|
||||
/**
|
||||
* This class is an implementation of the Bridge View between OpenCV and Java Camera.
|
||||
* This class relays on the functionality available in base class and only implements
|
||||
* required functions:
|
||||
* connectCamera - opens Java camera and sets the PreviewCallback to be delivered.
|
||||
* disconnectCamera - closes the camera and stops preview.
|
||||
* When frame is delivered via callback from Camera - it processed via OpenCV to be
|
||||
* converted to RGBA32 and then passed to the external callback for modifications if required.
|
||||
*/
|
||||
|
||||
@TargetApi(21)
|
||||
public class JavaCamera2View extends CameraBridgeViewBase {
|
||||
|
||||
private static final String LOGTAG = "JavaCamera2View";
|
||||
|
||||
private ImageReader mImageReader;
|
||||
private int mPreviewFormat = ImageFormat.YUV_420_888;
|
||||
|
||||
private CameraDevice mCameraDevice;
|
||||
private CameraCaptureSession mCaptureSession;
|
||||
private CaptureRequest.Builder mPreviewRequestBuilder;
|
||||
private String mCameraID;
|
||||
private android.util.Size mPreviewSize = new android.util.Size(-1, -1);
|
||||
|
||||
private HandlerThread mBackgroundThread;
|
||||
private Handler mBackgroundHandler;
|
||||
|
||||
public JavaCamera2View(Context context, int cameraId) {
|
||||
super(context, cameraId);
|
||||
}
|
||||
|
||||
public JavaCamera2View(Context context, AttributeSet attrs) {
|
||||
super(context, attrs);
|
||||
}
|
||||
|
||||
private void startBackgroundThread() {
|
||||
Log.i(LOGTAG, "startBackgroundThread");
|
||||
stopBackgroundThread();
|
||||
mBackgroundThread = new HandlerThread("OpenCVCameraBackground");
|
||||
mBackgroundThread.start();
|
||||
mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
|
||||
}
|
||||
|
||||
private void stopBackgroundThread() {
|
||||
Log.i(LOGTAG, "stopBackgroundThread");
|
||||
if (mBackgroundThread == null)
|
||||
return;
|
||||
mBackgroundThread.quitSafely();
|
||||
try {
|
||||
mBackgroundThread.join();
|
||||
mBackgroundThread = null;
|
||||
mBackgroundHandler = null;
|
||||
} catch (InterruptedException e) {
|
||||
Log.e(LOGTAG, "stopBackgroundThread", e);
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean initializeCamera() {
|
||||
Log.i(LOGTAG, "initializeCamera");
|
||||
CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
|
||||
try {
|
||||
String camList[] = manager.getCameraIdList();
|
||||
if (camList.length == 0) {
|
||||
Log.e(LOGTAG, "Error: camera isn't detected.");
|
||||
return false;
|
||||
}
|
||||
if (mCameraIndex == CameraBridgeViewBase.CAMERA_ID_ANY) {
|
||||
mCameraID = camList[0];
|
||||
} else {
|
||||
for (String cameraID : camList) {
|
||||
CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraID);
|
||||
if ((mCameraIndex == CameraBridgeViewBase.CAMERA_ID_BACK &&
|
||||
characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_BACK) ||
|
||||
(mCameraIndex == CameraBridgeViewBase.CAMERA_ID_FRONT &&
|
||||
characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT)
|
||||
) {
|
||||
mCameraID = cameraID;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (mCameraID != null) {
|
||||
Log.i(LOGTAG, "Opening camera: " + mCameraID);
|
||||
manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
|
||||
}
|
||||
return true;
|
||||
} catch (CameraAccessException e) {
|
||||
Log.e(LOGTAG, "OpenCamera - Camera Access Exception", e);
|
||||
} catch (IllegalArgumentException e) {
|
||||
Log.e(LOGTAG, "OpenCamera - Illegal Argument Exception", e);
|
||||
} catch (SecurityException e) {
|
||||
Log.e(LOGTAG, "OpenCamera - Security Exception", e);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private final CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback() {
|
||||
|
||||
@Override
|
||||
public void onOpened(CameraDevice cameraDevice) {
|
||||
mCameraDevice = cameraDevice;
|
||||
createCameraPreviewSession();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDisconnected(CameraDevice cameraDevice) {
|
||||
cameraDevice.close();
|
||||
mCameraDevice = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(CameraDevice cameraDevice, int error) {
|
||||
cameraDevice.close();
|
||||
mCameraDevice = null;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
private void createCameraPreviewSession() {
|
||||
final int w = mPreviewSize.getWidth(), h = mPreviewSize.getHeight();
|
||||
Log.i(LOGTAG, "createCameraPreviewSession(" + w + "x" + h + ")");
|
||||
if (w < 0 || h < 0)
|
||||
return;
|
||||
try {
|
||||
if (null == mCameraDevice) {
|
||||
Log.e(LOGTAG, "createCameraPreviewSession: camera isn't opened");
|
||||
return;
|
||||
}
|
||||
if (null != mCaptureSession) {
|
||||
Log.e(LOGTAG, "createCameraPreviewSession: mCaptureSession is already started");
|
||||
return;
|
||||
}
|
||||
|
||||
mImageReader = ImageReader.newInstance(w, h, mPreviewFormat, 2);
|
||||
mImageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() {
|
||||
@Override
|
||||
public void onImageAvailable(ImageReader reader) {
|
||||
Image image = reader.acquireLatestImage();
|
||||
if (image == null)
|
||||
return;
|
||||
|
||||
// sanity checks - 3 planes
|
||||
Image.Plane[] planes = image.getPlanes();
|
||||
assert (planes.length == 3);
|
||||
assert (image.getFormat() == mPreviewFormat);
|
||||
|
||||
// see also https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
|
||||
// Y plane (0) non-interleaved => stride == 1; U/V plane interleaved => stride == 2
|
||||
assert (planes[0].getPixelStride() == 1);
|
||||
assert (planes[1].getPixelStride() == 2);
|
||||
assert (planes[2].getPixelStride() == 2);
|
||||
|
||||
ByteBuffer y_plane = planes[0].getBuffer();
|
||||
ByteBuffer uv_plane = planes[1].getBuffer();
|
||||
Mat y_mat = new Mat(h, w, CvType.CV_8UC1, y_plane);
|
||||
Mat uv_mat = new Mat(h / 2, w / 2, CvType.CV_8UC2, uv_plane);
|
||||
JavaCamera2Frame tempFrame = new JavaCamera2Frame(y_mat, uv_mat, w, h);
|
||||
deliverAndDrawFrame(tempFrame);
|
||||
tempFrame.release();
|
||||
image.close();
|
||||
}
|
||||
}, mBackgroundHandler);
|
||||
Surface surface = mImageReader.getSurface();
|
||||
|
||||
mPreviewRequestBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
|
||||
mPreviewRequestBuilder.addTarget(surface);
|
||||
|
||||
mCameraDevice.createCaptureSession(Arrays.asList(surface),
|
||||
new CameraCaptureSession.StateCallback() {
|
||||
@Override
|
||||
public void onConfigured(CameraCaptureSession cameraCaptureSession) {
|
||||
Log.i(LOGTAG, "createCaptureSession::onConfigured");
|
||||
if (null == mCameraDevice) {
|
||||
return; // camera is already closed
|
||||
}
|
||||
mCaptureSession = cameraCaptureSession;
|
||||
try {
|
||||
mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE,
|
||||
CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
|
||||
mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AE_MODE,
|
||||
CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH);
|
||||
|
||||
mCaptureSession.setRepeatingRequest(mPreviewRequestBuilder.build(), null, mBackgroundHandler);
|
||||
Log.i(LOGTAG, "CameraPreviewSession has been started");
|
||||
} catch (Exception e) {
|
||||
Log.e(LOGTAG, "createCaptureSession failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onConfigureFailed(CameraCaptureSession cameraCaptureSession) {
|
||||
Log.e(LOGTAG, "createCameraPreviewSession failed");
|
||||
}
|
||||
},
|
||||
null
|
||||
);
|
||||
} catch (CameraAccessException e) {
|
||||
Log.e(LOGTAG, "createCameraPreviewSession", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void disconnectCamera() {
|
||||
Log.i(LOGTAG, "closeCamera");
|
||||
try {
|
||||
CameraDevice c = mCameraDevice;
|
||||
mCameraDevice = null;
|
||||
if (null != mCaptureSession) {
|
||||
mCaptureSession.close();
|
||||
mCaptureSession = null;
|
||||
}
|
||||
if (null != c) {
|
||||
c.close();
|
||||
}
|
||||
if (null != mImageReader) {
|
||||
mImageReader.close();
|
||||
mImageReader = null;
|
||||
}
|
||||
} finally {
|
||||
stopBackgroundThread();
|
||||
}
|
||||
}
|
||||
|
||||
boolean calcPreviewSize(final int width, final int height) {
|
||||
Log.i(LOGTAG, "calcPreviewSize: " + width + "x" + height);
|
||||
if (mCameraID == null) {
|
||||
Log.e(LOGTAG, "Camera isn't initialized!");
|
||||
return false;
|
||||
}
|
||||
CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
|
||||
try {
|
||||
CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraID);
|
||||
StreamConfigurationMap map = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
|
||||
int bestWidth = 0, bestHeight = 0;
|
||||
float aspect = (float) width / height;
|
||||
android.util.Size[] sizes = map.getOutputSizes(ImageReader.class);
|
||||
bestWidth = sizes[0].getWidth();
|
||||
bestHeight = sizes[0].getHeight();
|
||||
for (android.util.Size sz : sizes) {
|
||||
int w = sz.getWidth(), h = sz.getHeight();
|
||||
Log.d(LOGTAG, "trying size: " + w + "x" + h);
|
||||
if (width >= w && height >= h && bestWidth <= w && bestHeight <= h
|
||||
&& Math.abs(aspect - (float) w / h) < 0.2) {
|
||||
bestWidth = w;
|
||||
bestHeight = h;
|
||||
}
|
||||
}
|
||||
Log.i(LOGTAG, "best size: " + bestWidth + "x" + bestHeight);
|
||||
assert(!(bestWidth == 0 || bestHeight == 0));
|
||||
if (mPreviewSize.getWidth() == bestWidth && mPreviewSize.getHeight() == bestHeight)
|
||||
return false;
|
||||
else {
|
||||
mPreviewSize = new android.util.Size(bestWidth, bestHeight);
|
||||
return true;
|
||||
}
|
||||
} catch (CameraAccessException e) {
|
||||
Log.e(LOGTAG, "calcPreviewSize - Camera Access Exception", e);
|
||||
} catch (IllegalArgumentException e) {
|
||||
Log.e(LOGTAG, "calcPreviewSize - Illegal Argument Exception", e);
|
||||
} catch (SecurityException e) {
|
||||
Log.e(LOGTAG, "calcPreviewSize - Security Exception", e);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean connectCamera(int width, int height) {
|
||||
Log.i(LOGTAG, "setCameraPreviewSize(" + width + "x" + height + ")");
|
||||
startBackgroundThread();
|
||||
initializeCamera();
|
||||
try {
|
||||
boolean needReconfig = calcPreviewSize(width, height);
|
||||
mFrameWidth = mPreviewSize.getWidth();
|
||||
mFrameHeight = mPreviewSize.getHeight();
|
||||
|
||||
if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
|
||||
mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
|
||||
else
|
||||
mScale = 0;
|
||||
|
||||
AllocateCache();
|
||||
|
||||
if (needReconfig) {
|
||||
if (null != mCaptureSession) {
|
||||
Log.d(LOGTAG, "closing existing previewSession");
|
||||
mCaptureSession.close();
|
||||
mCaptureSession = null;
|
||||
}
|
||||
createCameraPreviewSession();
|
||||
}
|
||||
} catch (RuntimeException e) {
|
||||
throw new RuntimeException("Interrupted while setCameraPreviewSize.", e);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private class JavaCamera2Frame implements CvCameraViewFrame {
|
||||
@Override
|
||||
public Mat gray() {
|
||||
return mYuvFrameData.submat(0, mHeight, 0, mWidth);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mat rgba() {
|
||||
if (mPreviewFormat == ImageFormat.NV21)
|
||||
Imgproc.cvtColor(mYuvFrameData, mRgba, Imgproc.COLOR_YUV2RGBA_NV21, 4);
|
||||
else if (mPreviewFormat == ImageFormat.YV12)
|
||||
Imgproc.cvtColor(mYuvFrameData, mRgba, Imgproc.COLOR_YUV2RGB_I420, 4); // COLOR_YUV2RGBA_YV12 produces inverted colors
|
||||
else if (mPreviewFormat == ImageFormat.YUV_420_888) {
|
||||
assert (mUVFrameData != null);
|
||||
Imgproc.cvtColorTwoPlane(mYuvFrameData, mUVFrameData, mRgba, Imgproc.COLOR_YUV2RGBA_NV21);
|
||||
} else
|
||||
throw new IllegalArgumentException("Preview Format can be NV21 or YV12");
|
||||
|
||||
return mRgba;
|
||||
}
|
||||
|
||||
public JavaCamera2Frame(Mat Yuv420sp, int width, int height) {
|
||||
super();
|
||||
mWidth = width;
|
||||
mHeight = height;
|
||||
mYuvFrameData = Yuv420sp;
|
||||
mUVFrameData = null;
|
||||
mRgba = new Mat();
|
||||
}
|
||||
|
||||
public JavaCamera2Frame(Mat Y, Mat UV, int width, int height) {
|
||||
super();
|
||||
mWidth = width;
|
||||
mHeight = height;
|
||||
mYuvFrameData = Y;
|
||||
mUVFrameData = UV;
|
||||
mRgba = new Mat();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
mRgba.release();
|
||||
}
|
||||
|
||||
private Mat mYuvFrameData;
|
||||
private Mat mUVFrameData;
|
||||
private Mat mRgba;
|
||||
private int mWidth;
|
||||
private int mHeight;
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user