Merge pull request #24869 from alexlyulkov:al/android-camera-view-rotate

Added screen rotation support to JavaCamera2View amd NativeCameraView. Fixed JavaCamera2View initialization. #24869

Added automatic image rotation to JavaCamera2View and NativeCameraView so the video preview was matched with screen orientation.
Fixed double preview initialization bug in JavaCamera2View.
Added proper cameraID parsing to NativeCameraView similar to JavaCameraView

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

modules/java/generator/android-21/java/org/opencv/android/JavaCamera2View.java

@@ -46,6 +46,7 @@ public class JavaCamera2View extends CameraBridgeViewBase {
     protected ImageReader mImageReader;
     protected int mPreviewFormat = ImageFormat.YUV_420_888;
     protected int mRequestTemplate = CameraDevice.TEMPLATE_PREVIEW;
+    private int mFrameRotation;
 
     protected CameraDevice mCameraDevice;
     protected CameraCaptureSession mCaptureSession;
@@ -86,8 +87,8 @@ public class JavaCamera2View extends CameraBridgeViewBase {
         }
     }
 
-    protected boolean initializeCamera() {
+    protected boolean selectCamera() {
-        Log.i(LOGTAG, "initializeCamera");
+        Log.i(LOGTAG, "selectCamera");
         CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
         try {
             String camList[] = manager.getCameraIdList();
@@ -110,14 +111,10 @@ public class JavaCamera2View extends CameraBridgeViewBase {
                     }
                 }
             }
-            if (mCameraID != null) {
+            if (mCameraID == null) { // make JavaCamera2View behaves in the same way as JavaCameraView
-                Log.i(LOGTAG, "Opening camera: " + mCameraID);
+                Log.i(LOGTAG, "Selecting camera by index (" + mCameraIndex + ")");
-                manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
-            } else { // make JavaCamera2View behaves in the same way as JavaCameraView
-                Log.i(LOGTAG, "Trying to open camera with the value (" + mCameraIndex + ")");
                 if (mCameraIndex < camList.length) {
                     mCameraID = camList[mCameraIndex];
-                    manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
                 } else {
                     // CAMERA_DISCONNECTED is used when the camera id is no longer valid
                     throw new CameraAccessException(CameraAccessException.CAMERA_DISCONNECTED);
@@ -125,11 +122,11 @@ public class JavaCamera2View extends CameraBridgeViewBase {
             }
             return true;
         } catch (CameraAccessException e) {
-            Log.e(LOGTAG, "OpenCamera - Camera Access Exception", e);
+            Log.e(LOGTAG, "selectCamera - Camera Access Exception", e);
         } catch (IllegalArgumentException e) {
-            Log.e(LOGTAG, "OpenCamera - Illegal Argument Exception", e);
+            Log.e(LOGTAG, "selectCamera - Illegal Argument Exception", e);
         } catch (SecurityException e) {
-            Log.e(LOGTAG, "OpenCamera - Security Exception", e);
+            Log.e(LOGTAG, "selectCamera - Security Exception", e);
         }
         return false;
     }
@@ -204,6 +201,7 @@ public class JavaCamera2View extends CameraBridgeViewBase {
             mImageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() {
                 @Override
                 public void onImageAvailable(ImageReader reader) {
+
                     Image image = reader.acquireLatestImage();
                     if (image == null)
                         return;
@@ -213,8 +211,9 @@ public class JavaCamera2View extends CameraBridgeViewBase {
                     assert (planes.length == 3);
                     assert (image.getFormat() == mPreviewFormat);
 
-                    JavaCamera2Frame tempFrame = new JavaCamera2Frame(image);
+                    RotatedCameraFrame tempFrame = new RotatedCameraFrame(new JavaCamera2Frame(image), mFrameRotation);
                     deliverAndDrawFrame(tempFrame);
+                    tempFrame.mFrame.release();
                     tempFrame.release();
                     image.close();
                 }
@@ -303,11 +302,22 @@ public class JavaCamera2View extends CameraBridgeViewBase {
     protected boolean connectCamera(int width, int height) {
         Log.i(LOGTAG, "setCameraPreviewSize(" + width + "x" + height + ")");
         startBackgroundThread();
-        initializeCamera();
+        selectCamera();
         try {
+            CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
+            CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraID);
+            mFrameRotation = getFrameRotation(
+                    characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT,
+                    characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION));
+
             boolean needReconfig = calcPreviewSize(width, height);
-            mFrameWidth = mPreviewSize.getWidth();
+            if (mFrameRotation % 180 == 0) {
-            mFrameHeight = mPreviewSize.getHeight();
+                mFrameWidth = mPreviewSize.getWidth();
+                mFrameHeight = mPreviewSize.getHeight();
+            } else {
+                mFrameWidth = mPreviewSize.getHeight();
+                mFrameHeight = mPreviewSize.getWidth();
+            }
 
             if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
                 mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
@@ -322,12 +332,16 @@ public class JavaCamera2View extends CameraBridgeViewBase {
                     mCaptureSession.close();
                     mCaptureSession = null;
                 }
-                createCameraPreviewSession();
             }
 
             if (mFpsMeter != null) {
                 mFpsMeter.setResolution(mFrameWidth, mFrameHeight);
             }
+
+            Log.i(LOGTAG, "Opening camera: " + mCameraID);
+            manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
+        } catch (CameraAccessException e) {
+            Log.e(LOGTAG, "OpenCamera - Camera Access Exception", e);
         } catch (RuntimeException e) {
             throw new RuntimeException("Interrupted while setCameraPreviewSize.", e);
         }
@@ -442,6 +456,7 @@ public class JavaCamera2View extends CameraBridgeViewBase {
             mGray = new Mat();
         }
 
+        @Override
         public void release() {
             mRgba.release();
             mGray.release();

modules/java/generator/android-24/java/org/opencv/android/NativeCameraView.java

@@ -10,6 +10,7 @@ import org.opencv.videoio.VideoCapture;
 import org.opencv.videoio.VideoWriter;
 
 import android.content.Context;
+import android.hardware.Camera;
 import android.util.AttributeSet;
 import android.util.Log;
 import android.view.ViewGroup.LayoutParams;
@@ -25,7 +26,7 @@ public class NativeCameraView extends CameraBridgeViewBase {
     private Thread mThread;
 
     protected VideoCapture mCamera;
-    protected NativeCameraFrame mFrame;
+    protected RotatedCameraFrame mFrame;
 
     public NativeCameraView(Context context, int cameraId) {
         super(context, cameraId);
@@ -89,28 +90,65 @@ public class NativeCameraView extends CameraBridgeViewBase {
 
     private boolean initializeCamera(int width, int height) {
         synchronized (this) {
-
+            Camera.CameraInfo cameraInfo = new Camera.CameraInfo();
-            if (mCameraIndex == -1) {
+            int localCameraIndex = mCameraIndex;
+            if (mCameraIndex == CAMERA_ID_ANY) {
                 Log.d(TAG, "Try to open default camera");
-                mCamera = new VideoCapture(0, Videoio.CAP_ANDROID);
+                localCameraIndex = 0;
-            } else {
+            } else if (mCameraIndex == CAMERA_ID_BACK) {
-                Log.d(TAG, "Try to open camera with index " + mCameraIndex);
+                Log.i(TAG, "Trying to open back camera");
-                mCamera = new VideoCapture(mCameraIndex, Videoio.CAP_ANDROID);
+                for (int camIdx = 0; camIdx < Camera.getNumberOfCameras(); ++camIdx) {
+                    Camera.getCameraInfo( camIdx, cameraInfo );
+                    if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_BACK) {
+                        localCameraIndex = camIdx;
+                        break;
+                    }
+                }
+            } else if (mCameraIndex == CAMERA_ID_FRONT) {
+                Log.i(TAG, "Trying to open front camera");
+                for (int camIdx = 0; camIdx < Camera.getNumberOfCameras(); ++camIdx) {
+                    Camera.getCameraInfo( camIdx, cameraInfo );
+                    if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) {
+                        localCameraIndex = camIdx;
+                        break;
+                    }
+                }
             }
 
+            if (localCameraIndex == CAMERA_ID_BACK) {
+                Log.e(TAG, "Back camera not found!");
+                return false;
+            } else if (localCameraIndex == CAMERA_ID_FRONT) {
+                Log.e(TAG, "Front camera not found!");
+                return false;
+            }
+
+            Log.d(TAG, "Try to open camera with index " + localCameraIndex);
+            mCamera = new VideoCapture(localCameraIndex, Videoio.CAP_ANDROID);
+
             if (mCamera == null)
                 return false;
-
             if (mCamera.isOpened() == false)
                 return false;
 
-            mFrame = new NativeCameraFrame(mCamera);
+            if (mCameraIndex != CAMERA_ID_BACK && mCameraIndex != CAMERA_ID_FRONT)
+                Camera.getCameraInfo(localCameraIndex, cameraInfo);
+            int frameRotation = getFrameRotation(
+                    cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT,
+                    cameraInfo.orientation);
+
+            mFrame = new RotatedCameraFrame(new NativeCameraFrame(mCamera), frameRotation);
 
             mCamera.set(Videoio.CAP_PROP_FRAME_WIDTH, width);
             mCamera.set(Videoio.CAP_PROP_FRAME_HEIGHT, height);
 
-            mFrameWidth = (int)mCamera.get(Videoio.CAP_PROP_FRAME_WIDTH);
+            if (frameRotation % 180 == 0) {
-            mFrameHeight = (int)mCamera.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+                mFrameWidth = (int) mCamera.get(Videoio.CAP_PROP_FRAME_WIDTH);
+                mFrameHeight = (int) mCamera.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+            } else {
+                mFrameWidth = (int) mCamera.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+                mFrameHeight = (int) mCamera.get(Videoio.CAP_PROP_FRAME_WIDTH);
+            }
 
             if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
                 mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
@@ -131,7 +169,10 @@ public class NativeCameraView extends CameraBridgeViewBase {
 
     private void releaseCamera() {
         synchronized (this) {
-            if (mFrame != null) mFrame.release();
+            if (mFrame != null) {
+                mFrame.mFrame.release();
+                mFrame.release();
+            }
             if (mCamera != null) mCamera.release();
         }
     }
@@ -162,6 +203,7 @@ public class NativeCameraView extends CameraBridgeViewBase {
             mBgr = new Mat();
         }
 
+        @Override
         public void release() {
             if (mGray != null) mGray.release();
             if (mRgba != null) mRgba.release();

modules/java/generator/android/java/org/opencv/android/CameraBridgeViewBase.java

@@ -4,6 +4,7 @@ import java.util.List;
 
 import org.opencv.BuildConfig;
 import org.opencv.R;
+import org.opencv.core.Core;
 import org.opencv.core.Mat;
 import org.opencv.core.Size;
 
@@ -17,8 +18,10 @@ import android.graphics.Canvas;
 import android.graphics.Rect;
 import android.util.AttributeSet;
 import android.util.Log;
+import android.view.Surface;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
+import android.view.WindowManager;
 
 /**
  * This is a basic class, implementing the interaction with Camera and OpenCV library.
@@ -189,8 +192,93 @@ public abstract class CameraBridgeViewBase extends SurfaceView implements Surfac
          * This method returns single channel gray scale Mat with frame
          */
         public Mat gray();
+
+        public void release();
     };
 
+    public class RotatedCameraFrame implements CvCameraViewFrame {
+        @Override
+        public Mat gray() {
+            if (mRotation != 0) {
+                Core.rotate(mFrame.gray(), mGrayRotated, getCvRotationCode(mRotation));
+                return mGrayRotated;
+            } else {
+                return mFrame.gray();
+            }
+        }
+
+        @Override
+        public Mat rgba() {
+            if (mRotation != 0) {
+                Core.rotate(mFrame.rgba(), mRgbaRotated, getCvRotationCode(mRotation));
+                return mRgbaRotated;
+            } else {
+                return mFrame.rgba();
+            }
+        }
+
+        private int getCvRotationCode(int degrees) {
+            if  (degrees == 90) {
+                return Core.ROTATE_90_CLOCKWISE;
+            } else if (degrees == 180) {
+                return Core.ROTATE_180;
+            } else {
+                return Core.ROTATE_90_COUNTERCLOCKWISE;
+            }
+        }
+
+        public RotatedCameraFrame(CvCameraViewFrame frame, int rotation) {
+            super();
+            mFrame = frame;
+            mRgbaRotated = new Mat();
+            mGrayRotated = new Mat();
+            mRotation = rotation;
+        }
+
+        @Override
+        public void release() {
+            mRgbaRotated.release();
+            mGrayRotated.release();
+        }
+
+        public CvCameraViewFrame mFrame;
+        private Mat mRgbaRotated;
+        private Mat mGrayRotated;
+        private int mRotation;
+    };
+
+    /**
+     * Calculates how to rotate camera frame to match current screen orientation
+     */
+    protected int getFrameRotation(boolean cameraFacingFront, int cameraSensorOrientation) {
+        WindowManager windowManager = (WindowManager) getContext().getSystemService(Context.WINDOW_SERVICE);
+        int screenOrientation = windowManager.getDefaultDisplay().getRotation();
+        int screenRotation = 0;
+        switch (screenOrientation) {
+            case Surface.ROTATION_0:
+                screenRotation = 0;
+                break;
+            case Surface.ROTATION_90:
+                screenRotation = 90;
+                break;
+            case Surface.ROTATION_180:
+                screenRotation = 180;
+                break;
+            case Surface.ROTATION_270:
+                screenRotation = 270;
+                break;
+        }
+
+        int frameRotation;
+        if (cameraFacingFront) {
+            frameRotation = (cameraSensorOrientation + screenRotation) % 360;
+        } else {
+            frameRotation = (cameraSensorOrientation - screenRotation + 360) % 360;
+        }
+
+        return frameRotation;
+    }
+
     public void surfaceChanged(SurfaceHolder arg0, int arg1, int arg2, int arg3) {
         Log.d(TAG, "call surfaceChanged event");
         synchronized(mSyncObject) {

modules/java/generator/android/java/org/opencv/android/JavaCameraView.java

@@ -42,7 +42,7 @@ public class JavaCameraView extends CameraBridgeViewBase implements PreviewCallb
     private boolean mStopThread;
 
     protected Camera mCamera;
-    protected JavaCameraFrame[] mCameraFrame;
+    protected RotatedCameraFrame[] mCameraFrame;
     private SurfaceTexture mSurfaceTexture;
     private int mPreviewFormat = ImageFormat.NV21;
 
@@ -132,7 +132,11 @@ public class JavaCameraView extends CameraBridgeViewBase implements PreviewCallb
             if (mCamera == null)
                 return false;
 
-            int frameRotation = getFrameRotation(cameraId);
+            android.hardware.Camera.CameraInfo info = new android.hardware.Camera.CameraInfo();
+            android.hardware.Camera.getCameraInfo(cameraId, info);
+            int frameRotation = getFrameRotation(
+                    info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT,
+                    info.orientation);
             /* Now set camera parameters */
             try {
                 Camera.Parameters params = mCamera.getParameters();
@@ -206,9 +210,9 @@ public class JavaCameraView extends CameraBridgeViewBase implements PreviewCallb
 
                     AllocateCache();
 
-                    mCameraFrame = new JavaCameraFrame[2];
+                    mCameraFrame = new RotatedCameraFrame[2];
-                    mCameraFrame[0] = new JavaCameraFrame(mFrameChain[0], rawFrameWidth, rawFrameHeight, frameRotation);
+                    mCameraFrame[0] = new RotatedCameraFrame(new JavaCameraFrame(mFrameChain[0], rawFrameWidth, rawFrameHeight), frameRotation);
-                    mCameraFrame[1] = new JavaCameraFrame(mFrameChain[1], rawFrameWidth, rawFrameHeight, frameRotation);
+                    mCameraFrame[1] = new RotatedCameraFrame(new JavaCameraFrame(mFrameChain[1], rawFrameWidth, rawFrameHeight), frameRotation);
 
                     if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.HONEYCOMB) {
                         mSurfaceTexture = new SurfaceTexture(MAGIC_TEXTURE_ID);
@@ -245,7 +249,9 @@ public class JavaCameraView extends CameraBridgeViewBase implements PreviewCallb
                 mFrameChain[1].release();
             }
             if (mCameraFrame != null) {
+                mCameraFrame[0].mFrame.release();
                 mCameraFrame[0].release();
+                mCameraFrame[1].mFrame.release();
                 mCameraFrame[1].release();
             }
         }
@@ -318,14 +324,7 @@ public class JavaCameraView extends CameraBridgeViewBase implements PreviewCallb
     private class JavaCameraFrame implements CvCameraViewFrame {
         @Override
         public Mat gray() {
-            mGray = mYuvFrameData.submat(0, mHeight, 0, mWidth);
+            return mYuvFrameData.submat(0, mHeight, 0, mWidth);
-
-            if (mRotation != 0) {
-                Core.rotate(mGray, mGrayRotated, getCvRotationCode(mRotation));
-                return mGrayRotated;
-            } else {
-                return mGray;
-            }
         }
 
         @Override
@@ -337,85 +336,28 @@ public class JavaCameraView extends CameraBridgeViewBase implements PreviewCallb
             else
                 throw new IllegalArgumentException("Preview Format can be NV21 or YV12");
 
-            if (mRotation != 0) {
+            return mRgba;
-                Core.rotate(mRgba, mRgbaRotated, getCvRotationCode(mRotation));
-                return mRgbaRotated;
-            } else {
-                return mRgba;
-            }
         }
 
-        private int getCvRotationCode(int degrees) {
+        public JavaCameraFrame(Mat Yuv420sp, int width, int height) {
-           if  (degrees == 90) {
-               return Core.ROTATE_90_CLOCKWISE;
-           } else if (degrees == 180) {
-               return Core.ROTATE_180;
-           } else {
-               return Core.ROTATE_90_COUNTERCLOCKWISE;
-           }
-        }
-
-        public JavaCameraFrame(Mat Yuv420sp, int width, int height, int rotation) {
             super();
             mWidth = width;
             mHeight = height;
             mYuvFrameData = Yuv420sp;
             mRgba = new Mat();
-            mRgbaRotated = new Mat();
-            mGrayRotated = new Mat();
-            mRotation = rotation;
         }
 
+        @Override
         public void release() {
             mRgba.release();
         }
 
         private Mat mYuvFrameData;
         private Mat mRgba;
-        private Mat mRgbaRotated;
-        private Mat mGray;
-        private Mat mGrayRotated;
         private int mWidth;
         private int mHeight;
-        private int mRotation;
     };
 
-    /**
-     * Calculates how to rotate camera frame to match current screen orientation
-     */
-    private int getFrameRotation(int cameraId) {
-        WindowManager windowManager = (WindowManager) getContext().getSystemService(Context.WINDOW_SERVICE);
-        int screenOrientation = windowManager.getDefaultDisplay().getRotation();
-        int screenRotation = 0;
-        switch (screenOrientation) {
-            case Surface.ROTATION_0:
-                screenRotation = 0;
-                break;
-            case Surface.ROTATION_90:
-                screenRotation = 90;
-                break;
-            case Surface.ROTATION_180:
-                screenRotation = 180;
-                break;
-            case Surface.ROTATION_270:
-                screenRotation = 270;
-                break;
-        }
-
-        android.hardware.Camera.CameraInfo info = new android.hardware.Camera.CameraInfo();
-        android.hardware.Camera.getCameraInfo(cameraId, info);
-
-        int frameRotation;
-        if (info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) {
-            frameRotation = (info.orientation + screenRotation) % 360;
-            frameRotation = (360 - frameRotation) % 360;
-        } else {
-            frameRotation = (info.orientation - screenRotation + 360) % 360;
-        }
-
-        return frameRotation;
-    }
-
     private class CameraWorker implements Runnable {
 
         @Override