Merge pull request #24592 from savuor:recorder_android

Android sample for VideoWriter #24592

This PR:
* adds an Android sample for video recording with MediaNDK and built-in MJPEG.
* adds a flag `--no_media_ndk` for `build_sdk.py` script to disable MediaNDK linkage.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

platforms/android/build_sdk.py

@@ -215,7 +215,7 @@ class Builder:
         for d in ["CMakeCache.txt", "CMakeFiles/", "bin/", "libs/", "lib/", "package/", "install/samples/"]:
             rm_one(d)
 
-    def build_library(self, abi, do_install):
+    def build_library(self, abi, do_install, no_media_ndk):
         cmd = [self.cmake_path, "-GNinja"]
         cmake_vars = dict(
             CMAKE_TOOLCHAIN_FILE=self.get_toolchain_file(),
@@ -260,6 +260,9 @@ class Builder:
         if do_install:
             cmd.extend(["-DBUILD_TESTS=ON", "-DINSTALL_TESTS=ON"])
 
+        if no_media_ndk:
+            cmake_vars['WITH_ANDROID_MEDIANDK'] = "OFF"
+
         cmake_vars.update(abi.cmake_vars)
         cmd += [ "-D%s='%s'" % (k, v) for (k, v) in cmake_vars.items() if v is not None]
         cmd.append(self.opencvdir)
@@ -370,6 +373,7 @@ if __name__ == "__main__":
     parser.add_argument('--opencl', action="store_true", help="Enable OpenCL support")
     parser.add_argument('--no_kotlin', action="store_true", help="Disable Kotlin extensions")
     parser.add_argument('--shared', action="store_true", help="Build shared libraries")
+    parser.add_argument('--no_media_ndk', action="store_true", help="Do not link Media NDK (required for video I/O support)")
     args = parser.parse_args()
 
     log.basicConfig(format='%(message)s', level=log.DEBUG)
@@ -447,7 +451,7 @@ if __name__ == "__main__":
 
         os.chdir(builder.libdest)
         builder.clean_library_build_dir()
-        builder.build_library(abi, do_install)
+        builder.build_library(abi, do_install, args.no_media_ndk)
 
     builder.gather_results()
 

samples/android/CMakeLists.txt

@@ -13,6 +13,7 @@ add_subdirectory(image-manipulations)
 add_subdirectory(camera-calibration)
 add_subdirectory(color-blob-detection)
 add_subdirectory(mobilenet-objdetect)
+add_subdirectory(video-recorder)
 add_subdirectory(tutorial-1-camerapreview)
 add_subdirectory(tutorial-2-mixedprocessing)
 add_subdirectory(tutorial-3-cameracontrol)

samples/android/video-recorder/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(sample example-video-recorder)
+
+if(BUILD_FAT_JAVA_LIB)
+  set(native_deps opencv_java)
+else()
+  set(native_deps videoio)
+endif()
+
+add_android_project(${sample} "${CMAKE_CURRENT_SOURCE_DIR}" LIBRARY_DEPS "${OPENCV_ANDROID_LIB_DIR}" SDK_TARGET 11 "${ANDROID_SDK_TARGET}" NATIVE_DEPS ${native_deps})
+if(TARGET ${sample})
+  add_dependencies(opencv_android_examples ${sample})
+endif()

samples/android/video-recorder/build.gradle.in

@@ -0,0 +1,36 @@
+apply plugin: 'com.android.application'
+
+android {
+    namespace 'org.opencv.samples.recorder'
+    compileSdkVersion @ANDROID_COMPILE_SDK_VERSION@
+    defaultConfig {
+        applicationId "org.opencv.samples.recorder"
+        minSdkVersion @ANDROID_MIN_SDK_VERSION@
+        targetSdkVersion @ANDROID_TARGET_SDK_VERSION@
+        versionCode 301
+        versionName "3.01"
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+    sourceSets {
+        main {
+            java.srcDirs = @ANDROID_SAMPLE_JAVA_PATH@
+            aidl.srcDirs = @ANDROID_SAMPLE_JAVA_PATH@
+            res.srcDirs = @ANDROID_SAMPLE_RES_PATH@
+            manifest.srcFile '@ANDROID_SAMPLE_MANIFEST_PATH@'
+        }
+    }
+}
+
+dependencies {
+    //implementation fileTree(dir: 'libs', include: ['*.jar'])
+    if (gradle.opencv_source == "sdk_path") {
+        implementation project(':opencv')
+    } else if (gradle.opencv_source == "maven_local" || gradle.opencv_source == "maven_cenral") {
+        implementation 'org.opencv:opencv:@OPENCV_VERSION_PLAIN@'
+    }
+}

samples/android/video-recorder/gradle/AndroidManifest.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          package="org.opencv.samples.recorder"
+>
+
+    <application
+        android:label="@string/app_name"
+        android:icon="@drawable/icon"
+        android:theme="@android:style/Theme.NoTitleBar.Fullscreen" >
+
+        <activity
+                  android:exported="true"
+                  android:name="RecorderActivity"
+                  android:label="@string/app_name"
+                  android:screenOrientation="landscape"
+                  android:configChanges="keyboardHidden|orientation">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+    <supports-screens android:resizeable="true"
+                      android:smallScreens="true"
+                      android:normalScreens="true"
+                      android:largeScreens="true"
+                      android:anyDensity="true" />
+
+    <uses-permission android:name="android.permission.CAMERA"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+
+    <uses-feature android:name="android.hardware.camera" android:required="false"/>
+    <uses-feature android:name="android.hardware.camera.autofocus" android:required="false"/>
+    <uses-feature android:name="android.hardware.camera.front" android:required="false"/>
+    <uses-feature android:name="android.hardware.camera.front.autofocus" android:required="false"/>
+
+</manifest>

samples/android/video-recorder/res/layout/recorder_surface_view.xml

@@ -0,0 +1,38 @@
+<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    xmlns:opencv="http://schemas.android.com/apk/res-auto"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent" >
+
+    <Button
+        android:id="@+id/btn1"
+        android:layout_width="wrap_content"
+        android:layout_height="50dp"
+        android:layout_margin="10dp"
+        android:text="Start Camera" />
+
+    <TextView
+        android:id="@+id/textview1"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_gravity="right"
+        android:layout_margin="10dp"
+        android:text="Status: Initialized"
+        android:textColor="#FF0000" />
+
+    <org.opencv.android.JavaCameraView
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:visibility="gone"
+        android:id="@+id/recorder_activity_java_surface_view"
+        opencv:show_fps="true"
+        opencv:camera_id="any" />
+
+    <ImageView
+        android:id="@+id/image_view"
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:visibility="gone"
+        />
+
+</FrameLayout>

samples/android/video-recorder/res/values/strings.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="app_name">OpenCV Video Recorder</string>
+</resources>

samples/android/video-recorder/src/org/opencv/samples/recorder/RecorderActivity.java

@@ -0,0 +1,340 @@
+package org.opencv.samples.recorder;
+
+import org.opencv.android.CameraActivity;
+import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame;
+import org.opencv.android.OpenCVLoader;
+import org.opencv.android.Utils;
+import org.opencv.core.Mat;
+import org.opencv.core.Size;
+import org.opencv.imgproc.Imgproc;
+import org.opencv.videoio.VideoCapture;
+import org.opencv.videoio.VideoWriter;
+import org.opencv.android.CameraBridgeViewBase;
+import org.opencv.android.CameraBridgeViewBase.CvCameraViewListener2;
+import org.opencv.videoio.Videoio;
+
+import android.graphics.Bitmap;
+import android.os.Bundle;
+import android.os.Handler;
+import android.util.Log;
+import android.view.MenuItem;
+import android.view.SurfaceView;
+import android.view.WindowManager;
+import android.view.View;
+import android.widget.Button;
+import android.widget.ImageView;
+import android.widget.TextView;
+import android.widget.Toast;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+
+public class RecorderActivity extends CameraActivity implements CvCameraViewListener2, View.OnClickListener {
+    private static final String TAG = "OCVSample::Activity";
+    private static final String FILENAME_MP4 = "sample_video1.mp4";
+    private static final String FILENAME_AVI = "sample_video1.avi";
+
+    private static final int STATUS_FINISHED_PLAYBACK = 0;
+    private static final int STATUS_PREVIEW = 1;
+    private static final int STATUS_RECORDING = 2;
+    private static final int STATUS_PLAYING = 3;
+    private static final int STATUS_ERROR = 4;
+
+    private String mVideoFilename;
+    private boolean mUseBuiltInMJPG = false;
+
+    private int mStatus = STATUS_FINISHED_PLAYBACK;
+    private int mFPS = 30;
+    private int mWidth = 0, mHeight = 0;
+
+    private CameraBridgeViewBase mOpenCvCameraView;
+    private ImageView mImageView;
+    private Button mTriggerButton;
+    private TextView mStatusTextView;
+    Runnable mPlayerThread;
+
+    private VideoWriter mVideoWriter = null;
+    private VideoCapture mVideoCapture = null;
+    private Mat mVideoFrame;
+    private Mat mRenderFrame;
+
+    public RecorderActivity() {
+        Log.i(TAG, "Instantiated new " + this.getClass());
+    }
+
+    /** Called when the activity is first created. */
+    @Override
+    public void onCreate(Bundle savedInstanceState) {
+        Log.i(TAG, "called onCreate");
+        super.onCreate(savedInstanceState);
+        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
+
+        setContentView(R.layout.recorder_surface_view);
+
+        mStatusTextView = (TextView) findViewById(R.id.textview1);
+        mStatusTextView.bringToFront();
+
+        if (OpenCVLoader.initLocal()) {
+            Log.i(TAG, "OpenCV loaded successfully");
+        } else {
+            Log.e(TAG, "OpenCV initialization failed!");
+            mStatus = STATUS_ERROR;
+            mStatusTextView.setText("Error: Can't initialize OpenCV");
+            return;
+        }
+
+        mOpenCvCameraView = (CameraBridgeViewBase) findViewById(R.id.recorder_activity_java_surface_view);
+        mOpenCvCameraView.setVisibility(SurfaceView.GONE);
+        mOpenCvCameraView.setCvCameraViewListener(this);
+        mOpenCvCameraView.disableView();
+
+        mImageView = (ImageView) findViewById(R.id.image_view);
+
+        mTriggerButton = (Button) findViewById(R.id.btn1);
+        mTriggerButton.setOnClickListener(this);
+        mTriggerButton.bringToFront();
+
+        if (mUseBuiltInMJPG)
+            mVideoFilename = getFilesDir() + "/" + FILENAME_AVI;
+        else
+            mVideoFilename = getFilesDir() + "/" + FILENAME_MP4;
+    }
+
+    @Override
+    public void onPause()
+    {
+        Log.d(TAG, "Pause");
+        super.onPause();
+        if (mOpenCvCameraView != null)
+            mOpenCvCameraView.disableView();
+        mImageView.setVisibility(SurfaceView.GONE);
+        if (mVideoWriter != null) {
+            mVideoWriter.release();
+            mVideoWriter = null;
+        }
+        if (mVideoCapture != null) {
+            mVideoCapture.release();
+            mVideoCapture = null;
+        }
+        mStatus = STATUS_FINISHED_PLAYBACK;
+        mStatusTextView.setText("Status: Finished playback");
+        mTriggerButton.setText("Start Camera");
+
+        mVideoFrame.release();
+        mRenderFrame.release();
+    }
+
+    @Override
+    public void onResume()
+    {
+        Log.d(TAG, "onResume");
+        super.onResume();
+
+        mVideoFrame = new Mat();
+        mRenderFrame = new Mat();
+
+        changeStatus();
+    }
+
+    @Override
+    protected List<? extends CameraBridgeViewBase> getCameraViewList() {
+        return Collections.singletonList(mOpenCvCameraView);
+    }
+
+    public void onDestroy() {
+        Log.d(TAG, "called onDestroy");
+        super.onDestroy();
+        if (mOpenCvCameraView != null)
+            mOpenCvCameraView.disableView();
+        if (mVideoWriter != null)
+            mVideoWriter.release();
+        if (mVideoCapture != null)
+            mVideoCapture.release();
+    }
+
+    public void onCameraViewStarted(int width, int height) {
+        Log.d(TAG, "Camera view started " + String.valueOf(width) + "x" + String.valueOf(height));
+        mWidth = width;
+        mHeight = height;
+    }
+
+    public void onCameraViewStopped() {
+        Log.d(TAG, "Camera view stopped");
+    }
+
+    public Mat onCameraFrame(CvCameraViewFrame inputFrame)
+    {
+        Log.d(TAG, "Camera frame arrived");
+
+        Mat rgbMat = inputFrame.rgba();
+
+        Log.d(TAG, "Size: " + rgbMat.width() + "x" + rgbMat.height());
+
+        if (mVideoWriter != null && mVideoWriter.isOpened()) {
+            Imgproc.cvtColor(rgbMat, mVideoFrame, Imgproc.COLOR_RGBA2BGR);
+            mVideoWriter.write(mVideoFrame);
+        }
+
+        return rgbMat;
+    }
+
+    @Override
+    public void onClick(View view) {
+        Log.i(TAG,"onClick event");
+        changeStatus();
+    }
+
+    public void changeStatus() {
+        switch(mStatus) {
+            case STATUS_ERROR:
+                Toast.makeText(this, "Error", Toast.LENGTH_LONG).show();
+                break;
+            case STATUS_FINISHED_PLAYBACK:
+                if (!startPreview()) {
+                    setErrorStatus();
+                    break;
+                }
+                mStatus = STATUS_PREVIEW;
+                mStatusTextView.setText("Status: Camera preview");
+                mTriggerButton.setText("Start recording");
+                break;
+            case STATUS_PREVIEW:
+                if (!startRecording()) {
+                    setErrorStatus();
+                    break;
+                }
+                mStatus = STATUS_RECORDING;
+                mStatusTextView.setText("Status: recording video");
+                mTriggerButton.setText(" Stop and play video");
+                break;
+            case STATUS_RECORDING:
+                if (!stopRecording()) {
+                    setErrorStatus();
+                    break;
+                }
+                if (!startPlayback()) {
+                    setErrorStatus();
+                    break;
+                }
+                mStatus = STATUS_PLAYING;
+                mStatusTextView.setText("Status: Playing video");
+                mTriggerButton.setText("Stop playback");
+                break;
+            case STATUS_PLAYING:
+                if (!stopPlayback()) {
+                    setErrorStatus();
+                    break;
+                }
+                mStatus = STATUS_FINISHED_PLAYBACK;
+                mStatusTextView.setText("Status: Finished playback");
+                mTriggerButton.setText("Start Camera");
+                break;
+        }
+    }
+
+    public void setErrorStatus() {
+        mStatus = STATUS_ERROR;
+        mStatusTextView.setText("Status: Error");
+    }
+
+    public boolean startPreview() {
+        mOpenCvCameraView.enableView();
+        mOpenCvCameraView.setVisibility(View.VISIBLE);
+        return true;
+    }
+
+    public boolean startRecording() {
+        Log.i(TAG,"Starting recording");
+
+        File file = new File(mVideoFilename);
+        file.delete();
+
+        mVideoWriter = new VideoWriter();
+        if (!mUseBuiltInMJPG) {
+            mVideoWriter.open(mVideoFilename, Videoio.CAP_ANDROID, VideoWriter.fourcc('H', '2', '6', '4'), mFPS, new Size(mWidth, mHeight));
+            if (!mVideoWriter.isOpened()) {
+                Log.i(TAG,"Can't record H264. Switching to MJPG");
+                mUseBuiltInMJPG = true;
+                mVideoFilename = getFilesDir() + "/" + FILENAME_AVI;
+            }
+        }
+
+        if (mUseBuiltInMJPG) {
+            mVideoWriter.open(mVideoFilename, VideoWriter.fourcc('M', 'J', 'P', 'G'), mFPS, new Size(mWidth, mHeight));
+        }
+
+        Log.d(TAG, "Size: " + String.valueOf(mWidth) + "x" + String.valueOf(mHeight));
+        Log.d(TAG, "File: " + mVideoFilename);
+
+        if (mVideoWriter.isOpened()) {
+            Toast.makeText(this, "Record started to file " + mVideoFilename, Toast.LENGTH_LONG).show();
+            return true;
+        } else {
+            Toast.makeText(this, "Failed to start a record", Toast.LENGTH_LONG).show();
+            return false;
+        }
+    }
+
+    public boolean stopRecording() {
+        Log.i(TAG, "Finishing recording");
+        mOpenCvCameraView.disableView();
+        mOpenCvCameraView.setVisibility(SurfaceView.GONE);
+        mVideoWriter.release();
+        mVideoWriter = null;
+        return true;
+    }
+
+    public boolean startPlayback() {
+        mImageView.setVisibility(SurfaceView.VISIBLE);
+
+        if (!mUseBuiltInMJPG){
+            mVideoCapture = new VideoCapture(mVideoFilename, Videoio.CAP_ANDROID);
+        } else {
+            mVideoCapture = new VideoCapture(mVideoFilename, Videoio.CAP_OPENCV_MJPEG);
+        }
+
+        if (!mVideoCapture.isOpened()) {
+            Log.e(TAG, "Can't open video");
+            Toast.makeText(this, "Can't open file " + mVideoFilename, Toast.LENGTH_SHORT).show();
+            return false;
+        }
+
+        Toast.makeText(this, "Starting playback from file " + mVideoFilename, Toast.LENGTH_SHORT).show();
+
+        mPlayerThread = new Runnable() {
+            @Override
+            public void run() {
+                if (mVideoCapture == null || !mVideoCapture.isOpened()) {
+                    return;
+                }
+                mVideoCapture.read(mVideoFrame);
+                if (mVideoFrame.empty()) {
+                    if (mStatus == STATUS_PLAYING) {
+                        changeStatus();
+                    }
+                    return;
+                }
+                // VideoCapture with CAP_ANDROID generates RGB frames instead of BGR
+                // https://github.com/opencv/opencv/issues/24687
+                Imgproc.cvtColor(mVideoFrame, mRenderFrame, mUseBuiltInMJPG ? Imgproc.COLOR_BGR2RGBA: Imgproc.COLOR_RGB2RGBA);
+                Bitmap bmp = Bitmap.createBitmap(mRenderFrame.cols(), mRenderFrame.rows(), Bitmap.Config.ARGB_8888);
+                Utils.matToBitmap(mRenderFrame, bmp);
+                mImageView.setImageBitmap(bmp);
+                Handler h = new Handler();
+                h.postDelayed(this, 33);
+            }
+        };
+
+        mPlayerThread.run();
+        return true;
+    }
+
+    public boolean stopPlayback() {
+        mVideoCapture.release();
+        mVideoCapture = null;
+        mImageView.setVisibility(SurfaceView.GONE);
+        return true;
+    }
+
+}