Merge pull request #21145 from MaximMilashchenko:AudioUpdate

Audio MSMF: added the ability to set sample per second

* Audio MSMF: added the ability to set sample per second

* changed the valid sampling rate check

* fixed docs

* add test

* fixed warning

* fixed error

* fixed error

modules/videoio/include/opencv2/videoio.hpp

@@ -195,7 +195,7 @@ enum VideoCaptureProperties {
        CAP_PROP_AUDIO_POS = 59, //!< (read-only) Audio position is measured in samples. Accurate audio sample timestamp of previous grabbed fragment. See CAP_PROP_AUDIO_SAMPLES_PER_SECOND and CAP_PROP_AUDIO_SHIFT_NSEC.
        CAP_PROP_AUDIO_SHIFT_NSEC = 60, //!< (read only) Contains the time difference between the start of the audio stream and the video stream in nanoseconds. Positive value means that audio is started after the first video frame. Negative value means that audio is started before the first video frame.
        CAP_PROP_AUDIO_DATA_DEPTH = 61, //!< (open, read) Alternative definition to bits-per-sample, but with clear handling of 32F / 32S
-       CAP_PROP_AUDIO_SAMPLES_PER_SECOND = 62, //!< (read-only) determined from file/codec input. If not specified, then selected audio sample rate is 44100
+       CAP_PROP_AUDIO_SAMPLES_PER_SECOND = 62, //!< (open, read) determined from file/codec input. If not specified, then selected audio sample rate is 44100
        CAP_PROP_AUDIO_BASE_INDEX = 63, //!< (read-only) Index of the first audio channel for .retrieve() calls. That audio channel number continues enumeration after video channels.
        CAP_PROP_AUDIO_TOTAL_CHANNELS = 64, //!< (read-only) Number of audio channels in the selected audio stream (mono, stereo, etc)
        CAP_PROP_AUDIO_TOTAL_STREAMS = 65, //!< (read-only) Number of audio streams.

modules/videoio/src/cap_msmf.cpp

@@ -727,6 +727,7 @@ protected:
     bool configureHW(bool enable);
     bool configureStreams(const cv::VideoCaptureParameters&);
     bool setAudioProperties(const cv::VideoCaptureParameters&);
+    bool checkAudioProperties();
 
     template <typename CtrlT>
     bool readComplexPropery(long prop, long& val) const;
@@ -766,6 +767,7 @@ protected:
     unsigned int audioBaseIndex;
     int outputVideoFormat;
     int outputAudioFormat;
+    UINT32 audioSamplesPerSecond;
     bool convertFormat;
     MFTIME duration;
     LONGLONG frameStep;
@@ -818,6 +820,7 @@ CvCapture_MSMF::CvCapture_MSMF():
     audioBaseIndex(1),
     outputVideoFormat(CV_CAP_MODE_BGR),
     outputAudioFormat(CV_16S),
+    audioSamplesPerSecond(0),
     convertFormat(true),
     duration(0),
     frameStep(0),
@@ -1047,7 +1050,7 @@ bool CvCapture_MSMF::configureAudioOutput(MediaType newType)
     MediaType newFormat = bestMatch.second;
 
     newFormat.majorType = MFMediaType_Audio;
-    newFormat.nSamplesPerSec = 44100;
+    newFormat.nSamplesPerSec = (audioSamplesPerSecond == 0) ? 44100 : audioSamplesPerSecond;
     switch (outputAudioFormat)
     {
     case CV_8S:
@@ -1147,7 +1150,8 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params)
     if (params)
     {
         configureHW(*params);
-        configureStreams(*params);
+        if (!(configureStreams(*params) && setAudioProperties(*params)))
+            return false;
     }
     if (videoStream != -1 && audioStream != -1 || videoStream == -1 && audioStream == -1)
     {
@@ -1189,6 +1193,12 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params)
         close();
         return false;
     }
+    if (isOpen)
+    {
+        if (audioStream != -1)
+            if (!checkAudioProperties())
+                return false;
+    }
 
     return isOpen;
 }
@@ -1202,8 +1212,8 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar
     if (params)
     {
         configureHW(*params);
-        configureStreams(*params);
-        setAudioProperties(*params);
+        if (!(configureStreams(*params) && setAudioProperties(*params)))
+            return false;
     }
     // Set source reader parameters
     _ComPtr<IMFAttributes> attr = getDefaultSourceConfig();
@@ -1235,12 +1245,19 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar
         return false;
     }
     if (isOpen)
-        if (audioStream != -1 && videoStream != -1)
+    {
+        if (audioStream != -1)
         {
-            isOpen = grabFrame();
-            if (isOpen)
-                grabIsDone = true;
+            if (!checkAudioProperties())
+                return false;
+            if (videoStream != -1)
+            {
+                isOpen = grabFrame();
+                if (isOpen)
+                    grabIsDone = true;
+            }
         }
+    }
     return isOpen;
 }
 
@@ -1318,14 +1335,49 @@ bool CvCapture_MSMF::setAudioProperties(const cv::VideoCaptureParameters& params
             outputAudioFormat = value;
         }
     }
+    if (params.has(CAP_PROP_AUDIO_SAMPLES_PER_SECOND))
+    {
+        int value = static_cast<int>(params.get<double>(CAP_PROP_AUDIO_SAMPLES_PER_SECOND));
+        if (value < 0)
+        {
+            CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter can't be negative: " << value);
+            return false;
+        }
+        else
+        {
+            audioSamplesPerSecond = value;
+        }
+    }
     if (params.has(CAP_PROP_AUDIO_SYNCHRONIZE))
     {
-        int value = static_cast<int>(params.get<double>(CAP_PROP_AUDIO_SYNCHRONIZE));
+        int value = static_cast<UINT32>(params.get<double>(CAP_PROP_AUDIO_SYNCHRONIZE));
         syncLastFrame = (value != 0) ? true : false;
     }
     return true;
 }
-
+bool CvCapture_MSMF::checkAudioProperties()
+{
+    if (audioSamplesPerSecond != 0)
+    {
+        _ComPtr<IMFMediaType> type;
+        UINT32 actualAudioSamplesPerSecond = 0;
+        HRESULT hr = videoFileSource->GetCurrentMediaType(dwAudioStreamIndex, &type);
+        if (SUCCEEDED(hr))
+        {
+            type->GetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND , &actualAudioSamplesPerSecond);
+            if (actualAudioSamplesPerSecond != audioSamplesPerSecond)
+            {
+                CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter value is invalid/unsupported: " << audioSamplesPerSecond
+                            << ". Current value of CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << actualAudioSamplesPerSecond);
+                close();
+                return false;
+            }
+            return true;
+        }
+        return false;
+    }
+    return true;
+}
 bool CvCapture_MSMF::grabVideoFrame()
 {
     DWORD streamIndex,  flags;

modules/videoio/test/test_audio.cpp

@@ -281,4 +281,15 @@ TEST(AudioOpenCheck, bad_arg_invalid_audio_stream)
     ASSERT_FALSE(cap.isOpened());
 }
 
+TEST(AudioOpenCheck, bad_arg_invalid_audio_sample_per_second)
+{
+    std::string fileName = "audio/test_audio.mp4";
+    std::vector<int> params {   CAP_PROP_AUDIO_STREAM, 0,
+                                CAP_PROP_VIDEO_STREAM, -1,
+                                CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9   };
+    VideoCapture cap;
+    cap.open(findDataFile(fileName), cv::CAP_MSMF, params);
+    ASSERT_FALSE(cap.isOpened());
+}
+
 }} //namespace