mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-08-06 06:26:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

videoio(MSMF): add queue for async ReadSample()

Browse Source
This commit is contained in:
Alexander Alekhin 2021-12-06 09:54:48 +03:00
parent 973e1acb67
commit 0c5b6e5556
2 changed files with 74 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
modules/videoio/src/cap_msmf.cpp
View File

@ -32,6 +32,7 @@
#endif #endif
#include <new> #include <new>
#include <map> #include <map>
#include <queue>
#include <vector> #include <vector>
#include <string> #include <string>
#include <algorithm> #include <algorithm>
@ -395,8 +396,10 @@ private:
class SourceReaderCB : public IMFSourceReaderCallback class SourceReaderCB : public IMFSourceReaderCallback
{ {
public: public:
static const size_t MSMF_READER_MAX_QUEUE_SIZE = 3;
SourceReaderCB() : SourceReaderCB() :
m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0), m_lastSampleTimestamp(0) m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0)
{ {
} }
@ -441,12 +444,19 @@ public:
if (pSample) if (pSample)
{ {
CV_LOG_DEBUG(NULL, "videoio(MSMF): got frame at " << llTimestamp); CV_LOG_DEBUG(NULL, "videoio(MSMF): got frame at " << llTimestamp);
if (m_lastSample.Get()) if (m_capturedFrames.size() >= MSMF_READER_MAX_QUEUE_SIZE)
{ {
CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed)"); #if 0
CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed). Timestamp=" << m_capturedFrames.front().timestamp);
m_capturedFrames.pop();
#else
// this branch reduces latency if we drop frames due to slow processing.
// avoid fetching of already outdated frames from the queue's front.
CV_LOG_DEBUG(NULL, "videoio(MSMF): drop previous frames (not processed): " << m_capturedFrames.size());
std::queue<CapturedFrameInfo>().swap(m_capturedFrames); // similar to missing m_capturedFrames.clean();
#endif
} }
m_lastSampleTimestamp = llTimestamp; m_capturedFrames.emplace(CapturedFrameInfo{ llTimestamp, _ComPtr<IMFSample>(pSample), hrStatus });
m_lastSample = pSample;
} }
} }
else else
@ -483,30 +493,43 @@ public:
return S_OK; return S_OK;
} }
HRESULT Wait(DWORD dwMilliseconds, _ComPtr<IMFSample>& mediaSample, BOOL& pbEOS) HRESULT Wait(DWORD dwMilliseconds, _ComPtr<IMFSample>& mediaSample, LONGLONG& sampleTimestamp, BOOL& pbEOS)
{ {
pbEOS = FALSE; pbEOS = FALSE;
DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); for (;;)
if (dwResult == WAIT_TIMEOUT)
{ {
return E_PENDING; {
} cv::AutoLock lock(m_mutex);
else if (dwResult != WAIT_OBJECT_0)
{
return HRESULT_FROM_WIN32(GetLastError());
}
pbEOS = m_bEOS; pbEOS = m_bEOS && m_capturedFrames.empty();
if (!pbEOS) if (pbEOS)
{ return m_hrStatus;
cv::AutoLock lock(m_mutex);
mediaSample = m_lastSample; if (!m_capturedFrames.empty())
CV_Assert(mediaSample); {
m_lastSample.Release(); CV_Assert(!m_capturedFrames.empty());
ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. CapturedFrameInfo frameInfo = m_capturedFrames.front(); m_capturedFrames.pop();
CV_LOG_DEBUG(NULL, "videoio(MSMF): handle frame at " << frameInfo.timestamp);
mediaSample = frameInfo.sample;
CV_Assert(mediaSample);
sampleTimestamp = frameInfo.timestamp;
ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold.
return frameInfo.hrStatus;
}
}
CV_LOG_DEBUG(NULL, "videoio(MSMF): waiting for frame... ");
DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds);
if (dwResult == WAIT_TIMEOUT)
{
return E_PENDING;
}
else if (dwResult != WAIT_OBJECT_0)
{
return HRESULT_FROM_WIN32(GetLastError());
}
} }
return m_hrStatus;
} }
private: private:
@ -525,8 +548,14 @@ public:
IMFSourceReader *m_reader; IMFSourceReader *m_reader;
DWORD m_dwStreamIndex; DWORD m_dwStreamIndex;
LONGLONG m_lastSampleTimestamp;
_ComPtr<IMFSample> m_lastSample; struct CapturedFrameInfo {
LONGLONG timestamp;
_ComPtr<IMFSample> sample;
HRESULT hrStatus;
};
std::queue<CapturedFrameInfo> m_capturedFrames;
}; };
//================================================================================================== //==================================================================================================
@ -1569,7 +1598,6 @@ bool CvCapture_MSMF::configureAudioFrame()
} }
audioDataInUse.clear(); audioDataInUse.clear();
audioDataInUse.shrink_to_fit(); audioDataInUse.shrink_to_fit();
return true; return true;
} }
else else
@ -1690,6 +1718,7 @@ bool CvCapture_MSMF::grabFrame()
if (grabIsDone) if (grabIsDone)
{ {
grabIsDone = false; grabIsDone = false;
CV_LOG_DEBUG(NULL, "videoio(MSMF): return pre-grabbed frame " << usedVideoSampleTime);
return true; return true;
} }
@ -1717,7 +1746,8 @@ bool CvCapture_MSMF::grabFrame()
} }
} }
BOOL bEOS = false; BOOL bEOS = false;
if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], bEOS))) // 10 sec LONGLONG timestamp = 0;
if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], timestamp, bEOS))) // 10 sec
{ {
CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr);
return false; return false;
@ -1728,10 +1758,11 @@ bool CvCapture_MSMF::grabFrame()
return false; return false;
} }
if (videoStream != -1) if (videoStream != -1)
usedVideoSampleTime = reader->m_lastSampleTimestamp; usedVideoSampleTime = timestamp;
if (audioStream != -1) if (audioStream != -1)
return configureAudioFrame(); return configureAudioFrame();
CV_LOG_DEBUG(NULL, "videoio(MSMF): grabbed frame " << usedVideoSampleTime);
return true; return true;
} }
else if (isOpen) else if (isOpen)
@ -1765,6 +1796,7 @@ bool CvCapture_MSMF::grabFrame()
bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame) bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame)
{ {
CV_TRACE_FUNCTION(); CV_TRACE_FUNCTION();
CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame start...");
do do
{ {
if (!usedVideoSample) if (!usedVideoSample)
@ -1855,6 +1887,7 @@ bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame)
buffer2d->Unlock2D(); buffer2d->Unlock2D();
else else
buf->Unlock(); buf->Unlock();
CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame done!");
return !frame.empty(); return !frame.empty();
} while (0); } while (0);

15
modules/videoio/test/test_camera.cpp
View File

@ -26,15 +26,26 @@ static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100,
const bool validTickAndFps = cvTickFreq != 0 && fps != 0.; const bool validTickAndFps = cvTickFreq != 0 && fps != 0.;
testTimestamps &= validTickAndFps; testTimestamps &= validTickAndFps;
double frame0ts = 0;
for (int i = 0; i < N; i++) for (int i = 0; i < N; i++)
{ {
SCOPED_TRACE(cv::format("frame=%d", i)); SCOPED_TRACE(cv::format("frame=%d", i));
capture >> frame; capture >> frame;
const int64 sysTimeCurr = cv::getTickCount();
const double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC);
ASSERT_FALSE(frame.empty()); ASSERT_FALSE(frame.empty());
const int64 sysTimeCurr = cv::getTickCount();
double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC);
if (i == 0)
frame0ts = camTimeCurr;
camTimeCurr -= frame0ts; // normalized timestamp based on the first frame
if (cvtest::debugLevel > 0)
{
std::cout << i << ": " << camTimeCurr << std::endl;
}
// Do we have a previous frame? // Do we have a previous frame?
if (i > 0 && testTimestamps) if (i > 0 && testTimestamps)
{ {