2012-10-17 07:18:30 +08:00
|
|
|
/*M///////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
//
|
|
|
|
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
|
|
|
|
//
|
|
|
|
// By downloading, copying, installing or using the software you agree to this license.
|
|
|
|
// If you do not agree to this license, do not download, install,
|
|
|
|
// copy or use the software.
|
|
|
|
//
|
|
|
|
//
|
|
|
|
// License Agreement
|
|
|
|
// For Open Source Computer Vision Library
|
|
|
|
//
|
|
|
|
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
|
|
|
|
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
|
|
|
|
// Third party copyrights are property of their respective owners.
|
|
|
|
//
|
|
|
|
// Redistribution and use in source and binary forms, with or without modification,
|
|
|
|
// are permitted provided that the following conditions are met:
|
|
|
|
//
|
|
|
|
// * Redistribution's of source code must retain the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer.
|
|
|
|
//
|
|
|
|
// * Redistribution's in binary form must reproduce the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer in the documentation
|
|
|
|
// and/or other materials provided with the distribution.
|
|
|
|
//
|
|
|
|
// * The name of the copyright holders may not be used to endorse or promote products
|
|
|
|
// derived from this software without specific prior written permission.
|
|
|
|
//
|
|
|
|
// This software is provided by the copyright holders and contributors "as is" and
|
|
|
|
// any express or implied warranties, including, but not limited to, the implied
|
|
|
|
// warranties of merchantability and fitness for a particular purpose are disclaimed.
|
|
|
|
// In no event shall the Intel Corporation or contributors be liable for any direct,
|
|
|
|
// indirect, incidental, special, exemplary, or consequential damages
|
|
|
|
// (including, but not limited to, procurement of substitute goods or services;
|
|
|
|
// loss of use, data, or profits; or business interruption) however caused
|
|
|
|
// and on any theory of liability, whether in contract, strict liability,
|
|
|
|
// or tort (including negligence or otherwise) arising in any way out of
|
|
|
|
// the use of this software, even if advised of the possibility of such damage.
|
|
|
|
//
|
|
|
|
//M*/
|
|
|
|
|
|
|
|
#include "precomp.hpp"
|
|
|
|
|
|
|
|
using namespace cv;
|
2013-08-28 19:45:13 +08:00
|
|
|
using namespace cv::cuda;
|
2012-10-17 07:18:30 +08:00
|
|
|
|
2014-12-17 23:51:15 +08:00
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
/// MemoryStack
|
|
|
|
|
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
|
|
|
|
namespace
|
|
|
|
{
|
|
|
|
class MemoryPool;
|
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
class MemoryStack
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
uchar* requestMemory(size_t size);
|
|
|
|
void returnMemory(uchar* ptr);
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
uchar* datastart;
|
|
|
|
uchar* dataend;
|
|
|
|
uchar* tip;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
bool isFree;
|
|
|
|
MemoryPool* pool;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
#if !defined(NDEBUG)
|
|
|
|
std::vector<size_t> allocations;
|
|
|
|
#endif
|
|
|
|
};
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
uchar* MemoryStack::requestMemory(size_t size)
|
|
|
|
{
|
|
|
|
const size_t freeMem = dataend - tip;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
if (size > freeMem)
|
|
|
|
return 0;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
uchar* ptr = tip;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
tip += size;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
#if !defined(NDEBUG)
|
|
|
|
allocations.push_back(size);
|
|
|
|
#endif
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
return ptr;
|
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
void MemoryStack::returnMemory(uchar* ptr)
|
|
|
|
{
|
|
|
|
CV_DbgAssert( ptr >= datastart && ptr < dataend );
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
#if !defined(NDEBUG)
|
|
|
|
const size_t allocSize = tip - ptr;
|
|
|
|
CV_Assert( allocSize == allocations.back() );
|
|
|
|
allocations.pop_back();
|
|
|
|
#endif
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
tip = ptr;
|
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
/// MemoryPool
|
|
|
|
|
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
|
|
|
|
namespace
|
|
|
|
{
|
|
|
|
class MemoryPool
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
MemoryPool();
|
|
|
|
|
|
|
|
void initialize(size_t stackSize, int stackCount);
|
|
|
|
void release();
|
|
|
|
|
|
|
|
MemoryStack* getFreeMemStack();
|
|
|
|
void returnMemStack(MemoryStack* memStack);
|
|
|
|
|
|
|
|
private:
|
|
|
|
void initilizeImpl();
|
|
|
|
|
|
|
|
Mutex mtx_;
|
|
|
|
|
|
|
|
bool initialized_;
|
|
|
|
size_t stackSize_;
|
|
|
|
int stackCount_;
|
|
|
|
|
|
|
|
uchar* mem_;
|
|
|
|
|
|
|
|
std::vector<MemoryStack> stacks_;
|
|
|
|
};
|
|
|
|
|
|
|
|
MemoryPool::MemoryPool() : initialized_(false), mem_(0)
|
|
|
|
{
|
|
|
|
// default : 10 Mb, 5 stacks
|
|
|
|
stackSize_ = 10 * 1024 * 1024;
|
|
|
|
stackCount_ = 5;
|
|
|
|
}
|
|
|
|
|
|
|
|
void MemoryPool::initialize(size_t stackSize, int stackCount)
|
|
|
|
{
|
|
|
|
AutoLock lock(mtx_);
|
|
|
|
|
|
|
|
release();
|
|
|
|
|
|
|
|
stackSize_ = stackSize;
|
|
|
|
stackCount_ = stackCount;
|
|
|
|
|
|
|
|
initilizeImpl();
|
|
|
|
}
|
|
|
|
|
|
|
|
void MemoryPool::initilizeImpl()
|
|
|
|
{
|
|
|
|
const size_t totalSize = stackSize_ * stackCount_;
|
|
|
|
|
|
|
|
if (totalSize > 0)
|
|
|
|
{
|
|
|
|
cudaError_t err = cudaMalloc(&mem_, totalSize);
|
|
|
|
if (err != cudaSuccess)
|
|
|
|
return;
|
|
|
|
|
|
|
|
stacks_.resize(stackCount_);
|
|
|
|
|
|
|
|
uchar* ptr = mem_;
|
|
|
|
|
|
|
|
for (int i = 0; i < stackCount_; ++i)
|
|
|
|
{
|
|
|
|
stacks_[i].datastart = ptr;
|
|
|
|
stacks_[i].dataend = ptr + stackSize_;
|
|
|
|
stacks_[i].tip = ptr;
|
|
|
|
stacks_[i].isFree = true;
|
|
|
|
stacks_[i].pool = this;
|
|
|
|
|
|
|
|
ptr += stackSize_;
|
|
|
|
}
|
|
|
|
|
|
|
|
initialized_ = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void MemoryPool::release()
|
|
|
|
{
|
|
|
|
if (mem_)
|
|
|
|
{
|
|
|
|
#if !defined(NDEBUG)
|
|
|
|
for (int i = 0; i < stackCount_; ++i)
|
|
|
|
{
|
|
|
|
CV_DbgAssert( stacks_[i].isFree );
|
|
|
|
CV_DbgAssert( stacks_[i].tip == stacks_[i].datastart );
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
cudaFree(mem_);
|
|
|
|
|
|
|
|
mem_ = 0;
|
|
|
|
initialized_ = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
MemoryStack* MemoryPool::getFreeMemStack()
|
|
|
|
{
|
|
|
|
AutoLock lock(mtx_);
|
|
|
|
|
|
|
|
if (!initialized_)
|
|
|
|
initilizeImpl();
|
|
|
|
|
|
|
|
if (!mem_)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
for (int i = 0; i < stackCount_; ++i)
|
|
|
|
{
|
|
|
|
if (stacks_[i].isFree)
|
|
|
|
{
|
|
|
|
stacks_[i].isFree = false;
|
|
|
|
return &stacks_[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void MemoryPool::returnMemStack(MemoryStack* memStack)
|
|
|
|
{
|
|
|
|
AutoLock lock(mtx_);
|
|
|
|
|
|
|
|
CV_DbgAssert( !memStack->isFree );
|
|
|
|
|
|
|
|
#if !defined(NDEBUG)
|
|
|
|
bool found = false;
|
|
|
|
for (int i = 0; i < stackCount_; ++i)
|
|
|
|
{
|
|
|
|
if (memStack == &stacks_[i])
|
|
|
|
{
|
|
|
|
found = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
CV_DbgAssert( found );
|
|
|
|
#endif
|
|
|
|
|
|
|
|
CV_DbgAssert( memStack->tip == memStack->datastart );
|
|
|
|
|
|
|
|
memStack->isFree = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2013-04-23 21:11:45 +08:00
|
|
|
////////////////////////////////////////////////////////////////
|
2014-12-17 23:51:15 +08:00
|
|
|
/// Stream::Impl
|
2013-04-23 21:11:45 +08:00
|
|
|
|
2013-04-16 21:43:49 +08:00
|
|
|
#ifndef HAVE_CUDA
|
2012-10-17 07:18:30 +08:00
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
class cv::cuda::Stream::Impl
|
2012-10-17 07:18:30 +08:00
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
public:
|
|
|
|
Impl(void* ptr = 0)
|
2012-10-17 07:18:30 +08:00
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
(void) ptr;
|
|
|
|
throw_no_cuda();
|
2013-02-13 19:51:27 +08:00
|
|
|
}
|
2013-04-16 21:43:49 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
#else
|
2012-10-17 07:18:30 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
namespace
|
|
|
|
{
|
|
|
|
class StackAllocator;
|
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
class cv::cuda::Stream::Impl
|
2013-04-16 21:43:49 +08:00
|
|
|
{
|
|
|
|
public:
|
2012-10-17 07:18:30 +08:00
|
|
|
cudaStream_t stream;
|
2013-10-07 22:25:55 +08:00
|
|
|
Ptr<StackAllocator> stackAllocator_;
|
2013-04-16 21:43:49 +08:00
|
|
|
|
|
|
|
Impl();
|
|
|
|
Impl(cudaStream_t stream);
|
|
|
|
|
|
|
|
~Impl();
|
2013-02-13 19:51:27 +08:00
|
|
|
};
|
2012-10-17 07:18:30 +08:00
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Stream::Impl::Impl() : stream(0)
|
2013-02-13 19:51:27 +08:00
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
cudaSafeCall( cudaStreamCreate(&stream) );
|
2013-10-07 22:25:55 +08:00
|
|
|
|
|
|
|
stackAllocator_ = makePtr<StackAllocator>(stream);
|
2012-10-17 07:18:30 +08:00
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Stream::Impl::Impl(cudaStream_t stream_) : stream(stream_)
|
2012-10-17 07:18:30 +08:00
|
|
|
{
|
2013-10-07 22:25:55 +08:00
|
|
|
stackAllocator_ = makePtr<StackAllocator>(stream);
|
2012-10-17 07:18:30 +08:00
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Stream::Impl::~Impl()
|
2013-02-13 19:51:27 +08:00
|
|
|
{
|
2013-10-07 22:25:55 +08:00
|
|
|
stackAllocator_.release();
|
|
|
|
|
2013-04-16 21:43:49 +08:00
|
|
|
if (stream)
|
|
|
|
cudaStreamDestroy(stream);
|
2013-02-13 19:51:27 +08:00
|
|
|
}
|
2012-10-17 07:18:30 +08:00
|
|
|
|
2014-12-17 23:51:15 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
/// DefaultDeviceInitializer
|
|
|
|
|
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
|
|
|
|
namespace cv { namespace cuda
|
2012-10-17 07:18:30 +08:00
|
|
|
{
|
2014-12-17 23:51:15 +08:00
|
|
|
class DefaultDeviceInitializer
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
DefaultDeviceInitializer();
|
|
|
|
~DefaultDeviceInitializer();
|
|
|
|
|
|
|
|
Stream& getNullStream(int deviceId);
|
|
|
|
MemoryPool* getMemoryPool(int deviceId);
|
|
|
|
|
|
|
|
private:
|
|
|
|
void initStreams();
|
|
|
|
void initPools();
|
|
|
|
|
|
|
|
Mutex streams_mtx_;
|
|
|
|
volatile bool streams_initialized_;
|
|
|
|
|
|
|
|
Mutex pools_mtx_;
|
|
|
|
volatile bool pools_initialized_;
|
|
|
|
|
|
|
|
std::vector<Ptr<Stream> > streams_;
|
|
|
|
std::vector<MemoryPool> pools_;
|
|
|
|
};
|
|
|
|
|
|
|
|
DefaultDeviceInitializer::DefaultDeviceInitializer()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
DefaultDeviceInitializer::~DefaultDeviceInitializer()
|
|
|
|
{
|
|
|
|
streams_.clear();
|
|
|
|
|
|
|
|
for (size_t i = 0; i < pools_.size(); ++i)
|
|
|
|
{
|
|
|
|
cudaSetDevice(static_cast<int>(i));
|
|
|
|
pools_[i].release();
|
|
|
|
}
|
|
|
|
|
|
|
|
pools_.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
Stream& DefaultDeviceInitializer::getNullStream(int deviceId)
|
|
|
|
{
|
|
|
|
initStreams();
|
|
|
|
|
|
|
|
CV_DbgAssert( deviceId >= 0 && deviceId < static_cast<int>(streams_.size()) );
|
|
|
|
|
|
|
|
return *streams_[deviceId];
|
|
|
|
}
|
|
|
|
|
|
|
|
MemoryPool* DefaultDeviceInitializer::getMemoryPool(int deviceId)
|
|
|
|
{
|
|
|
|
initPools();
|
|
|
|
|
|
|
|
CV_DbgAssert( deviceId >= 0 && deviceId < static_cast<int>(pools_.size()) );
|
|
|
|
|
|
|
|
return &pools_[deviceId];
|
|
|
|
}
|
|
|
|
|
|
|
|
void DefaultDeviceInitializer::initStreams()
|
|
|
|
{
|
|
|
|
AutoLock lock(streams_mtx_);
|
|
|
|
|
|
|
|
if (!streams_initialized_)
|
|
|
|
{
|
|
|
|
int deviceCount = getCudaEnabledDeviceCount();
|
|
|
|
|
|
|
|
if (deviceCount > 0)
|
|
|
|
{
|
|
|
|
streams_.resize(deviceCount);
|
|
|
|
|
|
|
|
for (int i = 0; i < deviceCount; ++i)
|
|
|
|
{
|
|
|
|
cudaStream_t stream = NULL;
|
|
|
|
Ptr<Stream::Impl> impl = makePtr<Stream::Impl>(stream);
|
|
|
|
streams_[i] = Ptr<Stream>(new Stream(impl));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
streams_initialized_ = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void DefaultDeviceInitializer::initPools()
|
|
|
|
{
|
|
|
|
AutoLock lock(pools_mtx_);
|
|
|
|
|
|
|
|
if (!pools_initialized_)
|
|
|
|
{
|
|
|
|
int deviceCount = getCudaEnabledDeviceCount();
|
|
|
|
|
|
|
|
if (deviceCount > 0)
|
|
|
|
pools_.resize(deviceCount);
|
|
|
|
|
|
|
|
pools_initialized_ = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
DefaultDeviceInitializer initializer;
|
|
|
|
}}
|
2013-02-13 19:51:27 +08:00
|
|
|
|
2013-04-16 21:43:49 +08:00
|
|
|
#endif
|
2013-02-13 19:51:27 +08:00
|
|
|
|
2014-12-17 23:51:15 +08:00
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
/// Stream
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Stream::Stream()
|
2013-04-16 21:43:49 +08:00
|
|
|
{
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
2013-09-06 19:44:44 +08:00
|
|
|
impl_ = makePtr<Impl>();
|
2013-04-16 21:43:49 +08:00
|
|
|
#endif
|
2012-10-17 07:18:30 +08:00
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
bool cv::cuda::Stream::queryIfComplete() const
|
2012-10-17 07:18:30 +08:00
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
throw_no_cuda();
|
|
|
|
return false;
|
|
|
|
#else
|
|
|
|
cudaError_t err = cudaStreamQuery(impl_->stream);
|
2012-10-17 07:18:30 +08:00
|
|
|
|
|
|
|
if (err == cudaErrorNotReady || err == cudaSuccess)
|
|
|
|
return err == cudaSuccess;
|
|
|
|
|
2013-04-08 16:37:36 +08:00
|
|
|
cudaSafeCall(err);
|
2012-10-17 07:18:30 +08:00
|
|
|
return false;
|
2013-04-16 21:43:49 +08:00
|
|
|
#endif
|
2012-10-17 07:18:30 +08:00
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
void cv::cuda::Stream::waitForCompletion()
|
2013-02-13 19:51:27 +08:00
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
|
|
|
cudaSafeCall( cudaStreamSynchronize(impl_->stream) );
|
|
|
|
#endif
|
2012-10-17 07:18:30 +08:00
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
void cv::cuda::Stream::waitEvent(const Event& event)
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
(void) event;
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
|
|
|
cudaSafeCall( cudaStreamWaitEvent(impl_->stream, EventAccessor::getEvent(event), 0) );
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2013-04-16 21:43:49 +08:00
|
|
|
#if defined(HAVE_CUDA) && (CUDART_VERSION >= 5000)
|
2013-02-13 19:51:27 +08:00
|
|
|
|
|
|
|
namespace
|
2012-10-17 07:18:30 +08:00
|
|
|
{
|
2013-02-13 19:51:27 +08:00
|
|
|
struct CallbackData
|
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
Stream::StreamCallback callback;
|
2013-02-13 19:51:27 +08:00
|
|
|
void* userData;
|
2013-04-16 21:43:49 +08:00
|
|
|
|
|
|
|
CallbackData(Stream::StreamCallback callback_, void* userData_) : callback(callback_), userData(userData_) {}
|
2013-02-13 19:51:27 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
void CUDART_CB cudaStreamCallback(cudaStream_t, cudaError_t status, void* userData)
|
|
|
|
{
|
|
|
|
CallbackData* data = reinterpret_cast<CallbackData*>(userData);
|
2013-04-16 21:43:49 +08:00
|
|
|
data->callback(static_cast<int>(status), data->userData);
|
2013-02-13 19:51:27 +08:00
|
|
|
delete data;
|
|
|
|
}
|
2012-10-17 07:18:30 +08:00
|
|
|
}
|
|
|
|
|
2013-02-13 19:51:27 +08:00
|
|
|
#endif
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
void cv::cuda::Stream::enqueueHostCallback(StreamCallback callback, void* userData)
|
2013-02-13 19:51:27 +08:00
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
#ifndef HAVE_CUDA
|
2013-02-13 19:51:27 +08:00
|
|
|
(void) callback;
|
|
|
|
(void) userData;
|
2013-04-16 21:43:49 +08:00
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
|
|
|
#if CUDART_VERSION < 5000
|
|
|
|
(void) callback;
|
|
|
|
(void) userData;
|
2014-12-17 23:51:15 +08:00
|
|
|
CV_Error(cv::Error::StsNotImplemented, "This function requires CUDA >= 5.0");
|
2013-04-16 21:43:49 +08:00
|
|
|
#else
|
|
|
|
CallbackData* data = new CallbackData(callback, userData);
|
|
|
|
|
|
|
|
cudaSafeCall( cudaStreamAddCallback(impl_->stream, cudaStreamCallback, data, 0) );
|
|
|
|
#endif
|
2013-02-13 19:51:27 +08:00
|
|
|
#endif
|
|
|
|
}
|
2012-10-17 07:18:30 +08:00
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
Stream& cv::cuda::Stream::Null()
|
2012-10-17 07:18:30 +08:00
|
|
|
{
|
2014-12-17 23:51:15 +08:00
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
throw_no_cuda();
|
|
|
|
static Stream stream;
|
|
|
|
return stream;
|
|
|
|
#else
|
|
|
|
const int deviceId = getDevice();
|
|
|
|
return initializer.getNullStream(deviceId);
|
|
|
|
#endif
|
2012-10-17 07:18:30 +08:00
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Stream::operator bool_type() const
|
2013-02-13 19:51:27 +08:00
|
|
|
{
|
2013-04-16 21:43:49 +08:00
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
return 0;
|
|
|
|
#else
|
|
|
|
return (impl_->stream != 0) ? &Stream::this_type_does_not_support_comparisons : 0;
|
|
|
|
#endif
|
2013-02-13 19:51:27 +08:00
|
|
|
}
|
|
|
|
|
2014-12-17 23:51:15 +08:00
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
|
|
|
|
cudaStream_t cv::cuda::StreamAccessor::getStream(const Stream& stream)
|
|
|
|
{
|
|
|
|
return stream.impl_->stream;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
/// StackAllocator
|
|
|
|
|
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
|
|
|
|
namespace
|
|
|
|
{
|
|
|
|
bool enableMemoryPool = true;
|
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
class StackAllocator : public GpuMat::Allocator
|
2014-12-17 23:51:15 +08:00
|
|
|
{
|
2014-12-20 00:33:32 +08:00
|
|
|
public:
|
|
|
|
explicit StackAllocator(cudaStream_t stream);
|
|
|
|
~StackAllocator();
|
|
|
|
|
|
|
|
bool allocate(GpuMat* mat, int rows, int cols, size_t elemSize);
|
|
|
|
void free(GpuMat* mat);
|
|
|
|
|
|
|
|
private:
|
|
|
|
StackAllocator(const StackAllocator&);
|
|
|
|
StackAllocator& operator =(const StackAllocator&);
|
|
|
|
|
|
|
|
cudaStream_t stream_;
|
|
|
|
MemoryStack* memStack_;
|
|
|
|
size_t alignment_;
|
|
|
|
};
|
|
|
|
|
|
|
|
StackAllocator::StackAllocator(cudaStream_t stream) : stream_(stream), memStack_(0)
|
|
|
|
{
|
|
|
|
if (enableMemoryPool)
|
|
|
|
{
|
|
|
|
const int deviceId = getDevice();
|
|
|
|
memStack_ = initializer.getMemoryPool(deviceId)->getFreeMemStack();
|
|
|
|
DeviceInfo devInfo(deviceId);
|
|
|
|
alignment_ = devInfo.textureAlignment();
|
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
}
|
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
StackAllocator::~StackAllocator()
|
|
|
|
{
|
|
|
|
cudaStreamSynchronize(stream_);
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
if (memStack_ != 0)
|
|
|
|
memStack_->pool->returnMemStack(memStack_);
|
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
|
|
|
|
size_t alignUp(size_t what, size_t alignment)
|
|
|
|
{
|
|
|
|
size_t alignMask = alignment-1;
|
|
|
|
size_t inverseAlignMask = ~alignMask;
|
|
|
|
size_t res = (what + alignMask) & inverseAlignMask;
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
bool StackAllocator::allocate(GpuMat* mat, int rows, int cols, size_t elemSize)
|
|
|
|
{
|
|
|
|
if (memStack_ == 0)
|
|
|
|
return false;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
size_t pitch, memSize;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
if (rows > 1 && cols > 1)
|
|
|
|
{
|
|
|
|
pitch = alignUp(cols * elemSize, alignment_);
|
|
|
|
memSize = pitch * rows;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
// Single row or single column must be continuous
|
|
|
|
pitch = elemSize * cols;
|
|
|
|
memSize = alignUp(elemSize * cols * rows, 64);
|
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
uchar* ptr = memStack_->requestMemory(memSize);
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
if (ptr == 0)
|
|
|
|
return false;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
mat->data = ptr;
|
|
|
|
mat->step = pitch;
|
|
|
|
mat->refcount = (int*) fastMalloc(sizeof(int));
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
return true;
|
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
void StackAllocator::free(GpuMat* mat)
|
|
|
|
{
|
|
|
|
if (memStack_ == 0)
|
|
|
|
return;
|
2014-12-17 23:51:15 +08:00
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
memStack_->returnMemory(mat->datastart);
|
|
|
|
fastFree(mat->refcount);
|
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
}
|
|
|
|
|
2014-12-20 00:33:32 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////
|
|
|
|
/// BufferPool
|
|
|
|
|
2014-12-17 23:51:15 +08:00
|
|
|
void cv::cuda::setBufferPoolUsage(bool on)
|
|
|
|
{
|
2014-12-20 00:33:32 +08:00
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
(void)on;
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
2014-12-17 23:51:15 +08:00
|
|
|
enableMemoryPool = on;
|
2014-12-20 00:33:32 +08:00
|
|
|
#endif
|
2014-12-17 23:51:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void cv::cuda::setBufferPoolConfig(int deviceId, size_t stackSize, int stackCount)
|
|
|
|
{
|
2014-12-20 00:33:32 +08:00
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
(void)deviceId;
|
|
|
|
(void)stackSize;
|
|
|
|
(void)stackCount;
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
2014-12-17 23:51:15 +08:00
|
|
|
const int currentDevice = getDevice();
|
|
|
|
|
|
|
|
if (deviceId >= 0)
|
|
|
|
{
|
|
|
|
setDevice(deviceId);
|
|
|
|
initializer.getMemoryPool(deviceId)->initialize(stackSize, stackCount);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
const int deviceCount = getCudaEnabledDeviceCount();
|
|
|
|
|
|
|
|
for (deviceId = 0; deviceId < deviceCount; ++deviceId)
|
|
|
|
{
|
|
|
|
setDevice(deviceId);
|
|
|
|
initializer.getMemoryPool(deviceId)->initialize(stackSize, stackCount);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
setDevice(currentDevice);
|
|
|
|
#endif
|
2014-12-20 00:33:32 +08:00
|
|
|
}
|
2014-12-17 23:51:15 +08:00
|
|
|
|
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
|
|
|
|
cv::cuda::BufferPool::BufferPool(Stream& stream) : allocator_(stream.impl_->stackAllocator_.get())
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
GpuMat cv::cuda::BufferPool::getBuffer(int rows, int cols, int type)
|
|
|
|
{
|
|
|
|
GpuMat buf(allocator_);
|
|
|
|
buf.create(rows, cols, type);
|
|
|
|
return buf;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
2013-04-23 21:11:45 +08:00
|
|
|
|
|
|
|
////////////////////////////////////////////////////////////////
|
2013-10-03 21:22:18 +08:00
|
|
|
// Event
|
2013-04-23 21:11:45 +08:00
|
|
|
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
class cv::cuda::Event::Impl
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
public:
|
|
|
|
Impl(unsigned int)
|
|
|
|
{
|
|
|
|
throw_no_cuda();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
class cv::cuda::Event::Impl
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
public:
|
|
|
|
cudaEvent_t event;
|
|
|
|
|
|
|
|
Impl(unsigned int flags);
|
|
|
|
~Impl();
|
|
|
|
};
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Event::Impl::Impl(unsigned int flags) : event(0)
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
cudaSafeCall( cudaEventCreateWithFlags(&event, flags) );
|
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Event::Impl::~Impl()
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
if (event)
|
|
|
|
cudaEventDestroy(event);
|
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cudaEvent_t cv::cuda::EventAccessor::getEvent(const Event& event)
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
return event.impl_->event;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
cv::cuda::Event::Event(CreateFlags flags)
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
(void) flags;
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
2013-09-06 19:44:44 +08:00
|
|
|
impl_ = makePtr<Impl>(flags);
|
2013-04-23 21:11:45 +08:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
void cv::cuda::Event::record(Stream& stream)
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
(void) stream;
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
|
|
|
cudaSafeCall( cudaEventRecord(impl_->event, StreamAccessor::getStream(stream)) );
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
bool cv::cuda::Event::queryIfComplete() const
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
throw_no_cuda();
|
|
|
|
return false;
|
|
|
|
#else
|
|
|
|
cudaError_t err = cudaEventQuery(impl_->event);
|
|
|
|
|
|
|
|
if (err == cudaErrorNotReady || err == cudaSuccess)
|
|
|
|
return err == cudaSuccess;
|
|
|
|
|
|
|
|
cudaSafeCall(err);
|
|
|
|
return false;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
void cv::cuda::Event::waitForCompletion()
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
throw_no_cuda();
|
|
|
|
#else
|
|
|
|
cudaSafeCall( cudaEventSynchronize(impl_->event) );
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2013-08-28 19:45:13 +08:00
|
|
|
float cv::cuda::Event::elapsedTime(const Event& start, const Event& end)
|
2013-04-23 21:11:45 +08:00
|
|
|
{
|
|
|
|
#ifndef HAVE_CUDA
|
|
|
|
(void) start;
|
|
|
|
(void) end;
|
|
|
|
throw_no_cuda();
|
|
|
|
return 0.0f;
|
|
|
|
#else
|
|
|
|
float ms;
|
|
|
|
cudaSafeCall( cudaEventElapsedTime(&ms, start.impl_->event, end.impl_->event) );
|
|
|
|
return ms;
|
|
|
|
#endif
|
|
|
|
}
|