mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-28 13:10:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

ocl: drop obsolete cache directories after upgrade of OpenCL driver

Browse Source 
Entries with the same platform name, the same device name and with different driver versions
are assumed obsolete.
This commit is contained in:
Alexander Alekhin 2017-11-24 12:52:29 +03:00
parent f071a48ec7
commit b6abf0d3f9
5 changed files with 230 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
modules/core/include/opencv2/core/utils/filesystem.hpp
View File

@ -11,9 +11,42 @@ namespace cv { namespace utils { namespace fs {
CV_EXPORTS bool exists(const cv::String& path);
CV_EXPORTS bool isDirectory(const cv::String& path);
CV_EXPORTS void remove_all(const cv::String& path);
CV_EXPORTS cv::String getcwd();
/** Join path components */
CV_EXPORTS cv::String join(const cv::String& base, const cv::String& path);
/**
* Generate a list of all files that match the globbing pattern.
*
* Result entries are prefixed by base directory path.
*
* @param directory base directory
* @param pattern filter pattern (based on '*'/'?' symbols). Use empty string to disable filtering and return all results
* @param[out] result result of globing.
* @param recursive scan nested directories too
* @param includeDirectories include directories into results list
*/
CV_EXPORTS void glob(const cv::String& directory, const cv::String& pattern,
CV_OUT std::vector<cv::String>& result,
bool recursive = false, bool includeDirectories = false);
/**
* Generate a list of all files that match the globbing pattern.
*
* @param directory base directory
* @param pattern filter pattern (based on '*'/'?' symbols). Use empty string to disable filtering and return all results
* @param[out] result globbing result with relative paths from base directory
* @param recursive scan nested directories too
* @param includeDirectories include directories into results list
*/
CV_EXPORTS void glob_relative(const cv::String& directory, const cv::String& pattern,
CV_OUT std::vector<cv::String>& result,
bool recursive = false, bool includeDirectories = false);
CV_EXPORTS bool createDirectory(const cv::String& path);
CV_EXPORTS bool createDirectories(const cv::String& path);

6
modules/core/include/opencv2/core/utils/logger.hpp
View File

@ -58,9 +58,9 @@ enum LogLevel {
#endif
#define CV_LOG_FATAL(tag, ...) for(;;) { std::stringstream ss; ss << "[FATAL:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str(); break; }
#define CV_LOG_ERROR(tag, ...) for(;;) { std::stringstream ss; ss << "[ERROR:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str(); break; }
#define CV_LOG_WARNING(tag, ...) for(;;) { std::stringstream ss; ss << "[ WARN:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cout << ss.str(); break; }
#define CV_LOG_FATAL(tag, ...) for(;;) { std::stringstream ss; ss << "[FATAL:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str() << std::flush; break; }
#define CV_LOG_ERROR(tag, ...) for(;;) { std::stringstream ss; ss << "[ERROR:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str() << std::flush; break; }
#define CV_LOG_WARNING(tag, ...) for(;;) { std::stringstream ss; ss << "[ WARN:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cout << ss.str() << std::flush; break; }
#if CV_LOG_STRIP_LEVEL <= CV_LOG_LEVEL_INFO
#define CV_LOG_INFO(tag, ...)
#else

45
modules/core/src/glob.cpp
View File

@ -47,7 +47,6 @@
#if defined _WIN32 || defined WINCE
# include <windows.h>
const char dir_separators[] = "/\\";
const char native_separator = '\\';
namespace
{
@ -136,7 +135,6 @@ namespace
# include <dirent.h>
# include <sys/stat.h>
const char dir_separators[] = "/";
const char native_separator = '/';
#endif
static bool isDir(const cv::String& path, DIR* dir)
@ -225,34 +223,36 @@ static bool wildcmp(const char *string, const char *wild)
return *wild == 0;
}
static void glob_rec(const cv::String& directory, const cv::String& wildchart, std::vector<cv::String>& result, bool recursive)
static void glob_rec(const cv::String& directory, const cv::String& wildchart, std::vector<cv::String>& result,
bool recursive, bool includeDirectories, const cv::String& pathPrefix)
{
DIR *dir;
struct dirent *ent;
if ((dir = opendir (directory.c_str())) != 0)
{
/* find all the files and directories within directory */
try
{
struct dirent *ent;
while ((ent = readdir (dir)) != 0)
{
const char* name = ent->d_name;
if((name[0] == 0) || (name[0] == '.' && name[1] == 0) || (name[0] == '.' && name[1] == '.' && name[2] == 0))
continue;
cv::String path = directory + native_separator + name;
cv::String path = cv::utils::fs::join(directory, name);
cv::String entry = cv::utils::fs::join(pathPrefix, name);
if (isDir(path, dir))
{
if (recursive)
glob_rec(path, wildchart, result, recursive);
}
else
{
if (wildchart.empty() || wildcmp(name, wildchart.c_str()))
result.push_back(path);
glob_rec(path, wildchart, result, recursive, includeDirectories, entry);
if (!includeDirectories)
continue;
}
if (wildchart.empty() || wildcmp(name, wildchart.c_str()))
result.push_back(entry);
}
}
catch (...)
@ -262,7 +262,10 @@ static void glob_rec(const cv::String& directory, const cv::String& wildchart, s
}
closedir(dir);
}
else CV_Error(CV_StsObjectNotFound, cv::format("could not open directory: %s", directory.c_str()));
else
{
CV_Error_(CV_StsObjectNotFound, ("could not open directory: %s", directory.c_str()));
}
}
void cv::glob(String pattern, std::vector<String>& result, bool recursive)
@ -298,6 +301,22 @@ void cv::glob(String pattern, std::vector<String>& result, bool recursive)
}
}
glob_rec(path, wildchart, result, recursive);
glob_rec(path, wildchart, result, recursive, false, path);
std::sort(result.begin(), result.end());
}
void cv::utils::fs::glob(const cv::String& directory, const cv::String& pattern,
std::vector<cv::String>& result,
bool recursive, bool includeDirectories)
{
glob_rec(directory, pattern, result, recursive, includeDirectories, directory);
std::sort(result.begin(), result.end());
}
void cv::utils::fs::glob_relative(const cv::String& directory, const cv::String& pattern,
std::vector<cv::String>& result,
bool recursive, bool includeDirectories)
{
glob_rec(directory, pattern, result, recursive, includeDirectories, cv::String());
std::sort(result.begin(), result.end());
}

93
modules/core/src/ocl.cpp
View File

@ -180,6 +180,7 @@ void traceOpenCLCheck(cl_int status, const char* message)
static const bool CV_OPENCL_CACHE_ENABLE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_ENABLE", true);
static const bool CV_OPENCL_CACHE_WRITE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_WRITE", true);
static const bool CV_OPENCL_CACHE_LOCK_ENABLE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_LOCK_ENABLE", true);
static const bool CV_OPENCL_CACHE_CLEANUP = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_CLEANUP", true);
#if CV_OPENCL_VALIDATE_BINARY_PROGRAMS
static const bool CV_OPENCL_VALIDATE_BINARY_PROGRAMS_VALUE = utils::getConfigurationParameterBool("OPENCV_OPENCL_VALIDATE_BINARY_PROGRAMS", false);
@ -254,6 +255,7 @@ struct OpenCLBinaryCacheConfigurator
typedef std::map<std::string, std::string> ContextCacheType;
ContextCacheType prepared_contexts_;
Mutex mutex_prepared_contexts_;
OpenCLBinaryCacheConfigurator()
{
@ -355,14 +357,17 @@ struct OpenCLBinaryCacheConfigurator
cache_lock_.release();
}
std::string prepareCacheDirectoryForContext(const std::string& ctx_prefix)
std::string prepareCacheDirectoryForContext(const std::string& ctx_prefix,
const std::string& cleanup_prefix)
{
if (cache_path_.empty())
return std::string();
ContextCacheType::iterator i = prepared_contexts_.find(ctx_prefix);
if (i != prepared_contexts_.end())
return i->second;
AutoLock lock(mutex_prepared_contexts_);
ContextCacheType::iterator found_it = prepared_contexts_.find(ctx_prefix);
if (found_it != prepared_contexts_.end())
return found_it->second;
CV_LOG_INFO(NULL, "Preparing OpenCL cache configuration for context: " << ctx_prefix);
@ -390,8 +395,59 @@ struct OpenCLBinaryCacheConfigurator
target_directory = result ? target_directory : std::string();
prepared_contexts_.insert(std::pair<std::string, std::string>(ctx_prefix, target_directory));
CV_LOG_VERBOSE(NULL, 1, " Result: " << (target_directory.empty() ? std::string("Failed") : target_directory));
if (result && CV_OPENCL_CACHE_CLEANUP && CV_OPENCL_CACHE_WRITE && !cleanup_prefix.empty())
{
try
{
std::vector<String> entries;
utils::fs::glob_relative(cache_path_, cleanup_prefix + "*", entries, false, true);
std::vector<String> remove_entries;
for (size_t i = 0; i < entries.size(); i++)
{
const String& name = entries[i];
if (0 == name.find(cleanup_prefix))
{
if (0 == name.find(ctx_prefix))
continue; // skip current
remove_entries.push_back(name);
}
}
if (!remove_entries.empty())
{
CV_LOG_WARNING(NULL, (remove_entries.size() == 1
? "Detected OpenCL cache directory for other version of OpenCL device."
: "Detected OpenCL cache directories for other versions of OpenCL device.")
<< " We assume that these directories are obsolete after OpenCL runtime/drivers upgrade.");
CV_LOG_WARNING(NULL, "Trying to remove these directories...");
for (size_t i = 0; i < remove_entries.size(); i++)
{
CV_LOG_WARNING(NULL, "- " << remove_entries[i]);
}
CV_LOG_WARNING(NULL,"Note: You can disable this behavior via this option: CV_OPENCL_CACHE_CLEANUP=0");
for (size_t i = 0; i < remove_entries.size(); i++)
{
const String& name = remove_entries[i];
cv::String path = utils::fs::join(cache_path_, name);
try
{
utils::fs::remove_all(path);
CV_LOG_WARNING(NULL, "Removed: " << path);
}
catch (const cv::Exception& e)
{
CV_LOG_ERROR(NULL, "Exception during removal of obsolete OpenCL cache directory: " << path << std::endl << e.what());
}
}
}
}
catch (...)
{
CV_LOG_WARNING(NULL, "Can't check for obsolete OpenCL cache directories");
}
}
CV_LOG_VERBOSE(NULL, 1, " Result: " << (target_directory.empty() ? std::string("Failed") : target_directory));
return target_directory;
}
@ -1969,7 +2025,7 @@ struct Context::Impl
}
}
std::string getPrefixString()
std::string& getPrefixString()
{
if (prefix.empty())
{
@ -1988,12 +2044,32 @@ struct Context::Impl
return prefix;
}
std::string& getPrefixBase()
{
if (prefix_base.empty())
{
const Device& d = devices[0];
prefix_base = d.vendorName() + "--" + d.name() + "--";
// sanitize chars
for (size_t i = 0; i < prefix_base.size(); i++)
{
char c = prefix_base[i];
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '-'))
{
prefix_base[i] = '_';
}
}
}
return prefix_base;
}
IMPLEMENT_REFCOUNTABLE();
cl_context handle;
std::vector<Device> devices;
std::string prefix;
std::string prefix_base;
cv::Mutex program_cache_mutex;
typedef std::map<std::string, Program> phash_t;
@ -3233,7 +3309,10 @@ struct Program::Impl
{
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
OpenCLBinaryCacheConfigurator& config = OpenCLBinaryCacheConfigurator::getSingletonInstance();
const std::string base_dir = config.prepareCacheDirectoryForContext(ctx.getImpl()->getPrefixString());
const std::string base_dir = config.prepareCacheDirectoryForContext(
ctx.getImpl()->getPrefixString(),
ctx.getImpl()->getPrefixBase()
);
const std::string fname = base_dir.empty() ? std::string() :
std::string(base_dir + src.getImpl()->module_.c_str() + "--" + src.getImpl()->name_ + "_" + src.getImpl()->codeHash_ + ".bin");
const cv::Ptr<utils::fs::FileLock> fileLock = config.cache_lock_; // can be empty

81
modules/core/src/utils/filesystem.cpp
View File

@ -31,6 +31,8 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <io.h>
#include <stdio.h>
#elif defined __linux__ || defined __APPLE__
#include <sys/types.h>
#include <sys/stat.h>
@ -41,12 +43,43 @@
namespace cv { namespace utils { namespace fs {
#ifdef _WIN32
static const char native_separator = '\\';
#else
static const char native_separator = '/';
#endif
static inline
bool isPathSeparator(char c)
{
return c == '/' || c == '\\';
}
cv::String join(const cv::String& base, const cv::String& path)
{
if (base.empty())
return path;
if (path.empty())
return base;
bool baseSep = isPathSeparator(base[base.size() - 1]);
bool pathSep = isPathSeparator(path[0]);
String result;
if (baseSep && pathSep)
{
result = base + path.substr(1);
}
else if (!baseSep && !pathSep)
{
result = base + native_separator + path;
}
else
{
result = base + path;
}
return result;
}
bool exists(const cv::String& path)
{
CV_INSTRUMENT_REGION()
@ -72,6 +105,44 @@ bool exists(const cv::String& path)
#endif
}
CV_EXPORTS void remove_all(const cv::String& path)
{
if (!exists(path))
return;
if (isDirectory(path))
{
std::vector<String> entries;
utils::fs::glob(path, cv::String(), entries, false, true);
for (size_t i = 0; i < entries.size(); i++)
{
const String& e = entries[i];
remove_all(e);
}
#ifdef _MSC_VER
bool result = _rmdir(path.c_str()) == 0;
#else
bool result = rmdir(path.c_str()) == 0;
#endif
if (!result)
{
CV_LOG_ERROR(NULL, "Can't remove directory: " << path);
}
}
else
{
#ifdef _MSC_VER
bool result = _unlink(path.c_str()) == 0;
#else
bool result = unlink(path.c_str()) == 0;
#endif
if (!result)
{
CV_LOG_ERROR(NULL, "Can't remove file: " << path);
}
}
}
cv::String getcwd()
{
CV_INSTRUMENT_REGION()
@ -138,7 +209,7 @@ bool createDirectories(const cv::String& path_)
for (;;)
{
char last_char = path.empty() ? 0 : path[path.length() - 1];
if (last_char == '/' || last_char == '\\')
if (isPathSeparator(last_char))
{
path = path.substr(0, path.length() - 1);
continue;
@ -364,7 +435,7 @@ cv::String getCacheDirectory(const char* sub_directory_name, const char* configu
if (home_env && home_env[0] && utils::fs::isDirectory(home_env))
{
cv::String home_path = home_env;
cv::String home_cache_path = home_path + "/.cache/";
cv::String home_cache_path = utils::fs::join(home_path, ".cache/");
if (utils::fs::isDirectory(home_cache_path))
{
default_cache_path = home_cache_path;
@ -393,9 +464,9 @@ cv::String getCacheDirectory(const char* sub_directory_name, const char* configu
{
if (utils::fs::isDirectory(default_cache_path))
{
default_cache_path += "/opencv/" CV_VERSION "/";
default_cache_path = utils::fs::join(default_cache_path, utils::fs::join("opencv", CV_VERSION));
if (sub_directory_name && sub_directory_name[0] != '\0')
default_cache_path += cv::String(sub_directory_name) + "/";
default_cache_path = utils::fs::join(default_cache_path, cv::String(sub_directory_name) + native_separator);
if (!utils::fs::createDirectories(default_cache_path))
{
CV_LOG_DEBUG(NULL, "Can't create OpenCV cache sub-directory: " << default_cache_path);
@ -434,7 +505,7 @@ cv::String getCacheDirectory(const char* sub_directory_name, const char* configu
{
if (!isPathSeparator(cache_path[cache_path.size() - 1]))
{
cache_path += '/';
cache_path += native_separator;
}
}
return cache_path;