diff --git a/doc/opencv.bib b/doc/opencv.bib index 021c5b4b3f..17eedf7172 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -415,16 +415,6 @@ pages = {2548--2555}, organization = {IEEE} } -@ARTICLE{Louhichi07, - author = {Louhichi, H. and Fournel, T. and Lavest, J. M. and Ben Aissia, H.}, - title = {Self-calibration of Scheimpflug cameras: an easy protocol}, - year = {2007}, - pages = {2616–2622}, - journal = {Meas. Sci. Technol.}, - volume = {18}, - number = {8}, - publisher = {IOP Publishing Ltd} -} @ARTICLE{LibSVM, author = {Chang, Chih-Chung and Lin, Chih-Jen}, title = {LIBSVM: a library for support vector machines}, @@ -874,3 +864,11 @@ year={2007}, organization={IEEE} } +@incollection{bottou2010large, + title={Large-scale machine learning with stochastic gradient descent}, + author={Bottou, L{\'e}on}, + booktitle={Proceedings of COMPSTAT'2010}, + pages={177--186}, + year={2010}, + publisher={Springer} +} \ No newline at end of file diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index 7acce7f33c..d5debdbf18 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -1513,6 +1513,127 @@ CV_EXPORTS void randMVNormal( InputArray mean, InputArray cov, int nsamples, Out CV_EXPORTS void createConcentricSpheresTestSet( int nsamples, int nfeatures, int nclasses, OutputArray samples, OutputArray responses); +/****************************************************************************************\ +* Stochastic Gradient Descent SVM Classifier * +\****************************************************************************************/ + +/*! +@brief Stochastic Gradient Descent SVM classifier + +SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large. +The gradient descent show amazing performance for large-scale problems, reducing the computing time. This allows a fast and reliable online update of the classifier for each new feature which +is fundamental when dealing with variations of data over time (like weather and illumination changes in videosurveillance, for example). + +First, create the SVMSGD object. To enable the online update, a value for updateFrequency should be defined. + +Then the SVM model can be trained using the train features and the correspondent labels. + +After that, the label of a new feature vector can be predicted using the predict function. If the updateFrequency was defined in the constructor, the predict function will update the weights automatically. + +@code +// Initialize object +SVMSGD SvmSgd; + +// Train the Stochastic Gradient Descent SVM +SvmSgd.train(trainFeatures, labels); + +// Predict label for the new feature vector (1xM) +predictedLabel = SvmSgd.predict(newFeatureVector); +@endcode + +*/ +class CV_EXPORTS_W SVMSGD { + + public: + /** @brief SGDSVM constructor. + + @param lambda regularization + @param learnRate learning rate + @param nIterations number of training iterations + + */ + SVMSGD(float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000); + + /** @brief SGDSVM constructor. + + @param updateFrequency online update frequency + @param learnRateDecay learn rate decay over time: learnRate = learnRate * learnDecay + @param lambda regularization + @param learnRate learning rate + @param nIterations number of training iterations + + */ + SVMSGD(uint updateFrequency, float learnRateDecay = 1, float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000); + virtual ~SVMSGD(); + virtual SVMSGD* clone() const; + + /** @brief Train the SGDSVM classifier. + + The function trains the SGDSVM classifier using the train features and the correspondent labels (-1 or 1). + + @param trainFeatures features used for training. Each row is a new sample. + @param labels mat (size Nx1 with N = number of features) with the label of each training feature. + + */ + virtual void train(cv::Mat trainFeatures, cv::Mat labels); + + /** @brief Predict the label of a new feature vector. + + The function predicts and returns the label of a new feature vector, using the previously trained SVM model. + + @param newFeature new feature vector used for prediction + + */ + virtual float predict(cv::Mat newFeature); + + /** @brief Returns the weights of the trained model. + + */ + virtual std::vector getWeights(){ return _weights; }; + + /** @brief Sets the weights of the trained model. + + @param weights weights used to predict the label of a new feature vector. + + */ + virtual void setWeights(std::vector weights){ _weights = weights; }; + + private: + void updateWeights(); + void generateRandomIndex(); + float calcInnerProduct(float *rowDataPointer); + void updateWeights(float innerProduct, float *rowDataPointer, int label); + + // Vector with SVM weights + std::vector _weights; + + // Random index generation + long long int _randomNumber; + unsigned int _randomIndex; + + // Number of features and samples + unsigned int _nFeatures; + unsigned int _nTrainSamples; + + // Parameters for learning + float _lambda; //regularization + float _learnRate; //learning rate + unsigned int _nIterations; //number of training iterations + + // Vars to control the features slider matrix + bool _onlineUpdate; + bool _initPredict; + uint _slidingWindowSize; + uint _predictSlidingWindowSize; + float* _labelSlider; + float _learnRateDecay; + + // Mat with features slider and correspondent counter + unsigned int _sliderCounter; + cv::Mat _featuresSlider; + +}; + //! @} ml } diff --git a/modules/ml/src/svmsgd.cpp b/modules/ml/src/svmsgd.cpp new file mode 100644 index 0000000000..3114e43d9f --- /dev/null +++ b/modules/ml/src/svmsgd.cpp @@ -0,0 +1,201 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" + +/****************************************************************************************\ +* Stochastic Gradient Descent SVM Classifier * +\****************************************************************************************/ + +namespace cv { +namespace ml { + +SVMSGD::SVMSGD(float lambda, float learnRate, uint nIterations){ + + // Initialize with random seed + _randomNumber = 1; + + // Initialize constants + _slidingWindowSize = 0; + _nFeatures = 0; + _predictSlidingWindowSize = 1; + + // Initialize sliderCounter at index 0 + _sliderCounter = 0; + + // Parameters for learning + _lambda = lambda; // regularization + _learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration) + _nIterations = nIterations; // number of training iterations + + // True only in the first predict iteration + _initPredict = true; + + // Online update flag + _onlineUpdate = false; +} + +SVMSGD::SVMSGD(uint updateFrequency, float learnRateDecay, float lambda, float learnRate, uint nIterations){ + + // Initialize with random seed + _randomNumber = 1; + + // Initialize constants + _slidingWindowSize = 0; + _nFeatures = 0; + _predictSlidingWindowSize = updateFrequency; + + // Initialize sliderCounter at index 0 + _sliderCounter = 0; + + // Parameters for learning + _lambda = lambda; // regularization + _learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration) + _nIterations = nIterations; // number of training iterations + + // True only in the first predict iteration + _initPredict = true; + + // Online update flag + _onlineUpdate = true; + + // Learn rate decay: _learnRate = _learnRate * _learnDecay + _learnRateDecay = learnRateDecay; +} + +SVMSGD::~SVMSGD(){ + +} + +SVMSGD* SVMSGD::clone() const{ + return new SVMSGD(*this); +} + +void SVMSGD::train(cv::Mat trainFeatures, cv::Mat labels){ + + // Initialize _nFeatures + _slidingWindowSize = trainFeatures.rows; + _nFeatures = trainFeatures.cols; + + float innerProduct; + // Initialize weights vector with zeros + if (_weights.size()==0){ + _weights.reserve(_nFeatures); + for (uint feat = 0; feat < _nFeatures; ++feat){ + _weights.push_back(0.0); + } + } + + // Stochastic gradient descent SVM + for (uint iter = 0; iter < _nIterations; ++iter){ + generateRandomIndex(); + innerProduct = calcInnerProduct(trainFeatures.ptr(_randomIndex)); + int label = (labels.at(_randomIndex,0) > 0) ? 1 : -1; // ensure that labels are -1 or 1 + updateWeights(innerProduct, trainFeatures.ptr(_randomIndex), label ); + } +} + +float SVMSGD::predict(cv::Mat newFeature){ + float innerProduct; + + if (_initPredict){ + _nFeatures = newFeature.cols; + _slidingWindowSize = _predictSlidingWindowSize; + _featuresSlider = cv::Mat::zeros(_slidingWindowSize, _nFeatures, CV_32F); + _initPredict = false; + _labelSlider = new float[_predictSlidingWindowSize](); + _learnRate = _learnRate * _learnRateDecay; + } + + innerProduct = calcInnerProduct(newFeature.ptr(0)); + + // Resultant label (-1 or 1) + int label = (innerProduct>=0) ? 1 : -1; + + if (_onlineUpdate){ + // Update the featuresSlider with newFeature and _labelSlider with label + newFeature.row(0).copyTo(_featuresSlider.row(_sliderCounter)); + _labelSlider[_sliderCounter] = float(label); + + // Update weights with a random index + if (_sliderCounter == _slidingWindowSize-1){ + generateRandomIndex(); + updateWeights(innerProduct, _featuresSlider.ptr(_randomIndex), int(_labelSlider[_randomIndex]) ); + } + + // _sliderCounter++ if < _slidingWindowSize + _sliderCounter = (_sliderCounter == _slidingWindowSize-1) ? 0 : (_sliderCounter+1); + } + + return float(label); +} + +void SVMSGD::generateRandomIndex(){ + // Choose random sample, using Mikolov's fast almost-uniform random number + _randomNumber = _randomNumber * (unsigned long long) 25214903917 + 11; + _randomIndex = uint(_randomNumber % (unsigned long long) _slidingWindowSize); +} + +float SVMSGD::calcInnerProduct(float *rowDataPointer){ + float innerProduct = 0; + for (uint feat = 0; feat < _nFeatures; ++feat){ + innerProduct += _weights[feat] * rowDataPointer[feat]; + } + return innerProduct; +} + +void SVMSGD::updateWeights(float innerProduct, float *rowDataPointer, int label){ + if (label * innerProduct > 1) { + // Not a support vector, only apply weight decay + for (uint feat = 0; feat < _nFeatures; feat++) { + _weights[feat] -= _learnRate * _lambda * _weights[feat]; + } + } else { + // It's a support vector, add it to the weights + for (uint feat = 0; feat < _nFeatures; feat++) { + _weights[feat] -= _learnRate * (_lambda * _weights[feat] - label * rowDataPointer[feat]); + } + } +} + +} +}