/* Tracker based on Kernelized Correlation Filter (KCF) [1] and Circulant Structure with Kernels (CSK) [2]. CSK is implemented by using raw gray level features, since it is a single-channel filter. KCF is implemented by using HOG features (the default), since it extends CSK to multiple channels. [1] J. F. Henriques, R. Caseiro, P. Martins, J. Batista, "High-Speed Tracking with Kernelized Correlation Filters", TPAMI 2015. [2] J. F. Henriques, R. Caseiro, P. Martins, J. Batista, "Exploiting the Circulant Structure of Tracking-by-detection with Kernels", ECCV 2012. Authors: Joao Faro, Christian Bailer, Joao F. Henriques Contacts: joaopfaro@gmail.com, Christian.Bailer@dfki.de, henriques@isr.uc.pt Institute of Systems and Robotics - University of Coimbra / Department Augmented Vision DFKI Constructor parameters, all boolean: hog: use HOG features (default), otherwise use raw pixels fixed_window: fix window size (default), otherwise use ROI size (slower but more accurate) multiscale: use multi-scale tracking (default; cannot be used with fixed_window = true) Default values are set for all properties of the tracker depending on the above choices. Their values can be customized further before calling init(): interp_factor: linear interpolation factor for adaptation sigma: gaussian kernel bandwidth lambda: regularization cell_size: HOG cell size padding: area surrounding the target, relative to its size output_sigma_factor: bandwidth of gaussian target template_size: template size in pixels, 0 to use ROI size scale_step: scale step for multi-scale estimation, 1 to disable it scale_weight: to downweight detection scores of other scales for added stability For speed, the value (template_size/cell_size) should be a power of 2 or a product of small prime numbers. Inputs to init(): image is the initial frame. roi is a cv::Rect with the target positions in the initial frame Inputs to update(): image is the current frame. Outputs of update(): cv::Rect with target positions for the current frame By downloading, copying, installing or using the software you agree to this license. If you do not agree to this license, do not download, install, copy or use the software. License Agreement For Open Source Computer Vision Library (3-clause BSD License) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the names of the copyright holders nor the names of the contributors may be used to endorse or promote products derived from this software without specific prior written permission. This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall copyright holders or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. */ //#ifndef _KCFTRACKER_HEADERS #include "stdafx.h" #include "kcftracker.hpp" #include "ffttools.hpp" #include "recttools.hpp" #include "fhog.hpp" #include "labdata.hpp" //#endif // Constructor KCFTracker::KCFTracker(bool hog, bool fixed_window, bool multiscale, bool lab) { // Parameters equal in all cases lambda = 0.0001; padding = 2.5; //output_sigma_factor = 0.1; output_sigma_factor = 0.125; if (hog) { // HOG // VOT interp_factor = 0.012; sigma = 0.6; // TPAMI //interp_factor = 0.02; //sigma = 0.5; cell_size = 4; _hogfeatures = true; if (lab) { interp_factor = 0.005; sigma = 0.4; //output_sigma_factor = 0.025; output_sigma_factor = 0.1; _labfeatures = true; _labCentroids = cv::Mat(nClusters, 3, CV_32FC1, &data); cell_sizeQ = cell_size*cell_size; } else{ _labfeatures = false; } } else { // RAW interp_factor = 0.075; sigma = 0.2; cell_size = 1; _hogfeatures = false; if (lab) { printf("Lab features are only used with HOG features.\n"); _labfeatures = false; } } if (multiscale) { // multiscale template_size = 96; //template_size = 100; scale_step = 1.05; scale_weight = 0.95; if (!fixed_window) { //printf("Multiscale does not support non-fixed window.\n"); fixed_window = true; } } else if (fixed_window) { // fit correction without multiscale template_size = 96; //template_size = 100; scale_step = 1; } else { template_size = 1; scale_step = 1; } } // Initialize tracker void KCFTracker::init(const cv::Rect &roi, cv::Mat image) { _roi = roi; assert(roi.width >= 0 && roi.height >= 0); _tmpl = getFeatures(image, 1);//只有第一次初始化的时候，第二个形参才为1，对第一帧特征进行汉宁窗平滑 _prob = createGaussianPeak(size_patch[0], size_patch[1]);//创建高斯峰，只有第一帧才用到 _alphaf = cv::Mat(size_patch[0], size_patch[1], CV_32FC2, float(0));////alphaf初始化 //_num = cv::Mat(size_patch[0], size_patch[1], CV_32FC2, float(0)); //_den = cv::Mat(size_patch[0], size_patch[1], CV_32FC2, float(0)); train(_tmpl, 1.0); // train with initial frame } // Update position based on the new frame cv::Rect KCFTracker::update(cv::Mat image) { if (_roi.x + _roi.width <= 0) _roi.x = -_roi.width + 1; if (_roi.y + _roi.height <= 0) _roi.y = -_roi.height + 1; if (_roi.x >= image.cols - 1) _roi.x = image.cols - 2; if (_roi.y >= image.rows - 1) _roi.y = image.rows - 2; float cx = _roi.x + _roi.width / 2.0f;//中心点 float cy = _roi.y + _roi.height / 2.0f; float peak_value; cv::Point2f res = detect(_tmpl, getFeatures(image, 0, 1.0f), peak_value);//获取response的位置 //处理尺度变化的情况 if (scale_step != 1) { // Test at a smaller _scale float new_peak_value; cv::Point2f new_res = detect(_tmpl, getFeatures(image, 0, 1.0f / scale_step), new_peak_value); //update roi and other parameter 更新roi区域及其参数 if (scale_weight * new_peak_value > peak_value) { res = new_res; peak_value = new_peak_value; _scale /= scale_step; _roi.width /= scale_step; _roi.height /= scale_step; } // Test at a bigger _scale new_res = detect(_tmpl, getFeatures(image, 0, scale_step), new_peak_value); if (scale_weight * new_peak_value > peak_value) { res = new_res; peak_value = new_peak_value; _scale *= scale_step; _roi.width *= scale_step; _roi.height *= scale_step; } } // Adjust by cell size and _scale ????????????cell size???????????????????? _roi.x = cx - _roi.width / 2.0f + ((float) res.x * cell_size * _scale); _roi.y = cy - _roi.height / 2.0f + ((float) res.y * cell_size * _scale); //超出边界的情况 if (_roi.x >= image.cols - 1) _roi.x = image.cols - 1; if (_roi.y >= image.rows - 1) _roi.y = image.rows - 1; if (_roi.x + _roi.width <= 0) _roi.x = -_roi.width + 2; if (_roi.y + _roi.height <= 0) _roi.y = -_roi.height + 2; assert(_roi.width >= 0 && _roi.height >= 0); //上面得到roi新的位置之后，再重新训练得到相关滤波器 cv::Mat x = getFeatures(image, 0);//提取新的roi特征 train(x, interp_factor);//训练得到新的滤波器 return _roi; } // Detect object in the current frame. cv::Point2f KCFTracker::detect(cv::Mat z, cv::Mat x, float &peak_value) { using namespace FFTTools; cv::Mat k = gaussianCorrelation(x, z);//作相关运算 cv::Mat res = (real(fftd(complexMultiplication(_alphaf, fftd(k)), true)));//获得response //minMaxLoc only accepts doubles for the peak, and integer points for the coordinates cv::Point2i pi;//存放响应response最大值所在的位置 double pv;//pv存放响应response最大值 //找到输入数组的最大/最小值，此处寻找最大值，pv存放最大值，pi存放最大值所在的位置 cv::minMaxLoc(res, NULL, &pv, NULL, &pi); peak_value = (float) pv; //subpixel peak estimation, coordinates will be non-integer cv::Point2f p((float)pi.x, (float)pi.y); // //target location is at the maximum response. we must take into //account the fact that, if the target doesn't move, the peak //will appear at the top - left corner, not at the center(this is //discussed in the paper).the responses wrap around cyclically. if (pi.x > 0 && pi.x < res.cols-1) { p.x += subPixelPeak(res.at(pi.y, pi.x-1), peak_value, res.at(pi.y, pi.x+1)); } if (pi.y > 0 && pi.y < res.rows-1) { p.y += subPixelPeak(res.at(pi.y-1, pi.x), peak_value, res.at(pi.y+1, pi.x)); } p.x -= (res.cols) / 2; p.y -= (res.rows) / 2; return p; //responses最大响应对应的目标位置 } // train tracker with a single image void KCFTracker::train(cv::Mat x, float train_interp_factor) { using namespace FFTTools; cv::Mat k = gaussianCorrelation(x, x); cv::Mat alphaf = complexDivision(_prob, (fftd(k) + lambda)); _tmpl = (1 - train_interp_factor) * _tmpl + (train_interp_factor) * x; _alphaf = (1 - train_interp_factor) * _alphaf + (train_interp_factor) * alphaf; /*cv::Mat kf = fftd(gaussianCorrelation(x, x)); cv::Mat num = complexMultiplication(kf, _prob); cv::Mat den = complexMultiplication(kf, kf + lambda); _tmpl = (1 - train_interp_factor) * _tmpl + (train_interp_factor) * x; _num = (1 - train_interp_factor) * _num + (train_interp_factor) * num; _den = (1 - train_interp_factor) * _den + (train_interp_factor) * den; _alphaf = complexDivision(_num, _den);*/ } // Evaluates a Gaussian kernel with bandwidth SIGMA for all relative shifts between input images X and Y, // which must both be MxN. They must also be periodic (ie., pre-processed with a cosine window). cv::Mat KCFTracker::gaussianCorrelation(cv::Mat x1, cv::Mat x2) { using namespace FFTTools; cv::Mat c = cv::Mat( cv::Size(size_patch[1], size_patch[0]), CV_32F, cv::Scalar(0) ); // HOG features if (_hogfeatures) { cv::Mat caux; cv::Mat x1aux; cv::Mat x2aux; for (int i = 0; i < size_patch[2]; i++) { x1aux = x1.row(i); // Procedure do deal with cv::Mat multichannel bug x1aux = x1aux.reshape(1, size_patch[0]); x2aux = x2.row(i).reshape(1, size_patch[0]); cv::mulSpectrums(fftd(x1aux), fftd(x2aux), caux, 0, true); caux = fftd(caux, true); rearrange(caux); caux.convertTo(caux,CV_32F); c = c + real(caux); } } // Gray features else { cv::mulSpectrums(fftd(x1), fftd(x2), c, 0, true); c = fftd(c, true); rearrange(c); c = real(c); } cv::Mat d; cv::max(( (cv::sum(x1.mul(x1))[0] + cv::sum(x2.mul(x2))[0])- 2. * c) / (size_patch[0]*size_patch[1]*size_patch[2]) , 0, d); cv::Mat k; cv::exp((-d / (sigma * sigma)), k); return k; } // Create Gaussian Peak. Function called only in the first frame. cv::Mat KCFTracker::createGaussianPeak(int sizey, int sizex) { cv::Mat_ res(sizey, sizex); int syh = (sizey) / 2; int sxh = (sizex) / 2; float output_sigma = std::sqrt((float) sizex * sizey) / padding * output_sigma_factor; float mult = -0.5 / (output_sigma * output_sigma); for (int i = 0; i < sizey; i++) for (int j = 0; j < sizex; j++) { int ih = i - syh; int jh = j - sxh; res(i, j) = std::exp(mult * (float) (ih * ih + jh * jh)); } return FFTTools::fftd(res); } // Obtain sub-window from image, with replication-padding and extract features cv::Mat KCFTracker::getFeatures(const cv::Mat & image, bool inithann, float scale_adjust) { cv::Rect extracted_roi; float cx = _roi.x + _roi.width / 2; float cy = _roi.y + _roi.height / 2; if (inithann) {//汉宁窗?????? int padded_w = _roi.width * padding; int padded_h = _roi.height * padding; if (template_size > 1) { // Fit largest dimension to the given template size if (padded_w >= padded_h) //fit to width _scale = padded_w / (float) template_size; else _scale = padded_h / (float) template_size; _tmpl_sz.width = padded_w / _scale; _tmpl_sz.height = padded_h / _scale; } else { //No template size given, use ROI size _tmpl_sz.width = padded_w; _tmpl_sz.height = padded_h; _scale = 1; // original code from paper: /*if (sqrt(padded_w * padded_h) >= 100) { //Normal size _tmpl_sz.width = padded_w; _tmpl_sz.height = padded_h; _scale = 1; } else { //ROI is too big, track at half size _tmpl_sz.width = padded_w / 2; _tmpl_sz.height = padded_h / 2; _scale = 2; }*/ } if (_hogfeatures) { // Round to cell size and also make it even _tmpl_sz.width = ( ( (int)(_tmpl_sz.width / (2 * cell_size)) ) * 2 * cell_size ) + cell_size*2; _tmpl_sz.height = ( ( (int)(_tmpl_sz.height / (2 * cell_size)) ) * 2 * cell_size ) + cell_size*2; } else { //Make number of pixels even (helps with some logic involving half-dimensions) _tmpl_sz.width = (_tmpl_sz.width / 2) * 2; _tmpl_sz.height = (_tmpl_sz.height / 2) * 2; } } extracted_roi.width = scale_adjust * _scale * _tmpl_sz.width; extracted_roi.height = scale_adjust * _scale * _tmpl_sz.height; // center roi with new size extracted_roi.x = cx - extracted_roi.width / 2; extracted_roi.y = cy - extracted_roi.height / 2; cv::Mat FeaturesMap; //obtain a subwindow for detection at the position from last // frame, and convert to Fourier domain(its size is unchanged) // 在本帧中获取前一帧目标位置的子窗口 cv::Mat z = RectTools::subwindow(image, extracted_roi, cv::BORDER_REPLICATE); //然后要对z提取特征，然后再到频域上与相关滤波器作相关，得到response之后产生新的目标位置 ////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////// if (z.cols != _tmpl_sz.width || z.rows != _tmpl_sz.height) { cv::resize(z, z, _tmpl_sz); } // HOG features if (_hogfeatures) { IplImage z_ipl = z; CvLSVMFeatureMapCaskade *map; getFeatureMaps(&z_ipl, cell_size, &map); normalizeAndTruncate(map,0.2f); PCAFeatureMaps(map); size_patch[0] = map->sizeY; size_patch[1] = map->sizeX; size_patch[2] = map->numFeatures; FeaturesMap = cv::Mat(cv::Size(map->numFeatures,map->sizeX*map->sizeY), CV_32F, map->map); // Procedure do deal with cv::Mat multichannel bug FeaturesMap = FeaturesMap.t(); freeFeatureMapObject(&map); // Lab features //当lab = false时，用不到 if (_labfeatures) { cv::Mat imgLab; cvtColor(z, imgLab, CV_BGR2Lab); unsigned char *input = (unsigned char*)(imgLab.data); // Sparse output vector cv::Mat outputLab = cv::Mat(_labCentroids.rows, size_patch[0]*size_patch[1], CV_32F, float(0)); int cntCell = 0; // Iterate through each cell for (int cY = cell_size; cY < z.rows-cell_size; cY+=cell_size){ for (int cX = cell_size; cX < z.cols-cell_size; cX+=cell_size){ // Iterate through each pixel of cell (cX,cY) for(int y = cY; y < cY+cell_size; ++y){ for(int x = cX; x < cX+cell_size; ++x){ // Lab components for each pixel float l = (float)input[(z.cols * y + x) * 3]; float a = (float)input[(z.cols * y + x) * 3 + 1]; float b = (float)input[(z.cols * y + x) * 3 + 2]; // Iterate trough each centroid float minDist = FLT_MAX; int minIdx = 0; float *inputCentroid = (float*)(_labCentroids.data); for(int k = 0; k < _labCentroids.rows; ++k){ float dist = ( (l - inputCentroid[3*k]) * (l - inputCentroid[3*k]) ) + ( (a - inputCentroid[3*k+1]) * (a - inputCentroid[3*k+1]) ) + ( (b - inputCentroid[3*k+2]) * (b - inputCentroid[3*k+2]) ); if(dist < minDist){ minDist = dist; minIdx = k; } } // Store result at output outputLab.at(minIdx, cntCell) += 1.0 / cell_sizeQ; //((float*) outputLab.data)[minIdx * (size_patch[0]*size_patch[1]) + cntCell] += 1.0 / cell_sizeQ; } } cntCell++; } } // Update size_patch[2] and add features to FeaturesMap size_patch[2] += _labCentroids.rows; FeaturesMap.push_back(outputLab); } } else { //raw pixel FeaturesMap = RectTools::getGrayImage(z); FeaturesMap -= (float) 0.5; // In Paper; size_patch[0] = z.rows; size_patch[1] = z.cols; size_patch[2] = 1; } if (inithann) {//只有在第一帧的时候才会用到，创建/初始化 汉宁窗 createHanningMats(); } FeaturesMap = hann.mul(FeaturesMap);//特征与汉宁窗相乘，起平滑作用 return FeaturesMap; //最后返回的是与汉宁窗相乘后的结果，，，后续还要进行与相关滤波器作相关 ////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////// } // Initialize Hanning window. Function called only in the first frame. void KCFTracker::createHanningMats() { cv::Mat hann1t = cv::Mat(cv::Size(size_patch[1],1), CV_32F, cv::Scalar(0)); cv::Mat hann2t = cv::Mat(cv::Size(1,size_patch[0]), CV_32F, cv::Scalar(0)); for (int i = 0; i < hann1t.cols; i++) hann1t.at (0, i) = 0.5 * (1 - std::cos(2 * 3.14159265358979323846 * i / (hann1t.cols - 1))); for (int i = 0; i < hann2t.rows; i++) hann2t.at (i, 0) = 0.5 * (1 - std::cos(2 * 3.14159265358979323846 * i / (hann2t.rows - 1))); cv::Mat hann2d = hann2t * hann1t; // HOG features if (_hogfeatures) { cv::Mat hann1d = hann2d.reshape(1,1); // Procedure do deal with cv::Mat multichannel bug hann = cv::Mat(cv::Size(size_patch[0]*size_patch[1], size_patch[2]), CV_32F, cv::Scalar(0)); for (int i = 0; i < size_patch[2]; i++) { for (int j = 0; j(i,j) = hann1d.at(0,j); } } } // Gray features else { hann = hann2d; } } // Calculate sub-pixel peak for one dimension float KCFTracker::subPixelPeak(float left, float center, float right) { float divisor = 2 * center - right - left; //divisor = 0.28 0.35 0.27 0.37 0.30 0.33 0.24 0.38 //printf("%f \n", divisor); if (divisor == 0) return 0; return 0.5 * (right - left) / divisor; }