// Copyright (C) 2015 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_CORRELATION_TrACKER_H_
#define DLIB_CORRELATION_TrACKER_H_
#include "correlation_tracker_abstract.h"
#include "../geometry.h"
#include "../matrix.h"
#include "../array2d.h"
#include "../image_transforms/assign_image.h"
#include "../image_transforms/interpolation.h"
namespace dlib
{
// ----------------------------------------------------------------------------------------
class correlation_tracker
{
public:
explicit correlation_tracker (unsigned long filter_size = 6,
unsigned long num_scale_levels = 5,
unsigned long scale_window_size = 23,
double regularizer_space = 0.001,
double nu_space = 0.025,
double regularizer_scale = 0.001,
double nu_scale = 0.025,
double scale_pyramid_alpha = 1.020
)
: filter_size(1 << filter_size), num_scale_levels(1 << num_scale_levels),
scale_window_size(scale_window_size),
regularizer_space(regularizer_space), nu_space(nu_space),
regularizer_scale(regularizer_scale), nu_scale(nu_scale),
scale_pyramid_alpha(scale_pyramid_alpha)
{
// Create the cosine mask used for space filtering.
mask = make_cosine_mask();
// Create the cosine mask used for the scale filtering.
scale_cos_mask.resize(get_num_scale_levels());
const long max_level = get_num_scale_levels()/2;
for (unsigned long k = 0; k < get_num_scale_levels(); ++k)
{
double dist = std::abs((double)k-max_level)/max_level*pi/2;
dist = std::min(dist, pi/2);
scale_cos_mask[k] = std::cos(dist);
}
}
template <typename image_type>
void start_track (
const image_type& img,
const drectangle& p
)
{
DLIB_CASSERT(p.is_empty() == false,
"\t void correlation_tracker::start_track()"
<< "\n\t You can't give an empty rectangle."
);
B.set_size(0,0);
point_transform_affine tform = inv(make_chip(img, p, F));
for (unsigned long i = 0; i < F.size(); ++i)
fft_inplace(F[i]);
make_target_location_image(tform(center(p)), G);
A.resize(F.size());
for (unsigned long i = 0; i < F.size(); ++i)
{
A[i] = pointwise_multiply(G, F[i]);
B += squared(real(F[i]))+squared(imag(F[i]));
}
position = p;
// now do the scale space stuff
make_scale_space(img, Fs);
for (unsigned long i = 0; i < Fs.size(); ++i)
fft_inplace(Fs[i]);
make_scale_target_location_image(get_num_scale_levels()/2, Gs);
Bs.set_size(0);
As.resize(Fs.size());
for (unsigned long i = 0; i < Fs.size(); ++i)
{
As[i] = pointwise_multiply(Gs, Fs[i]);
Bs += squared(real(Fs[i]))+squared(imag(Fs[i]));
}
}
unsigned long get_filter_size (
) const { return filter_size; }
unsigned long get_num_scale_levels(
) const { return num_scale_levels; }
unsigned long get_scale_window_size (
) const { return scale_window_size; }
double get_regularizer_space (
) const { return regularizer_space; }
inline double get_nu_space (
) const { return nu_space;}
double get_regularizer_scale (
) const { return regularizer_scale; }
double get_nu_scale (
) const { return nu_scale;}
drectangle get_position (
) const
{
return position;
}
double get_scale_pyramid_alpha (
) const { return scale_pyramid_alpha; }
template <typename image_type>
double update_noscale(
const image_type& img,
const drectangle& guess
)
{
DLIB_CASSERT(get_position().is_empty() == false,
"\t double correlation_tracker::update()"
<< "\n\t You must call start_track() first before calling update()."
);
const point_transform_affine tform = make_chip(img, guess, F);
for (unsigned long i = 0; i < F.size(); ++i)
fft_inplace(F[i]);
// use the current filter to predict the object's location
G = 0;
for (unsigned long i = 0; i < F.size(); ++i)
G += pointwise_multiply(F[i],conj(A[i]));
G = pointwise_multiply(G, reciprocal(B+get_regularizer_space()));
ifft_inplace(G);
const dlib::vector<double,2> pp = max_point_interpolated(real(G));
// Compute the peak to side lobe ratio.
const point p = pp;
running_stats<double> rs;
const rectangle peak = centered_rect(p, 8,8);
for (long r = 0; r < G.nr(); ++r)
{
for (long c = 0; c < G.nc(); ++c)
{
if (!peak.contains(point(c,r)))
rs.add(G(r,c).real());
}
}
const double psr = (G(p.y(),p.x()).real()-rs.mean())/rs.stddev();
// update the position of the object
position = translate_rect(guess, tform(pp)-center(guess));
// now update the position filters
make_target_location_image(pp, G);
B *= (1-get_nu_space());
for (unsigned long i = 0; i < F.size(); ++i)
{
A[i] = get_nu_space()*pointwise_multiply(G, F[i]) + (1-get_nu_space())*A[i];
B += get_nu_space()*(squared(real(F[i]))+squared(imag(F[i])));
}
return psr;
}
template <typename image_type>
double update (
const image_type& img,
const drectangle& guess
)
{
double psr = update_noscale(img, guess);
// Now predict the scale change
make_scale_space(img, Fs);
for (unsigned long i = 0; i < Fs.size(); ++i)
fft_inplace(Fs[i]);
Gs = 0;
for (unsigned long i = 0; i < Fs.size(); ++i)
Gs += pointwise_multiply(Fs[i],conj(As[i]));
Gs = pointwise_multiply(Gs, reciprocal(Bs+get_regularizer_scale()));
ifft_inplace(Gs);
const double pos = max_point_interpolated(real(Gs)).y();
// update the rectangle's scale
position *= std::pow(get_scale_pyramid_alpha(), pos-(double)get_num_scale_levels()/2);
// Now update the scale filters
make_scale_target_location_image(pos, Gs);
Bs *= (1-get_nu_scale());
for (unsigned long i = 0; i < Fs.size(); ++i)
{
As[i] = get_nu_scale()*pointwise_multiply(Gs, Fs[i]) + (1-get_nu_scale())*As[i];
Bs += get_nu_scale()*(squared(real(Fs[i]))+squared(imag(Fs[i])));
}
return psr;
}
template <typename image_type>
double update_noscale (
const image_type& img
)
{
return update_noscale(img, get_position());
}
template <typename image_type>
double update(
const image_type& img
)
{
return update(img, get_position());
}
private:
template <typename image_type>
void make_scale_space(
const image_type& img,
std::vector<matrix<std::complex<double>,0,1> >& Fs
) const
{
typedef typename image_traits<image_type>::pixel_type pixel_type;
// Make an image pyramid and put it into the chips array.
const long chip_size = get_scale_window_size();
drectangle ppp = position*std::pow(get_scale_pyramid_alpha(), -(double)get_num_scale_levels()/2);
dlib::array<array2d<pixel_type> > chips;
std::vector<dlib::vector<double,2> > from_points, to_points;
from_points.push_back(point(0,0));
from_points.push_back(point(chip_size-1,0));
from_points.push_back(point(chip_size-1,chip_size-1));
for (unsigned long i = 0; i < get_num_scale_levels(); ++i)
{
array2d<pixel_type> chip(chip_size,chip_size);
// pull box into chip
to_points.clear();
to_points.push_back(ppp.tl_corner());
to_points.push_back(ppp.tr_corner());
to_points.push_back(ppp.br_corner());
transform_image(img,chip,interpolate_bilinear(),find_affine_transform(from_points, to_points));
chips.push_back(chip);
ppp *= get_scale_pyramid_alpha();
}
// extract HOG for each chip
dlib::array<dlib::array<array2d<float> > > hogs(chips.size());
for (unsigned long i = 0; i < chips.size(); ++i)
{
extract_fhog_features(chips[i], hogs[i], 4);
hogs[i].resize(32);
assign_image(hogs[i][31], chips[i]);
assign_image(hogs[i][31], mat(hogs[i][31])/255.0);
}
// Now copy the hog features into the Fs outputs and also apply the cosine
// windowing.
Fs.resize(hogs[0].size()*hogs[0][0].size());
unsigned long i = 0;
for (long r = 0; r < hogs[0][0].nr(); ++r)
{
for (long c = 0; c < hogs[0][0].nc(); ++c)
{
for (unsigned long j = 0; j < hogs[0].size(); ++j)
{
Fs[i].set_size(hogs.size());
for (unsigned long k = 0; k < hogs.size(); ++k)
{
Fs[i](k) = hogs[k][j][r][c]*scale_cos_mask[k];
}
++i;
}
}
}
}
template <typename image_type>
point_transform_affine make_chip (
const image_type& img,
drectangle p,
std::vector<matrix<std::complex<double> > >& chip
) const
{
typedef typename image_traits<image_type>::pixel_type pixel_type;
array2d<pixel_type> temp;
const double padding = 1.4;
const chip_details details(p*padding, chip_dims(get_filter_size(), get_filter_size()));
extract_image_chip(img, details, temp);
chip.resize(32);
dlib::array<array2d<float> > hog;
extract_fhog_features(temp, hog, 1, 3,3 );
for (unsigned long i = 0; i < hog.size(); ++i)
assign_image(chip[i], pointwise_multiply(matrix_cast<double>(mat(hog[i])), mask));
assign_image(chip[31], temp);
assign_image(chip[31], pointwise_multiply(mat(chip[31]), mask)/255.0);
return inv(get_mapping_to_chip(details));
}
void make_target_location_image (
const dlib::vector<double,2>& p,
matrix<std::complex<double> >& g
) const
{
g.set_size(get_filter_size(), get_filter_size());
g = 0;
rectangle area = centered_rect(p, 21,21).intersect(get_rect(g));
for (long r = area.top(); r <= area.bottom(); ++r)
{
for (long c = area.left(); c <= area.right(); ++c)
{
double dist = length(point(c,r)-p);
g(r,c) = std::exp(-dist/3.0);
}
}
fft_inplace(g);
g = conj(g);
}
void make_scale_target_location_image (
const double scale,
matrix<std::complex<double>,0,1>& g
) const
{
g.set_size(get_num_scale_levels());
for (long i = 0; i < g.size(); ++i)
{
double dist = std::pow((i-scale),2.0);
g(i) = std::exp(-dist/1.000);
}
fft_inplace(g);
g = conj(g);
}
matrix<double> make_cosine_mask (
) const
{
const long size = get_filter_size();
matrix<double> temp(size,size);
point cent = center(get_rect(temp));
for (long r = 0; r < temp.nr(); ++r)
{
for (long c = 0; c < temp.nc(); ++c)
{
point delta = point(c,r)-cent;
double dist = length(delta)/(size/2.0)*(pi/2);
dist = std::min(dist*1.0, pi/2);
temp(r,c) = std::cos(dist);
}
}
return temp;
}
std::vector<matrix<std::complex<double> > > A, F;
matrix<double> B;
std::vector<matrix<std::complex<double>,0,1> > As, Fs;
matrix<double,0,1> Bs;
drectangle position;
matrix<double> mask;
std::vector<double> scale_cos_mask;
// G and Gs do not logically contribute to the state of this object. They are
// here just so we can void reallocating them over and over.
matrix<std::complex<double> > G;
matrix<std::complex<double>,0,1> Gs;
unsigned long filter_size;
unsigned long num_scale_levels;
unsigned long scale_window_size;
double regularizer_space;
double nu_space;
double regularizer_scale;
double nu_scale;
double scale_pyramid_alpha;
};
}
#endif // DLIB_CORRELATION_TrACKER_H_