// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt /* This is an example showing how to define custom kernel functions for use with the machine learning tools in the dlib C++ Library. This example assumes you are somewhat familiar with the machine learning tools in dlib. In particular, you should be familiar with the krr_trainer and the matrix object. So you may want to read the krr_classification_ex.cpp and matrix_ex.cpp example programs if you haven't already. */ #include <iostream> #include <dlib/svm.h> using namespace std; using namespace dlib; // ---------------------------------------------------------------------------------------- /* Here we define our new kernel. It is the UKF kernel from Facilitating the applications of support vector machine by using a new kernel by Rui Zhang and Wenjian Wang. In the context of the dlib library a kernel function object is an object with an interface with the following properties: - a public typedef named sample_type - a public typedef named scalar_type which should be a float, double, or long double type. - an overloaded operator() that operates on two items of sample_type and returns a scalar_type. - a public typedef named mem_manager_type that is an implementation of dlib/memory_manager/memory_manager_kernel_abstract.h or dlib/memory_manager_global/memory_manager_global_kernel_abstract.h or dlib/memory_manager_stateless/memory_manager_stateless_kernel_abstract.h - an overloaded == operator that tells you if two kernels are identical or not. Below we define such a beast for the UKF kernel. In this case we are expecting the sample type (i.e. the T type) to be a dlib::matrix. However, note that you can design kernels which operate on any type you like so long as you meet the above requirements. */ template < typename T > structukf_kernel{typedef typename T::type scalar_type; typedef T sample_type; // If your sample type, the T, doesn't have a memory manager then // you can use dlib::default_memory_manager here. typedef typename T::mem_manager_type mem_manager_type;ukf_kernel(const scalar_type g) : sigma(g){}ukf_kernel() : sigma(0.1){}scalar_type sigma; scalar_typeoperator() ( const sample_type& a, const sample_type& b ) const{// This is the formula for the UKF kernel from the above referenced paper. return 1/(length_squared(a-b) + sigma);}booloperator== ( const ukf_kernel& k ) const{return sigma == k.sigma;}}; // ---------------------------------------------------------------------------------------- /* Here we define serialize() and deserialize() functions for our new kernel. Defining these functions is optional. However, if you don't define them you won't be able to save your learned decision_function objects to disk. */ template < typename T >voidserialize( const ukf_kernel<T>& item, std::ostream& out){// save the state of the kernel to the output stream serialize(item.sigma, out);}template < typename T >voiddeserialize( ukf_kernel<T>& item, std::istream& in ){deserialize(item.sigma, in);}// ---------------------------------------------------------------------------------------- /* This next thing, the kernel_derivative specialization is optional. You only need to define it if you want to use the dlib::reduced2() or dlib::approximate_distance_function() routines. If so, then you need to supply code for computing the derivative of your kernel as shown below. Note also that you can only do this if your kernel operates on dlib::matrix objects which represent column vectors. */ namespace dlib{template < typename T > structkernel_derivative<ukf_kernel<T> >{typedef typename T::type scalar_type; typedef T sample_type; typedef typename T::mem_manager_type mem_manager_type;kernel_derivative(const ukf_kernel<T>& k_) : k(k_){}sample_typeoperator() (const sample_type& x, const sample_type& y) const{// return the derivative of the ukf kernel with respect to the second argument (i.e. y) return 2*(x-y)*std::pow(k(x,y),2);}const ukf_kernel<T>& k;};}// ----------------------------------------------------------------------------------------intmain(){// We are going to be working with 2 dimensional samples and trying to perform // binary classification on them using our new ukf_kernel. typedef matrix<double, 2, 1> sample_type; typedef ukf_kernel<sample_type> kernel_type; // Now let's generate some training data std::vector<sample_type> samples; std::vector<double> labels; for (doubler = -20; r <= 20; r += 0.9){for (doublec = -20; c <= 20; c += 0.9){sample_type samp; samp(0) = r; samp(1) = c; samples.push_back(samp); // if this point is less than 13 from the origin if (sqrt(r*r + c*c) <= 13) labels.push_back(+1); else labels.push_back(-1);}}cout << "samples generated: " << samples.size() << endl; cout << " number of +1 samples: " << sum(mat(labels) > 0) << endl; cout << " number of -1 samples: " << sum(mat(labels) < 0) << endl; // A valid kernel must always give rise to kernel matrices which are symmetric // and positive semidefinite (i.e. have nonnegative eigenvalues). This next // bit of code makes a kernel matrix and checks if it has these properties. const matrix<double> K = kernel_matrix(kernel_type(0.1), randomly_subsample(samples, 500)); cout << "\nIs it symmetric? (this value should be 0): "<< min(abs(K - trans(K))) << endl; cout << "Smallest eigenvalue (should be >= 0): " << min(real_eigenvalues(K)) << endl; // here we make an instance of the krr_trainer object that uses our new kernel. krr_trainer<kernel_type> trainer; trainer.use_classification_loss_for_loo_cv(); // Finally, let's test how good our new kernel is by doing some leave-one-out cross-validation. cout << "\ndoing leave-one-out cross-validation" << endl; for (doublesigma = 0.01; sigma <= 100; sigma *= 3){// tell the trainer the parameters we want to use trainer.set_kernel(kernel_type(sigma)); std::vector<double> loo_values; trainer.train(samples, labels, loo_values); // Print sigma and the fraction of samples correctly classified during LOO cross-validation. constdoubleclassification_accuracy = mean_sign_agreement(labels, loo_values); cout << "sigma: " << sigma << " LOO accuracy: " << classification_accuracy << endl;}const kernel_type kern(10); // Since it is very easy to make a mistake while coding a derivative it is a good idea // to compare your derivative function against a numerical approximation and see if // the results are similar. If they are very different then you probably made a // mistake. So here we compare the results at a test point. cout << "\nThese vectors should match, if they don't then we coded the kernel_derivative wrong!" << endl; cout << "approximate derivative: \n" << derivative(kern)(samples[0],samples[100]) << endl; cout << "exact derivative: \n" << kernel_derivative<kernel_type>(kern)(samples[0],samples[100]) << endl;}