// Copyright (C) 2012 Davis E. King (davis@dlib.net) // License: Boost Software License See LICENSE.txt for the full license. #include <dlib/svm.h> #include <dlib/rand.h> #include <dlib/statistics.h> #include "tester.h" namespace { using namespace test; using namespace dlib; using namespace std; logger dlog("test.svm_c_linear_dcd"); // ---------------------------------------------------------------------------------------- void test_sparse() { typedef std::map<unsigned long,double> sample_type; typedef sparse_linear_kernel<sample_type> kernel_type; svm_c_linear_trainer<kernel_type> linear_trainer_cpa; svm_c_linear_dcd_trainer<kernel_type> linear_trainer; svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state; const double C = 0.2; linear_trainer.set_epsilon(1e-10); linear_trainer_cpa.set_epsilon(1e-10); linear_trainer_cpa.set_relative_epsilon(1e-10); std::vector<sample_type> samples; std::vector<double> labels; // make an instance of a sample vector so we can use it below sample_type sample; decision_function<kernel_type> df, df2, df3; dlib::rand rnd; // Now lets go into a loop and randomly generate 10000 samples. double label = +1; for (int i = 0; i < 100; ++i) { // flip this flag label *= -1; sample.clear(); // now make a random sparse sample with at most 10 non-zero elements for (int j = 0; j < 5; ++j) { int idx = rnd.get_random_32bit_number()%10; double value = rnd.get_random_double(); sample[idx] = label*value; } // Also save the samples we are generating so we can let the svm_c_linear_trainer // learn from them below. samples.push_back(sample); labels.push_back(label); if (samples.size() > 1) { linear_trainer_cpa.set_c_class1(C); linear_trainer_cpa.set_c_class2(1.5*C); linear_trainer.set_c_class1(C/samples.size()); linear_trainer.set_c_class2(1.5*C/samples.size()); df = linear_trainer.train(samples, labels, state); df2 = linear_trainer_cpa.train(samples, labels); df3 = linear_trainer.train(samples, labels); DLIB_TEST_MSG( dlib::distance(df.basis_vectors(0), df2.basis_vectors(0)) < 1e-8, dlib::distance(df.basis_vectors(0), df2.basis_vectors(0))); DLIB_TEST( std::abs(df.b - df2.b) < 1e-8); DLIB_TEST( dlib::distance(df.basis_vectors(0), df3.basis_vectors(0)) < 1e-8); DLIB_TEST( std::abs(df.b - df3.b) < 1e-8); } } } // ---------------------------------------------------------------------------------------- void test_normal_no_bias() { typedef matrix<double,10,1> sample_type; typedef linear_kernel<sample_type> kernel_type; svm_c_linear_trainer<kernel_type> linear_trainer_cpa; svm_c_linear_dcd_trainer<kernel_type> linear_trainer; svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state; const double C = 1.0; linear_trainer.set_epsilon(1e-10); linear_trainer_cpa.set_epsilon(1e-10); linear_trainer_cpa.set_relative_epsilon(1e-10); linear_trainer.include_bias(false); std::vector<sample_type> samples, samples_explict_bias; std::vector<double> labels; // make an instance of a sample vector so we can use it below sample_type sample; decision_function<kernel_type> df, df2, df3; dlib::rand rnd; // Now lets go into a loop and randomly generate 10000 samples. double label = +1; for (int i = 0; i < 100; ++i) { // flip this flag label *= -1; sample = 0; // now make a random sparse sample with at most 10 non-zero elements for (int j = 0; j < 5; ++j) { int idx = rnd.get_random_32bit_number()%9; double value = rnd.get_random_double(); sample(idx) = label*value; } // Also save the samples we are generating so we can let the svm_c_linear_trainer // learn from them below. samples.push_back(sample); labels.push_back(label); sample(9) = -1; samples_explict_bias.push_back(sample); if (samples.size() > 1) { linear_trainer_cpa.set_c_class1(C); linear_trainer_cpa.set_c_class2(1.5*C); linear_trainer.set_c_class1(C/samples.size()); linear_trainer.set_c_class2(1.5*C/samples.size()); df = linear_trainer.train(samples_explict_bias, labels, state); df2 = linear_trainer_cpa.train(samples, labels); df3 = linear_trainer.train(samples_explict_bias, labels); DLIB_TEST( std::abs(df2.basis_vectors(0)(9)) < 1e-7); DLIB_TEST_MSG( max(abs(colm(df.basis_vectors(0),0,9) - colm(df2.basis_vectors(0),0,9))) < 1e-6, max(abs(colm(df.basis_vectors(0),0,9) - colm(df2.basis_vectors(0),0,9)))); DLIB_TEST( std::abs(df.basis_vectors(0)(9) - df2.b) < 1e-6); DLIB_TEST( max(abs(df.basis_vectors(0) - df3.basis_vectors(0))) < 1e-6); DLIB_TEST( std::abs(df.b - df3.b) < 1e-7); } } } // ---------------------------------------------------------------------------------------- void test_normal() { typedef matrix<double,10,1> sample_type; typedef linear_kernel<sample_type> kernel_type; svm_c_linear_trainer<kernel_type> linear_trainer_cpa; svm_c_linear_dcd_trainer<kernel_type> linear_trainer; svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state; const double C = 1; linear_trainer.set_epsilon(1e-10); linear_trainer_cpa.set_epsilon(1e-10); linear_trainer_cpa.set_relative_epsilon(1e-10); std::vector<sample_type> samples; std::vector<double> labels; // make an instance of a sample vector so we can use it below sample_type sample; decision_function<kernel_type> df, df2, df3; dlib::rand rnd; // Now lets go into a loop and randomly generate 10000 samples. double label = +1; for (int i = 0; i < 100; ++i) { // flip this flag label *= -1; sample = 0; // now make a random sparse sample with at most 10 non-zero elements for (int j = 0; j < 5; ++j) { int idx = rnd.get_random_32bit_number()%10; double value = rnd.get_random_double(); sample(idx) = label*value; } // Also save the samples we are generating so we can let the svm_c_linear_trainer // learn from them below. samples.push_back(sample); labels.push_back(label); if (samples.size() > 1) { linear_trainer_cpa.set_c_class1(C); linear_trainer_cpa.set_c_class2(1.5*C); linear_trainer.set_c_class1(C/samples.size()); linear_trainer.set_c_class2(1.5*C/samples.size()); df = linear_trainer.train(samples, labels, state); df2 = linear_trainer_cpa.train(samples, labels); df3 = linear_trainer.train(samples, labels); DLIB_TEST_MSG( max(abs(df.basis_vectors(0) - df2.basis_vectors(0))) < 1e-7, max(abs(df.basis_vectors(0) - df2.basis_vectors(0)))); DLIB_TEST( std::abs(df.b - df2.b) < 1e-7); DLIB_TEST( max(abs(df.basis_vectors(0) - df3.basis_vectors(0))) < 1e-7); DLIB_TEST( std::abs(df.b - df3.b) < 1e-7); } } } // ---------------------------------------------------------------------------------------- void test_normal_force_last_weight(bool have_bias, bool force_weight) { typedef matrix<double,10,1> sample_type; dlog << LINFO << "have_bias: "<< have_bias << " force_weight: "<< force_weight; typedef linear_kernel<sample_type> kernel_type; svm_c_linear_trainer<kernel_type> linear_trainer_cpa; svm_c_linear_dcd_trainer<kernel_type> linear_trainer; svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state; const double C = 1; linear_trainer.set_epsilon(1e-10); linear_trainer_cpa.set_epsilon(1e-11); linear_trainer_cpa.set_relative_epsilon(1e-11); linear_trainer_cpa.force_last_weight_to_1(force_weight); linear_trainer.force_last_weight_to_1(force_weight); linear_trainer.include_bias(have_bias); std::vector<sample_type> samples; std::vector<double> labels; // make an instance of a sample vector so we can use it below sample_type sample; decision_function<kernel_type> df, df2; running_stats<double> rs; dlib::rand rnd; // Now lets go into a loop and randomly generate 10000 samples. double label = +1; for (int i = 0; i < 40; ++i) { // flip this flag label *= -1; sample = 0; // now make a random sparse sample with at most 10 non-zero elements for (int j = 0; j < 5; ++j) { int idx = rnd.get_random_32bit_number()%9; double value = rnd.get_random_double(); sample(idx) = label*value + label; } sample(9) = 4; // Also save the samples we are generating so we can let the svm_c_linear_trainer // learn from them below. samples.push_back(sample); labels.push_back(label); linear_trainer.set_c(C); linear_trainer_cpa.set_c(C*samples.size()); df = linear_trainer.train(samples, labels, state); if (force_weight) { DLIB_TEST(std::abs(df.basis_vectors(0)(9) - 1) < 1e-8); DLIB_TEST(std::abs(df.b) < 1e-8); if (samples.size() > 1) { df2 = linear_trainer_cpa.train(samples, labels); DLIB_TEST_MSG( max(abs(df.basis_vectors(0) - df2.basis_vectors(0))) < 1e-7, max(abs(df.basis_vectors(0) - df2.basis_vectors(0)))); DLIB_TEST( std::abs(df.b - df2.b) < 1e-7); } } if (!have_bias) DLIB_TEST(std::abs(df.b) < 1e-8); for (unsigned long k = 0; k < samples.size(); ++k) { //cout << "pred: "<< labels[k]*df(samples[k]) << endl; rs.add(labels[k]*df(samples[k])); } } DLIB_TEST_MSG(std::abs(rs.min()-1) < 1e-7, std::abs(rs.min()-1)); } // ---------------------------------------------------------------------------------------- void test_normal_1_sample(double label) { typedef matrix<double,10,1> sample_type; typedef linear_kernel<sample_type> kernel_type; svm_c_linear_dcd_trainer<kernel_type> linear_trainer; svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state; const double C = 10; linear_trainer.set_epsilon(1e-10); linear_trainer.set_c(C); linear_trainer.force_last_weight_to_1(true); linear_trainer.include_bias(false); std::vector<sample_type> samples; std::vector<double> labels; // make an instance of a sample vector so we can use it below sample_type sample; sample = 0; sample(0) = -1; sample(1) = -1; sample(9) = 4; samples.push_back(sample); labels.push_back(label); for (int i = 0; i < 4; ++i) { decision_function<kernel_type> df; df = linear_trainer.train(samples, labels); if (label > 0) { DLIB_TEST(std::abs(df(samples[0])-4) < 1e-8); } else { DLIB_TEST(std::abs(df(samples[0])+1) < 1e-8); } } } // ---------------------------------------------------------------------------------------- void test_sparse_1_sample(double label) { typedef std::vector<std::pair<unsigned long,double> > sample_type; typedef sparse_linear_kernel<sample_type> kernel_type; svm_c_linear_dcd_trainer<kernel_type> linear_trainer; svm_c_linear_dcd_trainer<kernel_type>::optimizer_state state; const double C = 10; linear_trainer.set_epsilon(1e-10); linear_trainer.set_c(C); linear_trainer.force_last_weight_to_1(true); linear_trainer.include_bias(false); std::vector<sample_type> samples; std::vector<double> labels; // make an instance of a sample vector so we can use it below sample_type sample; sample.push_back(make_pair(0,-1)); sample.push_back(make_pair(1,1)); sample.push_back(make_pair(9,4)); for (int i = 0; i < 4; ++i) { samples.push_back(sample); labels.push_back(label); decision_function<kernel_type> df; df = linear_trainer.train(samples, labels); if (label > 0) { DLIB_TEST(std::abs(df(samples[0])-4) < 1e-8); } else { DLIB_TEST(std::abs(df(samples[0])+1) < 1e-8); } } } // ---------------------------------------------------------------------------------------- void test_l2_version () { typedef std::map<unsigned long,double> sample_type; typedef sparse_linear_kernel<sample_type> kernel_type; svm_c_linear_dcd_trainer<kernel_type> linear_trainer; linear_trainer.set_c(10); linear_trainer.set_epsilon(1e-5); std::vector<sample_type> samples; std::vector<double> labels; // make an instance of a sample vector so we can use it below sample_type sample; // Now let's go into a loop and randomly generate 10000 samples. double label = +1; for (int i = 0; i < 1000; ++i) { // flip this flag label *= -1; sample.clear(); // now make a random sparse sample with at most 10 non-zero elements for (int j = 0; j < 10; ++j) { int idx = std::rand()%100; double value = static_cast<double>(std::rand())/RAND_MAX; sample[idx] = label*value; } // Also save the samples we are generating so we can let the svm_c_linear_trainer // learn from them below. samples.push_back(sample); labels.push_back(label); } decision_function<kernel_type> df = linear_trainer.train(samples, labels); sample.clear(); sample[4] = 0.3; sample[10] = 0.9; DLIB_TEST(df(sample) > 0); sample.clear(); sample[83] = -0.3; sample[26] = -0.9; sample[58] = -0.7; DLIB_TEST(df(sample) < 0); sample.clear(); sample[0] = -0.2; sample[9] = -0.8; DLIB_TEST(df(sample) < 0); } class tester_svm_c_linear_dcd : public tester { public: tester_svm_c_linear_dcd ( ) : tester ("test_svm_c_linear_dcd", "Runs tests on the svm_c_linear_dcd_trainer.") {} void perform_test ( ) { test_normal(); print_spinner(); test_normal_no_bias(); print_spinner(); test_sparse(); print_spinner(); test_normal_force_last_weight(false,false); print_spinner(); test_normal_force_last_weight(false,true); print_spinner(); test_normal_force_last_weight(true,false); print_spinner(); test_normal_force_last_weight(true,true); print_spinner(); test_normal_1_sample(+1); print_spinner(); test_normal_1_sample(-1); print_spinner(); test_sparse_1_sample(+1); print_spinner(); test_sparse_1_sample(-1); print_spinner(); test_l2_version(); } } a; }