core/vnl/algo/vnl_levenberg_marquardt.cxx
Go to the documentation of this file.
00001 // This is core/vnl/algo/vnl_levenberg_marquardt.cxx
00002 #ifdef VCL_NEEDS_PRAGMA_INTERFACE
00003 #pragma implementation
00004 #endif
00005 //:
00006 // \file
00007 // \author Andrew W. Fitzgibbon, Oxford RRG
00008 // \date 31 Aug 96
00009 //
00010 //-----------------------------------------------------------------------------
00011 
00012 #include "vnl_levenberg_marquardt.h"
00013 
00014 #include <vcl_cassert.h>
00015 #include <vcl_iostream.h>
00016 
00017 #include <vnl/vnl_vector.h>
00018 #include <vnl/vnl_matrix.h>
00019 #include <vnl/vnl_fastops.h>
00020 #include <vnl/vnl_vector_ref.h>
00021 #include <vnl/vnl_matrix_ref.h>
00022 #include <vnl/vnl_least_squares_function.h>
00023 #include <vnl/algo/vnl_netlib.h> // lmdif_()
00024 
00025 // see header
00026 vnl_vector<double> vnl_levenberg_marquardt_minimize(vnl_least_squares_function& f,
00027                                                     vnl_vector<double> const& initial_estimate)
00028 {
00029   vnl_vector<double> x = initial_estimate;
00030   vnl_levenberg_marquardt lm(f);
00031   lm.minimize(x);
00032   return x;
00033 }
00034 
00035 // ctor
00036 void vnl_levenberg_marquardt::init(vnl_least_squares_function* f)
00037 {
00038   f_ = f;
00039 
00040   // If changing these defaults, check the help comments in vnl_levenberg_marquardt.h,
00041   // and MAKE SURE they're consistent.
00042   xtol = 1e-8;           // Termination tolerance on X (solution vector)
00043   maxfev = 400 * f->get_number_of_unknowns(); // Termination maximum number of iterations.
00044   ftol = xtol * 0.01;    // Termination tolerance on F (sum of squared residuals)
00045   gtol = 1e-5;           // Termination tolerance on Grad(F)' * F = 0
00046   epsfcn = xtol * 0.001; // Step length for FD Jacobian
00047 
00048   unsigned int m = f_->get_number_of_residuals(); // I  Number of residuals, must be > #unknowns
00049   unsigned int n = f_->get_number_of_unknowns();  // I  Number of unknowns
00050 
00051   set_covariance_ = false;
00052   fdjac_.set_size(n,m);
00053   fdjac_.fill(0.0);
00054   ipvt_.set_size(n);
00055   ipvt_.fill(0);
00056   inv_covar_.set_size(n,n);
00057   inv_covar_.fill(0.0);
00058   //fdjac_ = new vnl_matrix<double>(n,m);
00059   //ipvt_ = new vnl_vector<int>(n);
00060   //covariance_ = new vnl_matrix<double>(n,n);
00061 }
00062 
00063 vnl_levenberg_marquardt::~vnl_levenberg_marquardt()
00064 {
00065   //delete covariance_;
00066   //delete fdjac_;
00067   //delete ipvt_;
00068 }
00069 
00070 //--------------------------------------------------------------------------------
00071 
00072 #ifdef VCL_SUNPRO_CC
00073 extern "C"
00074 #endif
00075 void vnl_levenberg_marquardt::lmdif_lsqfun(long* n,     // I   Number of residuals
00076                                            long* p,     // I   Number of unknowns
00077                                            double* x,  // I   Solution vector, size n
00078                                            double* fx, // O   Residual vector f(x)
00079                                            long* iflag, // IO  0 ==> print, -1 ==> terminate
00080                                            void* userdata)
00081 {
00082   vnl_levenberg_marquardt* self =
00083     static_cast<vnl_levenberg_marquardt*>(userdata);
00084   vnl_least_squares_function* f = self->f_;
00085   assert(*p == (int)f->get_number_of_unknowns());
00086   assert(*n == (int)f->get_number_of_residuals());
00087   vnl_vector_ref<double> ref_x(*p, const_cast<double*>(x));
00088   vnl_vector_ref<double> ref_fx(*n, fx);
00089 
00090   if (*iflag == 0) {
00091     if (self->trace)
00092       vcl_cerr << "lmdif: iter " << self->num_iterations_ << " err ["
00093                << x[0] << ", " << x[1] << ", " << x[2] << ", " << x[3] << ", "
00094                << x[4] << ", ... ] = " << ref_fx.magnitude() << '\n';
00095 
00096     f->trace(self->num_iterations_, ref_x, ref_fx);
00097     ++(self->num_iterations_);
00098   } else {
00099     f->f(ref_x, ref_fx);
00100   }
00101 
00102   if (self->start_error_ == 0)
00103     self->start_error_ = ref_fx.rms();
00104 
00105   if (f->failure) {
00106     f->clear_failure();
00107     *iflag = -1; // fsm
00108   }
00109 }
00110 
00111 
00112 // This function shouldn't be inlined, because (1) modification of the
00113 // body will not cause the timestamp on the header to change, and so
00114 // others will not be forced to recompile, and (2) the cost of making
00115 // one more function call is far, far less than the cost of actually
00116 // performing the minimisation, so the inline doesn't gain you
00117 // anything.
00118 //
00119 bool vnl_levenberg_marquardt::minimize(vnl_vector<double>& x)
00120 {
00121   if ( f_->has_gradient() )
00122     return minimize_using_gradient(x);
00123   else
00124     return minimize_without_gradient(x);
00125 }
00126 
00127 
00128 //
00129 bool vnl_levenberg_marquardt::minimize_without_gradient(vnl_vector<double>& x)
00130 {
00131   //fsm
00132   if (f_->has_gradient()) {
00133     vcl_cerr << __FILE__ " : WARNING. calling minimize_without_gradient(), but f_ has gradient.\n";
00134   }
00135 
00136   // e04fcf
00137   long m = f_->get_number_of_residuals(); // I  Number of residuals, must be > #unknowns
00138   long n = f_->get_number_of_unknowns();  // I  Number of unknowns
00139 
00140   if (m < n) {
00141     vcl_cerr << "vnl_levenberg_marquardt: Number of unknowns("<<n<<") greater than number of data ("<<m<<")\n";
00142     failure_code_ = ERROR_DODGY_INPUT;
00143     return false;
00144   }
00145 
00146   if (int(x.size()) != n) {
00147     vcl_cerr << "vnl_levenberg_marquardt: Input vector length ("<<x.size()<<") not equal to num unknowns ("<<n<<")\n";
00148     failure_code_ = ERROR_DODGY_INPUT;
00149     return false;
00150   }
00151 
00152   vnl_vector<double> fx(m, 0.0);    // W m   Storage for target vector
00153   vnl_vector<double> diag(n, 0);  // I     Multiplicative scale factors for variables
00154   long user_provided_scale_factors = 1;  // 1 is no, 2 is yes
00155   double factor = 100;
00156   long nprint = 1;
00157 
00158   vnl_vector<double> qtf(n, 0);
00159   vnl_vector<double> wa1(n, 0);
00160   vnl_vector<double> wa2(n, 0);
00161   vnl_vector<double> wa3(n, 0);
00162   vnl_vector<double> wa4(m, 0);
00163 
00164 #ifdef DEBUG
00165   vcl_cerr << "STATUS: " << failure_code_ << '\n';
00166 #endif
00167 
00168   num_iterations_ = 0;
00169   set_covariance_ = false;
00170   long info;
00171   start_error_ = 0; // Set to 0 so first call to lmdif_lsqfun will know to set it.
00172   v3p_netlib_lmdif_(
00173          lmdif_lsqfun, &m, &n,
00174          x.data_block(),
00175          fx.data_block(),
00176          &ftol, &xtol, &gtol, &maxfev, &epsfcn,
00177          &diag[0],
00178          &user_provided_scale_factors, &factor, &nprint,
00179          &info, &num_evaluations_,
00180          fdjac_.data_block(), &m, ipvt_.data_block(),
00181          &qtf[0],
00182          &wa1[0], &wa2[0], &wa3[0], &wa4[0],
00183          this);
00184   failure_code_ = (ReturnCodes) info;
00185 
00186   // One more call to compute final error.
00187   lmdif_lsqfun(&m,              // I    Number of residuals
00188                &n,              // I    Number of unknowns
00189                x.data_block(),  // I    Solution vector, size n
00190                fx.data_block(), // O    Residual vector f(x)
00191                &info, this);
00192   end_error_ = fx.rms();
00193 
00194 #ifdef _SGI_CC_6_
00195   // Something fundamentally odd about the switch below on SGI native... FIXME
00196   vcl_cerr << "vnl_levenberg_marquardt: termination code = " << failure_code_ << vcl_endl;
00197   // diagnose_outcome(vcl_cerr);
00198   return 1;
00199 #endif
00200 
00201   // Translate status code
00202   switch ((int)failure_code_) {
00203   case 1: // ftol
00204   case 2: // xtol
00205   case 3: // both
00206   case 4: // gtol
00207     return true;
00208   default:
00209     return false;
00210   }
00211 }
00212 
00213 //--------------------------------------------------------------------------------
00214 
00215 #ifdef VCL_SUNPRO_CC
00216 extern "C"
00217 #endif
00218 void vnl_levenberg_marquardt::lmder_lsqfun(long* n,     // I   Number of residuals
00219                                            long* p,     // I   Number of unknowns
00220                                            double* x,  // I   Solution vector, size n
00221                                            double* fx, // O   Residual vector f(x)
00222                                            double* fJ, // O   m * n Jacobian f(x)
00223                                            long*,
00224                                            long* iflag, // I   1 -> calc fx, 2 -> calc fjac
00225                                            void* userdata)
00226 {
00227   vnl_levenberg_marquardt* self =
00228     static_cast<vnl_levenberg_marquardt*>(userdata);
00229   vnl_least_squares_function* f = self->f_;
00230   assert(*p == (int)f->get_number_of_unknowns());
00231   assert(*n == (int)f->get_number_of_residuals());
00232   vnl_vector_ref<double> ref_x(*p, (double*)x); // const violation!
00233   vnl_vector_ref<double> ref_fx(*n, fx);
00234   vnl_matrix_ref<double> ref_fJ(*n, *p, fJ);
00235 
00236   if (*iflag == 0) {
00237     if (self->trace)
00238       vcl_cerr << "lmder: iter " << self->num_iterations_ << " err ["
00239                << x[0] << ", " << x[1] << ", " << x[2] << ", " << x[3] << ", "
00240                << x[4] << ", ... ] = " << ref_fx.magnitude() << '\n';
00241     f->trace(self->num_iterations_, ref_x, ref_fx);
00242   }
00243   else if (*iflag == 1) {
00244     f->f(ref_x, ref_fx);
00245     if (self->start_error_ == 0)
00246       self->start_error_ = ref_fx.rms();
00247     ++(self->num_iterations_);
00248   }
00249   else if (*iflag == 2) {
00250     f->gradf(ref_x, ref_fJ);
00251     ref_fJ.inplace_transpose();
00252 
00253     // check derivative?
00254     if ( self->check_derivatives_ > 0 )
00255     {
00256       self->check_derivatives_--;
00257 
00258       // use finite difference to compute Jacobian
00259       vnl_vector<double> feval( *n );
00260       vnl_matrix<double> finite_jac( *p, *n, 0.0 );
00261       vnl_vector<double> wa1( *n );
00262       long info=1;
00263       double diff;
00264       f->f( ref_x, feval );
00265       v3p_netlib_fdjac2_(
00266               lmdif_lsqfun, n, p, x,
00267               feval.data_block(),
00268               finite_jac.data_block(),
00269               n,
00270               &info,
00271               &(self->epsfcn),
00272               wa1.data_block(),
00273               self);
00274       // compute difference
00275       for ( unsigned i=0; i<ref_fJ.cols(); ++i )
00276         for ( unsigned j=0; j<ref_fJ.rows(); ++j ) {
00277           diff = ref_fJ(j,i) - finite_jac(j,i);
00278           diff = diff*diff;
00279           if ( diff > self->epsfcn ) {
00280             vcl_cout << "Jac(" << i << ", " << j << ") diff: " << ref_fJ(j,i) 
00281                      << "  " << finite_jac(j,i)
00282                      << "  " << ref_fJ(j,i)-finite_jac(j,i) << '\n';
00283           }
00284         }
00285     }
00286   }
00287 
00288   if (f->failure) {
00289     f->clear_failure();
00290     *iflag = -1; // fsm
00291   }
00292 }
00293 
00294 
00295 //
00296 bool vnl_levenberg_marquardt::minimize_using_gradient(vnl_vector<double>& x)
00297 {
00298   //fsm
00299   if (! f_->has_gradient()) {
00300     vcl_cerr << __FILE__ ": called method minimize_using_gradient(), but f_ has no gradient.\n";
00301     return false;
00302   }
00303 
00304   long m = f_->get_number_of_residuals(); // I  Number of residuals, must be > #unknowns
00305   long n = f_->get_number_of_unknowns();  // I  Number of unknowns
00306 
00307   if (m < n) {
00308     vcl_cerr << __FILE__ ": Number of unknowns("<<n<<") greater than number of data ("<<m<<")\n";
00309     failure_code_ = ERROR_DODGY_INPUT;
00310     return false;
00311   }
00312 
00313   vnl_vector<double> fx(m, 0.0);    // W m   Explicitly set target to 0.0
00314 
00315   num_iterations_ = 0;
00316   set_covariance_ = false;
00317   long info;
00318   start_error_ = 0; // Set to 0 so first call to lmder_lsqfun will know to set it.
00319 
00320 
00321   double factor = 100;
00322   long nprint = 1;
00323   long mode=1, nfev, njev;
00324 
00325   vnl_vector<double> diag(n, 0);
00326   vnl_vector<double> qtf(n, 0);
00327   vnl_vector<double> wa1(n, 0);
00328   vnl_vector<double> wa2(n, 0);
00329   vnl_vector<double> wa3(n, 0);
00330   vnl_vector<double> wa4(m, 0);
00331 
00332 
00333   v3p_netlib_lmder_(
00334           lmder_lsqfun, &m, &n,
00335           x.data_block(),
00336           fx.data_block(),
00337           fdjac_.data_block(), &m,
00338           &ftol,
00339           &xtol,
00340           &gtol,
00341           &maxfev,
00342           diag.data_block(),
00343           &mode,
00344           &factor,
00345           &nprint,
00346           &info,
00347           &nfev, &njev,
00348           ipvt_.data_block(),
00349           qtf.data_block(),
00350           wa1.data_block(),
00351           wa2.data_block(),
00352           wa3.data_block(),
00353           wa4.data_block(),
00354           this);
00355 
00356 
00357 
00358 
00359   num_evaluations_ = num_iterations_; // for lmder, these are the same.
00360   if (info<0)
00361     info = ERROR_FAILURE;
00362   failure_code_ = (ReturnCodes) info;
00363   end_error_ = fx.rms();
00364 
00365   // Translate status code
00366   switch (failure_code_) {
00367   case 1: // ftol
00368   case 2: // xtol
00369   case 3: // both
00370   case 4: // gtol
00371     return true;
00372   default:
00373     return false;
00374   }
00375 }
00376 
00377 //--------------------------------------------------------------------------------
00378 
00379 void vnl_levenberg_marquardt::diagnose_outcome() const
00380 {
00381   diagnose_outcome(vcl_cerr);
00382 }
00383 
00384 // fsm: should this function be a method on vnl_nonlinear_minimizer?
00385 // if not, the return codes should be moved into LM.
00386 void vnl_levenberg_marquardt::diagnose_outcome(vcl_ostream& s) const
00387 {
00388 #define whoami "vnl_levenberg_marquardt"
00389   //if (!verbose_) return;
00390   switch (failure_code_) {
00391     //  case -1:
00392     // have already warned.
00393     //    return;
00394   case ERROR_FAILURE:
00395     s << (whoami ": OIOIOI -- failure in leastsquares function\n");
00396     break;
00397   case ERROR_DODGY_INPUT:
00398     s << (whoami ": OIOIOI -- lmdif dodgy input\n");
00399     break;
00400   case CONVERGED_FTOL: // ftol
00401     s << (whoami ": converged to ftol\n");
00402     break;
00403   case CONVERGED_XTOL: // xtol
00404     s << (whoami ": converged to xtol\n");
00405     break;
00406   case CONVERGED_XFTOL: // both
00407     s << (whoami ": converged nicely\n");
00408     break;
00409   case CONVERGED_GTOL:
00410     s << (whoami ": converged via gtol\n");
00411     break;
00412   case TOO_MANY_ITERATIONS:
00413     s << (whoami ": too many iterations\n");
00414     break;
00415   case FAILED_FTOL_TOO_SMALL:
00416     s << (whoami ": ftol is too small. no further reduction in the sum of squares is possible.\n");
00417     break;
00418   case FAILED_XTOL_TOO_SMALL:
00419     s << (whoami ": xtol is too small. no further improvement in the approximate solution x is possible.\n");
00420     break;
00421   case FAILED_GTOL_TOO_SMALL:
00422     s << (whoami ": gtol is too small. Fx is orthogonal to the columns of the jacobian to machine precision.\n");
00423     break;
00424   default:
00425     s << (whoami ": OIOIOI: unkown info code from lmder.\n");
00426     break;
00427   }
00428   unsigned int m = f_->get_number_of_residuals();
00429   s << whoami ": " << num_iterations_ << " iterations, "
00430     << num_evaluations_ << " evaluations, "<< m <<" residuals.  RMS error start/end "
00431     << get_start_error() << '/' << get_end_error() << vcl_endl;
00432 #undef whoami
00433 }
00434 
00435 // fjac is an output m by n array. the upper n by n submatrix
00436 //         of fjac contains an upper triangular matrix r with
00437 //         diagonal elements of nonincreasing magnitude such that
00438 //
00439 //                t     t           t
00440 //               p *(jac *jac)*p = r *r,
00441 //
00442 //         where p is a permutation matrix and jac is the final
00443 //         calculated jacobian. column j of p is column ipvt(j)
00444 //         (see below) of the identity matrix. the lower trapezoidal
00445 //         part of fjac contains information generated during
00446 //         the computation of r.
00447 
00448 // fdjac is target m*n
00449 
00450 //: Get INVERSE of covariance at last minimum.
00451 // Code thanks to Joss Knight (joss@robots.ox.ac.uk)
00452 vnl_matrix<double> const& vnl_levenberg_marquardt::get_JtJ()
00453 {
00454   if (!set_covariance_)
00455   {
00456     vcl_cerr << __FILE__ ": get_covariance() not confirmed tested  yet\n";
00457     unsigned int n = fdjac_.rows();
00458 
00459     // matrix in FORTRAN is column-wise.
00460     // transpose it to get C style order
00461     vnl_matrix<double> r = fdjac_.extract(n,n).transpose();
00462     // r is upper triangular matrix according to documentation.
00463     // But the lower part has non-zeros somehow.
00464     // clear the lower part
00465     for (unsigned int i=0; i<n; ++i)
00466       for (unsigned int j=0; j<i; ++j)
00467         r(i,j) = 0.0;
00468 
00469     // compute r^T * r
00470     vnl_matrix<double> rtr;
00471     vnl_fastops::AtA(rtr, r);
00472     vnl_matrix<double> rtrpt (n, n);
00473 
00474     // Permute. First order columns.
00475     // Note, *ipvt_ contains 1 to n, not 0 to n-1
00476     vnl_vector<int> jpvt(n);
00477     for (unsigned int j = 0; j < n; ++j) {
00478       unsigned int i = 0;
00479       for (; i < n; i++) {
00480         if (ipvt_[i] == (int)j+1) {
00481           jpvt (j) = i;
00482           break;
00483         }
00484       }
00485       rtrpt.set_column(j, rtr.get_column(i));
00486     }
00487 
00488     // Now order rows
00489     for (unsigned int j = 0; j < n; ++j) {
00490       inv_covar_.set_row (j, rtrpt.get_row (jpvt(j)));
00491     }
00492 
00493     set_covariance_ = true;
00494   }
00495   return inv_covar_;
00496 }