/* * File:        pr_loqo.c * Purpose:     solves quadratic programming problem for pattern recognition *              for support vectors * * Author:      Alex J. Smola * Created:     10/14/97 * Updated:     11/08/97 * Updated:     13/08/98 (removed exit(1) as it crashes svm lite when the margin *                        in a not sufficiently conservative manner) * *  * Copyright (c) 1997  GMD Berlin - All rights reserved * THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE of GMD Berlin * The copyright notice above does not evidence any * actual or intended publication of this work. * * Unauthorized commercial use of this software is not allowed */#include <math.h>#include <time.h>#include <stdlib.h>#include <stdio.h>#include "pr_loqo.h"#define	max(A, B)	((A) > (B) ? (A) : (B))#define	min(A, B)	((A) < (B) ? (A) : (B))#define sqr(A)          ((A) * (A))#define	ABS(A)  	((A) > 0 ? (A) : (-(A)))#define PREDICTOR 1#define CORRECTOR 2/*****************************************************************  replace this by any other function that will exit gracefully  in a larger system  ***************************************************************/void nrerror(char error_text[]){  printf("ERROR: terminating optimizer - %s\n", error_text);  /* exit(1); */}/*****************************************************************   taken from numerical recipes and modified to accept pointers   moreover numerical recipes code seems to be buggy (at least the   ones on the web)   cholesky solver and backsubstitution   leaves upper right triangle intact (rows first order)   ***************************************************************/int choldc(double a[], int n, double p[]){  void nrerror(char error_text[]);  int i, j, k;  double sum;  for (i = 0; i < n; i++){    for (j = i; j < n; j++) {      sum=a[n*i + j];      for (k=i-1; k>=0; k--) sum -= a[n*i + k]*a[n*j + k];      if (i == j) {	if (sum <= 0.0) {	  /*	  nrerror("choldc failed, matrix not positive definite"); */	  printf("choldc failed, matrix not positive definite"); fflush(stdout);	  sum = 0.0;	  return -1;	}	p[i]=sqrt(sum);      } else a[n*j + i] = sum/p[i];    }  }  return 0;}/* void choldc(double a[], int n, double p[]) *//* { *//*   void nrerror(char error_text[]); *//*   int i, j, k; *//*   double sum; *//*   double shift_amount; *//*   double rs; *//*   int shifted; *//*   shift_amount=0.0; *//*   shifted = 1; *//*   while(shifted){ *//*     shifted = 0; *//*     for (i = 0; i < n; i++){ *//*       if(shifted) break; *//*       for (j = i; j < n; j++) { *//* 	sum=a[n*i + j]; *//* 	for (k=i-1; k>=0; k--) sum -= a[n*i + k]*a[n*j + k]; *//* 	if (i == j) { *//* 	  if (sum <= 0.0) { *//* 	    rs=0.0; *//* 	    for(k=i;k<n;++k) rs += fabs(a[n*i + k]); *//* 	    for(k=0;k<i;++k) rs += fabs(a[n*k + i]); *//* 	    shift_amount = max(rs,1.1*shift_amount); *//* 	    printf("using shift_amount = %g\n", shift_amount); fflush(stdout); *//* 	    for(k=0;k<n;++k) *//* 	      a[n*k + k] = a[n*k + k]+shift_amount; *//* 	    shifted = 1; *//* 	    break; *//* 	  } *//* 	  p[i]=sqrt(sum); *//* 	} else a[n*j + i] = sum/p[i]; *//*       } *//*     } *//*   } *//* } */void cholsb(double a[], int n, double p[], double b[], double x[]){  int i, k;  double sum;  for (i=0; i<n; i++) {    sum=b[i];    for (k=i-1; k>=0; k--) sum -= a[n*i + k]*x[k];    x[i]=sum/p[i];  }  for (i=n-1; i>=0; i--) {    sum=x[i];    for (k=i+1; k<n; k++) sum -= a[n*k + i]*x[k];    x[i]=sum/p[i];  }}/*****************************************************************  sometimes we only need the forward or backward pass of the  backsubstitution, hence we provide these two routines separately   ***************************************************************/void chol_forward(double a[], int n, double p[], double b[], double x[]){  int i, k;  double sum;  for (i=0; i<n; i++) {    sum=b[i];    for (k=i-1; k>=0; k--) sum -= a[n*i + k]*x[k];    x[i]=sum/p[i];  }}void chol_backward(double a[], int n, double p[], double b[], double x[]){  int i, k;  double sum;  for (i=n-1; i>=0; i--) {    sum=b[i];    for (k=i+1; k<n; k++) sum -= a[n*k + i]*x[k];    x[i]=sum/p[i];  }}/*****************************************************************  solves the system | -H_x A' | |x_x| = |c_x|                    |  A   H_y| |x_y|   |c_y|  with H_x (and H_y) positive (semidefinite) matrices  and n, m the respective sizes of H_x and H_y  for variables see pg. 48 of notebook or do the calculations on a  sheet of paper again  predictor solves the whole thing, corrector assues that H_x didn't  change and relies on the results of the predictor. therefore do  _not_ modify workspace  if you want to speed tune anything in the code here's the right  place to do so: about 95% of the time is being spent in  here. something like an iterative refinement would be nice,  especially when switching from double to single precision. if you  have a fast parallel cholesky use it instead of the numrec  implementations.  side effects: changes H_y (but this is just the unit matrix or zero anyway  in our case)  ***************************************************************/int solve_reduced(int n, int m, double h_x[], double h_y[], 		  double a[], double x_x[], double x_y[],		  double c_x[], double c_y[],		  double workspace[], int step){  int i,j,k;  int ierr = 0;  double *p_x;  double *p_y;  double *t_a;  double *t_c;  double *t_y;  p_x = workspace;		/* together n + m + n*m + n + m = n*(m+2)+2*m */  p_y = p_x + n;  t_a = p_y + m;  t_c = t_a + n*m;  t_y = t_c + n;  if (step == PREDICTOR) {    ierr = choldc(h_x, n, p_x);	/* do cholesky decomposition */    if(ierr)      return ierr;    for (i=0; i<m; i++)         /* forward pass for A' */      chol_forward(h_x, n, p_x, a+i*n, t_a+i*n);				    for (i=0; i<m; i++)         /* compute (h_y + a h_x^-1A') */      for (j=i; j<m; j++)	for (k=0; k<n; k++) 	  h_y[m*i + j] += t_a[n*j + k] * t_a[n*i + k];				    ierr = choldc(h_y, m, p_y);	/* and cholesky decomposition */    if(ierr)      return ierr;  }    chol_forward(h_x, n, p_x, c_x, t_c);				/* forward pass for c */  for (i=0; i<m; i++) {		/* and solve for x_y */    t_y[i] = c_y[i];    for (j=0; j<n; j++)      t_y[i] += t_a[i*n + j] * t_c[j];  }  cholsb(h_y, m, p_y, t_y, x_y);  for (i=0; i<n; i++) {		/* finally solve for x_x */    t_c[i] = -t_c[i];    for (j=0; j<m; j++)      t_c[i] += t_a[j*n + i] * x_y[j];  }  chol_backward(h_x, n, p_x, t_c, x_x);  return ierr;}/*****************************************************************  matrix vector multiplication (symmetric matrix but only one triangle  given). computes m*x = y  no need to tune it as it's only of O(n^2) but cholesky is of  O(n^3). so don't waste your time _here_ although it isn't very  elegant.   ***************************************************************/void matrix_vector(int n, double m[], double x[], double y[]){  int i, j;  for (i=0; i<n; i++) {    y[i] = m[(n+1) * i] * x[i];    for (j=0; j<i; j++)      y[i] += m[i + n*j] * x[j];    for (j=i+1; j<n; j++)       y[i] += m[n*i + j] * x[j];   }}/*****************************************************************  call only this routine; this is the only one you're interested in  for doing quadratical optimization  the restart feature exists but it may not be of much use due to the  fact that an initial setting, although close but not very close the  the actual solution will result in very good starting diagnostics  (primal and dual feasibility and small infeasibility gap) but incur  later stalling of the optimizer afterwards as we have to enforce  positivity of the slacks.  ***************************************************************/int pr_loqo(int n, int m, double c[], double h_x[], double a[], double b[],	    double l[], double u[], double primal[], double dual[], 	    double dist[], int verb, double sigfig_max, int counter_max, 	    double margin, double bound, int restart) {  /* the knobs to be tuned ... */  /* double margin = -0.95;	   we will go up to 95% of the				   distance between old variables and zero */  /* double bound = 10;		   preset value for the start. small				   values give good initial				   feasibility but may result in slow				   convergence afterwards: we're too				   close to zero */  int ierr = 0;  /* to be allocated */  double *workspace;  double *diag_h_x;  double *h_y;  double *c_x;  double *c_y;  double *h_dot_x;  double *rho;  double *nu;  double *tau;  double *sigma;  double *gamma_z;  double *gamma_s;    double *hat_nu;  double *hat_tau;  double *delta_x;  double *delta_y;  double *delta_s;  double *delta_z;  double *delta_g;  double *delta_t;  double *d;  /* from the header - pointers into primal and dual */  double *x;  double *y;  double *g;  double *z;  double *s;  double *t;    /* auxiliary variables */  double b_plus_1;  double c_plus_1;  double x_h_x;  double primal_inf;  double dual_inf;  double sigfig;  double primal_obj, dual_obj;/*   double primal_obj = *p_obj; *//*   double dual_obj = *d_obj; */  double mu;  double alfa;  int counter = 0;  int status = STILL_RUNNING;