aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile2
-rw-r--r--include/msvmmaj.h7
-rw-r--r--include/msvmmaj_kernel.h6
-rw-r--r--include/msvmmaj_pred.h8
-rw-r--r--include/msvmmaj_train_dataset.h5
-rw-r--r--src/crossval.c15
-rw-r--r--src/msvmmaj_init.c2
-rw-r--r--src/msvmmaj_io.c9
-rw-r--r--src/msvmmaj_kernel.c194
-rw-r--r--src/msvmmaj_pred.c124
-rw-r--r--src/msvmmaj_train.c10
-rw-r--r--src/msvmmaj_train_dataset.c88
-rw-r--r--src/predMSVMMaj.c6
-rw-r--r--src/trainMSVMMaj.c3
-rw-r--r--src/trainMSVMMajdataset.c42
-rw-r--r--training/wine.training4
16 files changed, 425 insertions, 100 deletions
diff --git a/Makefile b/Makefile
index c432617..3e95865 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ CC=gcc
CFLAGS=-Wall -O3 -DVERSION=$(VERSION) -g
INCLUDE= -Iinclude/
LIB= -Llib/
-EXECS=trainMSVMMaj trainMSVMMajdataset predMSVMMaj
+EXECS=trainMSVMMaj trainMSVMMajdataset
.PHONY: all clean tar
diff --git a/include/msvmmaj.h b/include/msvmmaj.h
index 1dba211..3d04f30 100644
--- a/include/msvmmaj.h
+++ b/include/msvmmaj.h
@@ -36,8 +36,6 @@
* @param *Q pointer to the error matrix
* @param *H pointer to the Huber weighted error matrix
* @param *R pointer to the 0-1 auxiliary matrix
- * @param *J pointer to the diagonal matrix in the
- * regularization term
* @param *rho pointer to the instance weight vector
* @param training_error error after training has completed
* @param *data_file pointer to the filename of the data
@@ -65,7 +63,6 @@ struct MajModel {
double *Q;
double *H;
double *R;
- double *J;
double *rho;
double training_error;
char *data_file;
@@ -81,6 +78,8 @@ struct MajModel {
* @param m number of predictors
* @param *y pointer to vector of class labels
* @param *Z pointer to augmented data matrix
+ * @param *RAW pointer to augmented raw data matrix
+ * @param *J pointer to regularization vector
* @param kerneltype kerneltype used in MajData::Z
* @param *kernelparam kernel parameters used in MajData::Z
*
@@ -91,6 +90,8 @@ struct MajData {
long m;
long *y;
double *Z;
+ double *RAW;
+ double *J;
KernelType kerneltype;
double *kernelparam;
};
diff --git a/include/msvmmaj_kernel.h b/include/msvmmaj_kernel.h
index 2965c1c..d4f169a 100644
--- a/include/msvmmaj_kernel.h
+++ b/include/msvmmaj_kernel.h
@@ -23,7 +23,11 @@ struct MajModel;
// function declarations
void msvmmaj_make_kernel(struct MajModel *model, struct MajData *data);
-long msvmmaj_make_eigen(double *K, long n, double *P, double *Lambda);
+long msvmmaj_make_eigen(double *K, long n, double **P, double **Lambda);
+
+void msvmmaj_make_crosskernel(struct MajModel *model,
+ struct MajData *data_train, struct MajData *data_test,
+ double **K2);
double msvmmaj_compute_rbf(double *x1, double *x2, double *kernelparam,
long n);
diff --git a/include/msvmmaj_pred.h b/include/msvmmaj_pred.h
index ce22b10..c274cfa 100644
--- a/include/msvmmaj_pred.h
+++ b/include/msvmmaj_pred.h
@@ -19,7 +19,13 @@ struct MajData;
struct MajModel;
// function declarations
-void msvmmaj_predict_labels(struct MajData *data, struct MajModel *model,
+void msvmmaj_predict_labels(struct MajData *data_test,
+ struct MajData *data_train, struct MajModel *model,
+ long *predy);
+void msvmmaj_predict_labels_linear(struct MajData *data,
+ struct MajModel *model, long *predy);
+void msvmmaj_predict_labels_kernel(struct MajData *data_test,
+ struct MajData *data_train, struct MajModel *model,
long *predy);
double msvmmaj_prediction_perf(struct MajData *data, long *perdy);
diff --git a/include/msvmmaj_train_dataset.h b/include/msvmmaj_train_dataset.h
index 4ee39bf..2afcde8 100644
--- a/include/msvmmaj_train_dataset.h
+++ b/include/msvmmaj_train_dataset.h
@@ -29,7 +29,7 @@
* @param lambda parameter for the MajModel
* @param epsilon parameter for the MajModel
* @param kerneltype parameter for the MajModel
- * @param *kernel_param parameters for the MajModel
+ * @param *kernelparam parameters for the MajModel
* @param *train_data pointer to the training data
* @param *test_data pointer to the test data (if any)
* @param performance performance after cross validation
@@ -43,7 +43,7 @@ struct Task {
double kappa;
double lambda;
double epsilon;
- double *kernel_param;
+ double *kernelparam;
struct MajData *train_data;
struct MajData *test_data;
double performance;
@@ -137,4 +137,5 @@ void copy_model(struct MajModel *from, struct MajModel *to);
void msvmmaj_reallocate_model(struct MajModel *model, long n);
+void print_progress_string(struct Task *task, long N);
#endif
diff --git a/src/crossval.c b/src/crossval.c
index 10e3051..09bd377 100644
--- a/src/crossval.c
+++ b/src/crossval.c
@@ -110,8 +110,8 @@ void msvmmaj_get_tt_split(struct MajData *full_data, struct MajData *train_data,
train_data->y = Calloc(long, train_n);
test_data->y = Calloc(long, test_n);
- train_data->Z = Calloc(double, train_n*(m+1));
- test_data->Z = Calloc(double, test_n*(m+1));
+ train_data->RAW = Calloc(double, train_n*(m+1));
+ test_data->RAW = Calloc(double, test_n*(m+1));
k = 0;
l = 0;
@@ -119,17 +119,20 @@ void msvmmaj_get_tt_split(struct MajData *full_data, struct MajData *train_data,
if (cv_idx[i] == fold_idx) {
test_data->y[k] = full_data->y[i];
for (j=0; j<m+1; j++)
- matrix_set(test_data->Z, m+1, k, j,
- matrix_get(full_data->Z, m+1,
+ matrix_set(test_data->RAW, m+1, k, j,
+ matrix_get(full_data->RAW, m+1,
i, j));
k++;
} else {
train_data->y[l] = full_data->y[i];
for (j=0; j<m+1; j++)
- matrix_set(train_data->Z, m+1, l, j,
- matrix_get(full_data->Z, m+1,
+ matrix_set(train_data->RAW, m+1, l, j,
+ matrix_get(full_data->RAW, m+1,
i, j));
l++;
}
}
+
+ train_data->Z = train_data->RAW;
+ test_data->Z = test_data->RAW;
}
diff --git a/src/msvmmaj_init.c b/src/msvmmaj_init.c
index fcad0e2..dd1fd8f 100644
--- a/src/msvmmaj_init.c
+++ b/src/msvmmaj_init.c
@@ -38,6 +38,7 @@ struct MajModel *msvmmaj_init_model()
model->kappa = 0.0;
model->weight_idx = 1;
model->kerneltype = K_LINEAR;
+ model->kernelparam = NULL;
return model;
}
@@ -58,6 +59,7 @@ struct MajData *msvmmaj_init_data()
// set default values
data->kerneltype = K_LINEAR;
+ data->kernelparam = NULL;
return data;
}
diff --git a/src/msvmmaj_io.c b/src/msvmmaj_io.c
index fc7cc56..8a09b3d 100644
--- a/src/msvmmaj_io.c
+++ b/src/msvmmaj_io.c
@@ -56,12 +56,12 @@ void msvmmaj_read_data(struct MajData *dataset, char *data_file)
nr += fscanf(fid, "%ld", &m);
// Allocate memory
- dataset->Z = Malloc(double, n*(m+1));
+ dataset->RAW = Malloc(double, n*(m+1));
// Read first line of data
for (j=1; j<m+1; j++) {
nr += fscanf(fid, "%lf", &value);
- matrix_set(dataset->Z, n, 0, j, value);
+ matrix_set(dataset->RAW, n, 0, j, value);
}
// Check if there is a label at the end of the line
@@ -81,7 +81,7 @@ void msvmmaj_read_data(struct MajData *dataset, char *data_file)
for (i=1; i<n; i++) {
for (j=1; j<m+1; j++) {
nr += fscanf(fid, "%lf", &value);
- matrix_set(dataset->Z, m+1, i, j, value);
+ matrix_set(dataset->RAW, m+1, i, j, value);
}
if (dataset->y != NULL) {
nr += fscanf(fid, "%lf", &value);
@@ -112,11 +112,12 @@ void msvmmaj_read_data(struct MajData *dataset, char *data_file)
// Set the column of ones
for (i=0; i<n; i++)
- matrix_set(dataset->Z, m+1, i, 0, 1.0);
+ matrix_set(dataset->RAW, m+1, i, 0, 1.0);
dataset->n = n;
dataset->m = m;
dataset->K = K;
+ dataset->Z = dataset->RAW;
}
diff --git a/src/msvmmaj_kernel.c b/src/msvmmaj_kernel.c
index 5ac138c..a2c1193 100644
--- a/src/msvmmaj_kernel.c
+++ b/src/msvmmaj_kernel.c
@@ -32,14 +32,60 @@
void msvmmaj_make_kernel(struct MajModel *model, struct MajData *data)
{
long i, j;
+ // Determine if a kernel needs to be computed. This is not the case if
+ // a LINEAR kernel is requested in the model, or if the requested
+ // kernel is already in the data.
+
if (model->kerneltype == K_LINEAR) {
- model->J = Calloc(double, model->m+1);
- for (i=1; i<model->m+1; i++)
- matrix_set(model->J, 1, i, 0, 1.0);
+ data->J = Calloc(double, data->m+1);
+ for (i=1; i<data->m+1; i++) {
+ matrix_set(data->J, 1, i, 0, 1.0);
+ }
return;
}
- long n = model->n;
+ /*
+ switch (model->kerneltype) {
+ case K_LINEAR:
+ // if data has another kernel, free that matrix and
+ // assign Z to RAW
+ if (data->kerneltype != K_LINEAR) {
+ free(data->Z);
+ data->Z = data->RAW;
+ }
+ data->J = Calloc(double, data->m+1);
+ for (i=1; i<model->m+1; i++) {
+ matrix_set(data->J, 1, i, 0, 1.0);
+ }
+ return;
+ case K_POLY:
+ // if data has another kernel, we need to recalculate
+ if (data->kerneltype != K_POLY) {
+ break;
+ }
+ // if it is poly, we only recalculate if the kernel
+ // parameters differ
+ if (data->kernelparam[0] == model->kernelparam[0] &&
+ data->kernelparam[1] == model->kernelparam[1] &&
+ data->kernelparam[2] == model->kernelparam[2])
+ // < do something with J ?
+ return;
+ case K_RBF:
+ if (data->kerneltype != K_RBF)
+ break;
+ if (data->kernelparam[0] == model->kernelparam[0])
+ // < do something with J ?
+ return;
+ case K_SIGMOID:
+ if (data->kerneltype != K_SIGMOID)
+ break;
+ if (data->kernelparam[0] == model->kernelparam[0] &&
+ data->kernelparam[1] == model->kernelparam[1])
+ // < do something with J ?
+ return;
+ }
+ */
+ long n = data->n;
double value;
double *x1, *x2;
double *K = Calloc(double, n*n);
@@ -55,7 +101,7 @@ void msvmmaj_make_kernel(struct MajModel *model, struct MajData *data)
value = msvmmaj_compute_rbf(x1, x2,
model->kernelparam, data->m);
else if (model->kerneltype == K_SIGMOID)
- value = msvmmaj_compute_rbf(x1, x2,
+ value = msvmmaj_compute_sigmoid(x1, x2,
model->kernelparam, data->m);
else {
fprintf(stderr, "Unknown kernel type in "
@@ -67,25 +113,28 @@ void msvmmaj_make_kernel(struct MajModel *model, struct MajData *data)
}
}
- double *P = Malloc(double, n*n);
- double *Lambda = Malloc(double, n);
- long num_eigen = msvmmaj_make_eigen(K, n, P, Lambda);
+ double *P = NULL;
+ double *Sigma = NULL;
+ long num_eigen = msvmmaj_make_eigen(K, n, &P, &Sigma);
+ //printf("num eigen: %li\n", num_eigen);
+ data->m = num_eigen;
+ model->m = num_eigen;
// copy eigendecomp to data
- data->Z = realloc(data->Z, n*(n+1)*sizeof(double));
+ data->Z = Calloc(double, n*(num_eigen+1));
for (i=0; i<n; i++) {
- for (j=0; j<n; j++)
- matrix_set(data->Z, n+1, i, j+1,
- matrix_get(P, n, i, j));
- matrix_set(data->Z, n+1, i, 0, 1.0);
+ for (j=0; j<num_eigen; j++) {
+ value = matrix_get(P, num_eigen, i, j);
+ matrix_set(data->Z, num_eigen+1, i, j, value);
+ }
+ matrix_set(data->Z, num_eigen+1, i, 0, 1.0);
}
- data->m = n;
// Set the regularization matrix (change if not full rank used)
- model->J = Calloc(double, model->m+1);
- for (i=1; i<model->m+1; i++) {
- value = 1.0/matrix_get(Lambda, 1, i-1, 0);
- matrix_set(model->J, 1, i, 0, value);
+ data->J = Calloc(double, data->m+1);
+ for (i=1; i<data->m+1; i++) {
+ value = 1.0/matrix_get(Sigma, 1, i-1, 0);
+ matrix_set(data->J, 1, i, 0, value);
}
// let data know what it's made of
@@ -109,23 +158,25 @@ void msvmmaj_make_kernel(struct MajModel *model, struct MajData *data)
data->kernelparam[0] = model->kernelparam[0];
data->kernelparam[1] = model->kernelparam[1];
}
- model->m = n;
free(K);
+ free(Sigma);
+ free(P);
}
/**
- * @brief Find the eigendecomposition of a kernel matrix.
+ * @brief Find the (reduced) eigendecomposition of a kernel matrix.
*
* @details.
* tbd
*
- *
*/
-long msvmmaj_make_eigen(double *K, long n, double *P, double *Lambda)
+long msvmmaj_make_eigen(double *K, long n, double **P, double **Sigma)
{
int M, status, LWORK, *IWORK, *IFAIL;
- double abstol, *WORK;
+ long i, j, num_eigen, cutoff_idx;
+ double max_eigen, abstol, *WORK;
+ double *tempSigma = Malloc(double, n);
double *tempP = Malloc(double, n*n);
IWORK = Malloc(int, 5*n);
@@ -150,7 +201,7 @@ long msvmmaj_make_eigen(double *K, long n, double *P, double *Lambda)
0,
abstol,
&M,
- Lambda,
+ tempSigma,
tempP,
n,
WORK,
@@ -174,7 +225,7 @@ long msvmmaj_make_eigen(double *K, long n, double *P, double *Lambda)
0,
abstol,
&M,
- Lambda,
+ tempSigma,
tempP,
n,
WORK,
@@ -182,29 +233,94 @@ long msvmmaj_make_eigen(double *K, long n, double *P, double *Lambda)
IWORK,
IFAIL);
- printf("status = %i\n", status);
- printf("Number of eigenvalues found: %i\n", M);
-
if (status != 0) {
fprintf(stderr, "Nonzero exit status from dsyevx. Exiting...");
exit(1);
}
- // Here you can put code to select the necessary eigenvectors,
- // depending on the size of the eigenvalues.
- // For now, let's just print the eigenvalues and exit
+
+ // Select the desired number of eigenvalues, depending on their size.
+ // dsyevx sorts eigenvalues in ascending order.
+ //
+ max_eigen = tempSigma[n-1];
+ cutoff_idx = 0;
+
+ for (i=0; i<n; i++)
+ if (tempSigma[i]/max_eigen > 1e-10 ) {
+ cutoff_idx = i;
+ break;
+ }
+
+ num_eigen = n - cutoff_idx;
- print_matrix(Lambda, n, 1);
+ *Sigma = Calloc(double, num_eigen);
- // revert P to row-major order
- long i, j;
- for (i=0; i<n; i++)
- for (j=0; j<n; j++)
- P[i*n+j] = tempP[j*n+i];
+ for (i=0; i<num_eigen; i++) {
+ (*Sigma)[i] = tempSigma[n-1 - i];
+ }
+ // revert P to row-major order and copy only the the columns
+ // corresponding to the selected eigenvalues
+ //
+ *P = Calloc(double, n*num_eigen);
+ for (j=n-1; j>n-1-num_eigen; j--) {
+ for (i=0; i<n; i++) {
+ (*P)[i*num_eigen + (n-1)-j] = tempP[i + j*n];
+ }
+ }
+
+ free(tempSigma);
free(tempP);
- // replace by number of columns of P
- return n;
+ return num_eigen;
+}
+
+void msvmmaj_make_crosskernel(struct MajModel *model,
+ struct MajData *data_train, struct MajData *data_test,
+ double **K2)
+{
+ long i, j;
+ long n_train = data_train->n;
+ long n_test = data_test->n;
+ long m = data_test->m;
+ double value;
+ double *x1, *x2;
+
+ *K2 = Calloc(double, n_test*n_train);
+
+ //printf("Training RAW\n");
+ //print_matrix(data_train->RAW, n_train, m+1);
+
+ //printf("Testing RAW\n");
+ //print_matrix(data_test->RAW, n_test, m+1);
+
+ for (i=0; i<n_test; i++) {
+ for (j=0; j<n_train; j++) {
+ x1 = &data_test->RAW[i*(m+1)+1];
+ x2 = &data_train->RAW[j*(m+1)+1];
+ if (model->kerneltype == K_POLY)
+ value = msvmmaj_compute_poly(x1, x2,
+ model->kernelparam,
+ m);
+ else if (model->kerneltype == K_RBF)
+ value = msvmmaj_compute_rbf(x1, x2,
+ model->kernelparam,
+ m);
+ else if (model->kerneltype == K_SIGMOID)
+ value = msvmmaj_compute_sigmoid(x1, x2,
+ model->kernelparam,
+ m);
+ else {
+ fprintf(stderr, "Unknown kernel type in "
+ "msvmmaj_make_crosskernel\n");
+ exit(1);
+ }
+ matrix_set((*K2), n_train, i, j, value);
+ }
+ }
+
+ //printf("cross K2:\n");
+ //print_matrix((*K2), n_test, n_train);
+
}
/**
diff --git a/src/msvmmaj_pred.c b/src/msvmmaj_pred.c
index 98b6e0a..b551f12 100644
--- a/src/msvmmaj_pred.c
+++ b/src/msvmmaj_pred.c
@@ -15,9 +15,23 @@
#include "libMSVMMaj.h"
#include "msvmmaj.h"
+#include "msvmmaj_kernel.h"
#include "msvmmaj_matrix.h"
#include "msvmmaj_pred.h"
+#include "util.h" // testing
+
+void msvmmaj_predict_labels(struct MajData *data_test,
+ struct MajData *data_train, struct MajModel *model,
+ long *predy)
+{
+ if (model->kerneltype == K_LINEAR)
+ msvmmaj_predict_labels_linear(data_test, model, predy);
+ else
+ msvmmaj_predict_labels_kernel(data_test, data_train, model,
+ predy);
+}
+
/**
* @brief Predict class labels of data given and output in predy
*
@@ -32,7 +46,8 @@
* @param[in] model MajModel with optimized V
* @param[out] predy pre-allocated vector to record predictions in
*/
-void msvmmaj_predict_labels(struct MajData *data, struct MajModel *model, long *predy)
+void msvmmaj_predict_labels_linear(struct MajData *data,
+ struct MajModel *model, long *predy)
{
long i, j, k, label;
double norm, min_dist;
@@ -72,7 +87,8 @@ void msvmmaj_predict_labels(struct MajData *data, struct MajModel *model, long *
min_dist = 1000000000.0;
for (j=0; j<K; j++) {
for (k=0; k<K-1; k++) {
- S[k] = matrix_get(ZV, K-1, i, k) - matrix_get(U, K-1, j, k);
+ S[k] = matrix_get(ZV, K-1, i, k) -
+ matrix_get(U, K-1, j, k);
}
norm = cblas_dnrm2(K, S, 1);
if (norm < min_dist) {
@@ -88,6 +104,110 @@ void msvmmaj_predict_labels(struct MajData *data, struct MajModel *model, long *
free(S);
}
+void msvmmaj_predict_labels_kernel(struct MajData *data_test,
+ struct MajData *data_train, struct MajModel *model,
+ long *predy)
+{
+ long i, j, k, label;
+ double norm, min_dist;
+
+ long n_train = data_train->n;
+ long n_test = data_test->n;
+ long r = model->m;
+ long K = model->K;
+
+ double *K2 = NULL;
+ msvmmaj_make_crosskernel(model, data_train, data_test, &K2);
+
+ //printf("K2:\n");
+ //print_matrix(K2, n_test, n_train);
+
+ //printf("P:\n");
+ //print_matrix(data_train->Z, n_train, r+1);
+
+ //printf("Sigma:\n");
+ //print_matrix(data_train->J, 1, r+1);
+
+ double *S = Calloc(double, K-1);
+ double *ZV = Calloc(double, n_test*(r+1));
+ double *KPS = Calloc(double, n_test*(r+1));
+ double *U = Calloc(double, K*(K-1));
+
+ msvmmaj_simplex_gen(K, U);
+
+ // were doing the computations explicitly since P is included in
+ // data_train->Z. Might want to look at this some more if it turns out
+ // to be slow.
+
+ double value, rowvalue;
+ for (i=0; i<n_test; i++) {
+ for (j=1; j<r+1; j++) {
+ value = 0.0;
+ for (k=0; k<n_train; k++) {
+ rowvalue = matrix_get(K2, n_train, i, k);
+ rowvalue *= matrix_get(data_train->Z, r+1, k,
+ j);
+ value += rowvalue;
+ }
+ value *= matrix_get(data_train->J, 1, j, 0);
+ matrix_set(KPS, r+1, i, j, value);
+ }
+ matrix_set(KPS, r+1, i, 0, 1.0);
+ }
+
+ //printf("\nPrinting KPS:\n");
+ //print_matrix(KPS, n_test, r+1);
+
+ //printf("\nPrinting Omega (model->m = %li):\n", model->m);
+ //print_matrix(model->V, r+1, K-1);
+
+ cblas_dgemm(
+ CblasRowMajor,
+ CblasNoTrans,
+ CblasNoTrans,
+ n_test,
+ K-1,
+ r+1,
+ 1.0,
+ KPS,
+ r+1,
+ model->V,
+ K-1,
+ 0.0,
+ ZV,
+ K-1);
+
+ //printf("\nPrinting ZV:\n");
+ //print_matrix(ZV, n_test, K-1);
+ //printf("\n");
+
+ for (i=0; i<n_test; i++) {
+ label = 0;
+ min_dist = 1e10;
+ for (j=0; j<K; j++) {
+ for (k=0; k<K-1; k++) {
+ S[k] = matrix_get(ZV, K-1, i, k) -
+ matrix_get(U, K-1, j, k);
+ }
+ norm = cblas_dnrm2(K, S, 1);
+ if (norm < min_dist) {
+ label = j+1;
+ min_dist = norm;
+ }
+ }
+ predy[i] = label;
+ //printf("predy[%li] = %li\n", i, label);
+ }
+
+ free(ZV);
+ free(U);
+ free(S);
+ free(KPS);
+ free(K2);
+
+ //exit(1);
+}
+
/**
* @brief Calculate the predictive performance (percentage correct)
*
diff --git a/src/msvmmaj_train.c b/src/msvmmaj_train.c
index 5018c3f..0f42ff6 100644
--- a/src/msvmmaj_train.c
+++ b/src/msvmmaj_train.c
@@ -79,7 +79,8 @@ void msvmmaj_optimize(struct MajModel *model, struct MajData *data)
while ((it < MAX_ITER) && (Lbar - L)/L > model->epsilon)
{
- // ensure V contains newest V and Vbar contains V from previous
+ // ensure V contains newest V and Vbar contains V from
+ // previous
msvmmaj_get_update(model, data, B, ZAZ, ZAZV, ZAZVT);
if (it > 50)
msvmmaj_step_doubling(model);
@@ -87,7 +88,7 @@ void msvmmaj_optimize(struct MajModel *model, struct MajData *data)
Lbar = L;
L = msvmmaj_get_loss(model, data, ZV);
- if (it%1 == 0)
+ if (it%100 == 0)
note("iter = %li, L = %15.16f, Lbar = %15.16f, "
"reldiff = %15.16f\n", it, L, Lbar, (Lbar - L)/L);
it++;
@@ -161,7 +162,7 @@ double msvmmaj_get_loss(struct MajModel *model, struct MajData *data,
for (j=0; j<K-1; j++) {
rowvalue += pow(matrix_get(model->V, K-1, i, j), 2.0);
}
- value += model->J[i] * rowvalue;
+ value += data->J[i] * rowvalue;
}
loss += model->lambda * value;
@@ -422,6 +423,7 @@ void msvmmaj_get_update(struct MajModel *model, struct MajData *data, double *B,
1.0,
ZAZV,
K-1);
+
/*
* Add lambda to all diagonal elements except the first one. Recall
* that ZAZ is of size m+1 and is symmetric.
@@ -429,7 +431,7 @@ void msvmmaj_get_update(struct MajModel *model, struct MajData *data, double *B,
i = 0;
for (j=0; j<m; j++) {
i += (m+1) + 1;
- ZAZ[i] += model->lambda * model->J[j+1];
+ ZAZ[i] += model->lambda * data->J[j+1];
}
// For the LAPACK call we need to switch to Column-
diff --git a/src/msvmmaj_train_dataset.c b/src/msvmmaj_train_dataset.c
index 0221a89..f0c09e5 100644
--- a/src/msvmmaj_train_dataset.c
+++ b/src/msvmmaj_train_dataset.c
@@ -16,6 +16,7 @@
#include "libMSVMMaj.h"
#include "msvmmaj.h"
#include "msvmmaj_init.h"
+#include "msvmmaj_kernel.h"
#include "msvmmaj_matrix.h"
#include "msvmmaj_train.h"
#include "msvmmaj_train_dataset.h"
@@ -73,7 +74,7 @@ void make_queue(struct Training *training, struct Queue *queue,
task->test_data = test_data;
task->folds = training->folds;
task->kerneltype = training->kerneltype;
- task->kernel_param = Calloc(double, training->Ng +
+ task->kernelparam = Calloc(double, training->Ng +
training->Nc + training->Nd);
queue->tasks[i] = task;
}
@@ -135,7 +136,7 @@ void make_queue(struct Training *training, struct Queue *queue,
while (i < N && training->Ng > 0)
for (j=0; j<training->Ng; j++)
for (k=0; k<cnt; k++) {
- queue->tasks[i]->kernel_param[0] =
+ queue->tasks[i]->kernelparam[0] =
training->gammas[j];
i++;
}
@@ -145,7 +146,7 @@ void make_queue(struct Training *training, struct Queue *queue,
while (i < N && training->Nc > 0)
for (j=0; j<training->Nc; j++)
for (k=0; k<cnt; k++) {
- queue->tasks[i]->kernel_param[1] =
+ queue->tasks[i]->kernelparam[1] =
training->coefs[j];
i++;
}
@@ -155,7 +156,7 @@ void make_queue(struct Training *training, struct Queue *queue,
while (i < N && training->Nd > 0)
for (j=0; j<training->Nd; j++)
for (k=0; k<cnt; k++) {
- queue->tasks[i]->kernel_param[2] =
+ queue->tasks[i]->kernelparam[2] =
training->degrees[j];
i++;
}
@@ -448,6 +449,8 @@ double cross_validation(struct MajModel *model, struct MajData *data,
for (f=0; f<folds; f++) {
msvmmaj_get_tt_split(data, train_data, test_data, cv_idx, f);
+
+ msvmmaj_make_kernel(model, train_data);
// reallocate the model if necessary for the new train split
msvmmaj_reallocate_model(model, train_data->n);
@@ -512,11 +515,7 @@ void start_training_cv(struct Queue *q)
main_s = clock();
while (task) {
- note("(%03li/%03li)\tw = %li\te = %f\tp = %f\tk = %f\t "
- "l = %f\t",
- task->ID+1, q->N, task->weight_idx,
- task->epsilon,
- task->p, task->kappa, task->lambda);
+ print_progress_string(task, q->N);
make_model_from_task(task, model);
loop_s = clock();
@@ -590,7 +589,9 @@ void msvmmaj_reallocate_model(struct MajModel *model, long n)
* It would probably be better to train the model on the training set using
* cross validation and only use the test set when comparing with other
* methods. The way it is now, you're finding out which parameters predict
- * _this_ test set best, which is not what you want.
+ * _this_ test set best, which is not what you want. This function should
+ * therefore not be used and is considered deprecated, to be removed in the
+ * future .
*
* @param[in] q Queue with Task structs to run
*
@@ -638,9 +639,11 @@ void start_training_tt(struct Queue *q)
MSVMMAJ_OUTPUT_FILE = fid;
predy = Calloc(long, task->test_data->n);
- msvmmaj_predict_labels(task->test_data, model, predy);
+ msvmmaj_predict_labels(task->test_data, task->train_data,
+ model, predy);
if (task->test_data->y != NULL)
- total_perf = msvmmaj_prediction_perf(task->test_data, predy);
+ total_perf = msvmmaj_prediction_perf(task->test_data,
+ predy);
msvmmaj_seed_model_V(model, seed_model);
msvmmaj_free_model(model);
@@ -675,7 +678,7 @@ void free_queue(struct Queue *q)
{
long i;
for (i=0; i<q->N; i++) {
- free(q->tasks[i]->kernel_param);
+ free(q->tasks[i]->kernelparam);
free(q->tasks[i]);
}
free(q->tasks);
@@ -694,11 +697,16 @@ void free_queue(struct Queue *q)
*/
void make_model_from_task(struct Task *task, struct MajModel *model)
{
+ // copy basic model parameters
model->weight_idx = task->weight_idx;
model->epsilon = task->epsilon;
model->p = task->p;
model->kappa = task->kappa;
model->lambda = task->lambda;
+
+ // copy kernel parameters
+ model->kerneltype = task->kerneltype;
+ model->kernelparam = task->kernelparam;
}
/**
@@ -718,4 +726,58 @@ void copy_model(struct MajModel *from, struct MajModel *to)
to->p = from->p;
to->kappa = from->kappa;
to->lambda = from->lambda;
+
+ to->kerneltype = from->kerneltype;
+ switch (to->kerneltype) {
+ case K_LINEAR:
+ break;
+ case K_POLY:
+ to->kernelparam = Malloc(double, 3);
+ to->kernelparam[0] = from->kernelparam[0];
+ to->kernelparam[1] = from->kernelparam[1];
+ to->kernelparam[2] = from->kernelparam[2];
+ break;
+ case K_RBF:
+ to->kernelparam = Malloc(double, 1);
+ to->kernelparam[0] = from->kernelparam[0];
+ break;
+ case K_SIGMOID:
+ to->kernelparam = Malloc(double, 2);
+ to->kernelparam[0] = from->kernelparam[0];
+ to->kernelparam[1] = from->kernelparam[1];
+ break;
+ }
+}
+
+/**
+ * @brief Print the description of the current task on screen
+ *
+ * @details
+ * To track the progress of the grid search the parameters of the current task
+ * are written to the output specified in MSVMMAJ_OUTPUT_FILE. Since the
+ * parameters differ with the specified kernel, this function writes a
+ * parameter string depending on which kernel is used.
+ *
+ * @param[in] task the Task specified
+ * @param[in] N total number of tasks
+ *
+ */
+void print_progress_string(struct Task *task, long N)
+{
+ char buffer[MAX_LINE_LENGTH];
+ sprintf(buffer, "(%03li/%03li)\t", task->ID+1, N);
+ if (task->kerneltype == K_POLY)
+ sprintf(buffer + strlen(buffer), "d = %2.2f\t",
+ task->kernelparam[2]);
+ if (task->kerneltype == K_POLY || task->kerneltype == K_SIGMOID)
+ sprintf(buffer + strlen(buffer), "c = %2.2f\t",
+ task->kernelparam[1]);
+ if (task->kerneltype == K_POLY || task->kerneltype == K_SIGMOID ||
+ task->kerneltype == K_RBF)
+ sprintf(buffer + strlen(buffer), "g = %2.2f\t",
+ task->kernelparam[0]);
+ sprintf(buffer + strlen(buffer), "eps = %g\tw = %i\tk = %2.2f\t"
+ "l = %f\tp = %2.2f\t", task->epsilon,
+ task->weight_idx, task->kappa, task->lambda, task->p);
+ note(buffer);
}
diff --git a/src/predMSVMMaj.c b/src/predMSVMMaj.c
index e67e430..3dfcf08 100644
--- a/src/predMSVMMaj.c
+++ b/src/predMSVMMaj.c
@@ -1,3 +1,9 @@
+/*
+ * 20140317:
+ * THIS FUNCTION IS DEPRECATED, SINCE IT DOES NOT WORK WITH KERNELS.
+ *
+ */
+
/**
* @file predMSVMMaj.c
* @author Gertjan van den Burg
diff --git a/src/trainMSVMMaj.c b/src/trainMSVMMaj.c
index af91eaf..66f6450 100644
--- a/src/trainMSVMMaj.c
+++ b/src/trainMSVMMaj.c
@@ -105,8 +105,7 @@ int main(int argc, char **argv)
// seed the random number generator (only place in programs is in
// command line interfaces)
- //srand(time(NULL));
- srand(123456);
+ srand(time(NULL));
if (msvmmaj_check_argv_eq(argc, argv, "-m")) {
struct MajModel *seed_model = msvmmaj_init_model();
diff --git a/src/trainMSVMMajdataset.c b/src/trainMSVMMajdataset.c
index e9f8b8e..1929584 100644
--- a/src/trainMSVMMajdataset.c
+++ b/src/trainMSVMMajdataset.c
@@ -98,7 +98,8 @@ int main(int argc, char **argv)
struct Queue *q = Malloc(struct Queue, 1);
make_queue(training, q, train_data, test_data);
- srand(time(NULL));
+ // srand(time(NULL));
+ srand(123456);
note("Starting training\n");
if (training->traintype == TT)
@@ -163,6 +164,23 @@ void parse_command_line(int argc, char **argv, char *input_filename)
strcpy(input_filename, argv[i]);
}
+KernelType parse_kernel_str(char *kernel_line)
+{
+ if (str_endswith(kernel_line, "LINEAR\n")) {
+ return K_LINEAR;
+ } else if (str_endswith(kernel_line, "POLY\n")) {
+ return K_POLY;
+ } else if (str_endswith(kernel_line, "RBF\n")) {
+ return K_RBF;
+ } else if (str_endswith(kernel_line, "SIGMOID\n")) {
+ return K_SIGMOID;
+ } else {
+ fprintf(stderr, "Unknown kernel specified on line: %s\n",
+ kernel_line);
+ exit(1);
+ }
+}
+
/**
* @brief Read the Training struct from file
*
@@ -254,25 +272,7 @@ void read_training_from_file(char *input_filename, struct Training *training)
"takes one value. Additional "
"fields are ignored.\n");
} else if (str_startswith(buffer, "kernel:")) {
- nr = all_longs_str(buffer, 7, lparams);
- if (nr > 1)
- fprintf(stderr, "Field \"kernel\" only takes "
- "one value. Additional "
- "fields are ignored.\n");
- switch (lparams[0]) {
- case 0:
- training->kerneltype = K_LINEAR;
- break;
- case 1:
- training->kerneltype = K_POLY;
- break;
- case 2:
- training->kerneltype = K_RBF;
- break;
- case 3:
- training->kerneltype = K_SIGMOID;
- break;
- }
+ training->kerneltype = parse_kernel_str(buffer);
} else if (str_startswith(buffer, "gamma:")) {
nr = all_doubles_str(buffer, 6, params);
if (training->kerneltype == K_LINEAR) {
@@ -289,7 +289,7 @@ void read_training_from_file(char *input_filename, struct Training *training)
nr = all_doubles_str(buffer, 5, params);
if (training->kerneltype == K_LINEAR ||
training->kerneltype == K_RBF) {
- fprintf(stderr, "Field \"coef\" ignored with"
+ fprintf(stderr, "Field \"coef\" ignored with "
"specified kernel.\n");
training->Nc = 0;
break;
diff --git a/training/wine.training b/training/wine.training
index 7480756..00a8f31 100644
--- a/training/wine.training
+++ b/training/wine.training
@@ -1,8 +1,10 @@
train: ./data/wine.train
p: 1.0 1.5 2.0
kappa: -0.9 0.0 1.0
-lambda: 64 32 16 8 4 2 1 0.5 0.25 0.125 0.0625 0.03125 0.015625 0.0078125 0.00390625 0.001953125 0.0009765625 0.00048828125 0.000244140625
+lambda: 64 16 4 1 0.25 0.0625 0.015625 0.00390625 0.0009765625 0.000244140625
epsilon: 1e-6
weight: 1 2
folds: 10
repeats: 10
+kernel: RBF
+gamma: 1e-3 1e-2 1e-1 1e0 1e1 1e2 1e3
generated by cgit v1.2.3 (git 2.25.1) at 2026-01-10 00:41:56 +0000