/** * @file gensvm_update.c * @author G.J.J. van den Burg * @date 2016-10-14 * @brief Functions for getting an update of the majorization algorithm * * @copyright Copyright 2016, G.J.J. van den Burg. This file is part of GenSVM. GenSVM is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. GenSVM is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GenSVM. If not, see . */ #include "gensvm_update.h" /** * Number of rows in a single block for the ZAZ calculation in * gensvm_get_ZAZ_ZB_sparse(). */ #ifndef GENSVM_BLOCK_SIZE #define GENSVM_BLOCK_SIZE 512 #endif /** * @brief Calculate the value of omega for a single instance * * @details * This function calculates the value of the @f$ \omega_i @f$ variable for a * single instance, where * @f[ * \omega_i = \frac{1}{p} \left( \sum_{j \neq y_i} h^p\left( * \overline{q}_i^{(y_i j)} \right) \right)^{1/p-1} * @f] * Note that his function uses the precalculated values from GenModel::H and * GenModel::R to speed up the computation. * * @param[in] model GenModel structure with the current model * @param[in] data GenData structure with the data (used for y) * @param[in] i index of the instance for which to calculate omega * @returns the value of omega for instance i * */ double gensvm_calculate_omega(struct GenModel *model, struct GenData *data, long i) { long j; double h, omega = 0.0, p = model->p; for (j=0; jK; j++) { if (j == (data->y[i]-1)) continue; h = matrix_get(model->H, model->K, i, j); omega += pow(h, p); } omega = (1.0/p)*pow(omega, 1.0/p - 1.0); return omega; } /** * @brief Check if we can do simple majorization for a given instance * * @details * A simple majorization is possible if at most one of the Huberized hinge * errors is nonzero for an instance. This is checked here. For this we * compute the product of the Huberized error for all @f$j \neq y_i@f$ and * check if strictly less than 2 are nonzero. See also the @ref update_math. * * @param[in] model GenModel structure with the current model * @param[in] data GenData structure with the data (used for y) * @param[in] i index of the instance for which to check * @returns whether or not we can do simple majorization * */ bool gensvm_majorize_is_simple(struct GenModel *model, struct GenData *data, long i) { long j; double h, value = 0; for (j=0; jK; j++) { if (j == (data->y[i]-1)) continue; h = matrix_get(model->H, model->K, i, j); value += h > 0; if (value > 1) return false; } return true; } /** * @brief Compute majorization coefficients for non-simple instance * * @details * In this function we compute the majorization coefficients needed for an * instance with a non-simple majorization (@f$\varepsilon_i = 0@f$). In this * function, we distinguish a number of cases depending on the value of * GenModel::p and the respective value of @f$\overline{q}_i^{(y_ij)}@f$. Note * that the linear coefficient is of the form @f$b - a\overline{q}@f$, but * often the second term is included in the definition of @f$b@f$, so it can * be optimized out. The output argument \p b_aq contains this difference * therefore in one go. More details on this function can be found in the @ref * update_math. See also gensvm_calculate_ab_simple(). * * @param[in] model GenModel structure with the current model * @param[in] i index for the instance * @param[in] j index for the class * @param[out] *a output argument for the quadratic coefficient * @param[out] *b_aq output argument for the linear coefficient. * */ void gensvm_calculate_ab_non_simple(struct GenModel *model, long i, long j, double *a, double *b_aq) { double q = matrix_get(model->Q, model->K, i, j); double p = model->p; double kappa = model->kappa; const double a2g2 = 0.25*p*(2.0*p - 1.0)*pow((kappa+1.0)/2.0,p-2.0); if (2.0 - model->p < 1e-2) { if (q <= - kappa) { *b_aq = 0.5 - kappa/2.0 - q; } else if ( q <= 1.0) { *b_aq = pow(1.0 - q, 3.0)/(2.0*pow(kappa + 1.0, 2.0)); } else { *b_aq = 0; } *a = 1.5; } else { if (q <= (p + kappa - 1.0)/(p - 2.0)) { *a = 0.25*pow(p, 2.0)*pow(0.5 - kappa/2.0 - q, p - 2.0); } else if (q <= 1.0) { *a = a2g2; } else { *a = 0.25*pow(p, 2.0)*pow((p/(p - 2.0))*(0.5 - kappa/2.0 - q), p - 2.0); *b_aq = (*a)*(2.0*q + kappa - 1.0)/(p - 2.0) + 0.5*p*pow(p/(p - 2.0)*(0.5 - kappa/2.0 - q), p - 1.0); } if (q <= -kappa) { *b_aq = 0.5*p*pow(0.5 - kappa/2.0 - q, p - 1.0); } else if ( q <= 1.0) { *b_aq = p*pow(1.0 - q, 2.0*p - 1.0)/pow(2*kappa+2.0, p); } } } /** * @brief Compute majorization coefficients for simple instances * * @details * In this function we compute the majorization coefficients needed for an * instance with a simple majorization. This corresponds to the non-simple * majorization for the case where GenModel::p equals 1. Due to this condition * the majorization coefficients are quite simple to compute. Note that the * linear coefficient of the majorization is of the form @f$b - * a\overline{q}@f$, but often the second term is included in the definition * of @f$b@f$, so it can be optimized out. For more details see the @ref * update_math, and gensvm_calculate_ab_non_simple(). * * @param[in] model GenModel structure with the current model * @param[in] i index for the instance * @param[in] j index for the class * @param[out] *a output argument for the quadratic coefficient * @param[out] *b_aq output argument for the linear coefficient * */ void gensvm_calculate_ab_simple(struct GenModel *model, long i, long j, double *a, double *b_aq) { double q = matrix_get(model->Q, model->K, i, j); if (q <= - model->kappa) { *a = 0.25/(0.5 - model->kappa/2.0 - q); *b_aq = 0.5; } else if (q <= 1.0) { *a = 1.0/(2.0*model->kappa + 2.0); *b_aq = (1.0 - q)*(*a); } else { *a = -0.25/(0.5 - model->kappa/2.0 - q); *b_aq = 0; } } /** * @brief Compute the alpha_i and beta_i for an instance * * @details * This computes the @f$\alpha_i@f$ value for an instance, and simultaneously * updating the row of the B matrix corresponding to that * instance (the @f$\boldsymbol{\beta}_i'@f$). The advantage of doing this at * the same time is that we can compute the a and b values simultaneously in * the gensvm_calculate_ab_simple() and gensvm_calculate_ab_non_simple() * functions. * * The computation is done by first checking whether simple majorization is * possible for this instance. If so, the @f$\omega_i@f$ value is set to 1.0, * otherwise this value is computed. If simple majorization is possible, the * coefficients a and b_aq are computed by gensvm_calculate_ab_simple(), * otherwise they're computed by gensvm_calculate_ab_non_simple(). Next, the * beta_i updated through the efficient BLAS daxpy function, and part of the * value of @f$\alpha_i@f$ is computed. The final value of @f$\alpha_i@f$ is * returned. * * @param[in] model GenModel structure with the current model * @param[in] data GenData structure with the data * @param[in] i index of the instance to update * @param[out] beta beta vector of linear coefficients (assumed to * be allocated elsewhere, initialized here) * @returns the @f$\alpha_i@f$ value of this instance * */ double gensvm_get_alpha_beta(struct GenModel *model, struct GenData *data, long i, double *beta) { bool simple; long j, K = model->K; double omega, a, b_aq = 0.0, alpha = 0.0; double *uu_row = NULL; const double in = 1.0/((double) model->n); simple = gensvm_majorize_is_simple(model, data, i); omega = simple ? 1.0 : gensvm_calculate_omega(model, data, i); Memset(beta, double, K-1); for (j=0; jy[i]-1)) continue; // calculate the a_ijk and (b_ijk - a_ijk q_i^(kj)) values if (simple) { gensvm_calculate_ab_simple(model, i, j, &a, &b_aq); } else { gensvm_calculate_ab_non_simple(model, i, j, &a, &b_aq); } // daxpy on beta and UU // daxpy does: y = a*x + y // so y = beta, UU_row = x, a = factor b_aq *= model->rho[i] * omega * in; uu_row = &model->UU[((data->y[i]-1)*K+j)*(K-1)]; cblas_daxpy(K-1, b_aq, uu_row, 1, beta, 1); // increment Avalue alpha += a; } alpha *= omega * model->rho[i] * in; return alpha; } /** * @brief Perform a single step of the majorization algorithm to update V * * @details * This function contains the main update calculations of the algorithm. These * calculations are necessary to find a new update V. The calculations exist of * recalculating the majorization coefficients for all instances and all * classes, and solving a linear system to find V. * * Because the function gensvm_get_update() is always called after a call to * gensvm_get_loss() with the same GenModel::V, it is unnecessary to calculate * the updated errors GenModel::Q and GenModel::H here too. This saves on * computation time. * * In calculating the majorization coefficients we calculate the elements of a * diagonal matrix A with elements * @f[ * A_{i, i} = \frac{1}{n} \rho_i \sum_{j \neq k} \left[ * \varepsilon_i a_{ijk}^{(1)} + (1 - \varepsilon_i) \omega_i * a_{ijk}^{(p)} \right], * @f] * where @f$ k = y_i @f$. * Since this matrix is only used to calculate the matrix @f$ Z' A Z @f$, it * is efficient to update a matrix ZAZ through consecutive rank 1 updates with * a single element of A and the corresponding row of Z. The BLAS function * dsyr is used for this. * * The B matrix is has rows * @f[ * \boldsymbol{\beta}_i' = \frac{1}{n} \rho_i \sum_{j \neq k} \left[ * \varepsilon_i \left( b_{ijk}^{(1)} - a_{ijk}^{(1)} * \overline{q}_i^{(kj)} \right) + (1 - \varepsilon_i) * \omega_i \left( b_{ijk}^{(p)} - a_{ijk}^{(p)} * \overline{q}_i^{(kj)} \right) \right] * \boldsymbol{\delta}_{kj}' * @f] * This is also split into two cases, one for which @f$ \varepsilon_i = 1 @f$, * and one for when it is 0. The 3D simplex difference matrix is used here, in * the form of the @f$ \boldsymbol{\delta}_{kj}' @f$. * * Finally, the following system is solved * @f[ * (\textbf{Z}'\textbf{AZ} + \lambda \textbf{J})\textbf{V} = * (\textbf{Z}'\textbf{AZ}\overline{\textbf{V}} + \textbf{Z}' * \textbf{B}) * @f] * solving this system is done through dposv(). * * @todo * Consider using CblasColMajor everywhere * * @param[in,out] model model to be updated * @param[in] data data used in model * @param[in] work allocated workspace to use */ void gensvm_get_update(struct GenModel *model, struct GenData *data, struct GenWork *work) { int status; long i, j; long m = model->m; long K = model->K; // compute the ZAZ and ZB matrices gensvm_get_ZAZ_ZB(model, data, work); // Calculate right-hand side of system we want to solve // dsymm performs ZB := 1.0 * (ZAZ) * Vbar + 1.0 * ZB // the right-hand side is thus stored in ZB after this call // Note: LDB and LDC are second dimensions of the matrices due to // Row-Major order cblas_dsymm(CblasRowMajor, CblasLeft, CblasUpper, m+1, K-1, 1, work->ZAZ, m+1, model->V, K-1, 1.0, work->ZB, K-1); // Calculate left-hand side of system we want to solve // Add lambda to all diagonal elements except the first one. Recall // that ZAZ is of size m+1 and is symmetric. for (i=m+2; i<=m*(m+2); i+=m+2) work->ZAZ[i] += model->lambda; // Lapack uses column-major order, so we transform the ZB matrix to // correspond to this. for (i=0; iZBc[j*(m+1)+i] = work->ZB[i*(K-1)+j]; // Solve the system using dposv. Note that above the upper triangular // part has always been used in row-major order for ZAZ. This // corresponds to the lower triangular part in column-major order. status = dposv('L', m+1, K-1, work->ZAZ, m+1, work->ZBc, m+1); // Use dsysv as fallback, for when the ZAZ matrix is not positive // semi-definite for some reason (perhaps due to rounding errors). // This step shouldn't be necessary but is included for safety. if (status != 0) { err("[GenSVM Warning]: Received nonzero status from " "dposv: %i\n", status); int *IPIV = Malloc(int, m+1); double *WORK = Malloc(double, 1); status = dsysv('L', m+1, K-1, work->ZAZ, m+1, IPIV, work->ZBc, m+1, WORK, -1); int LWORK = WORK[0]; WORK = Realloc(WORK, double, LWORK); status = dsysv('L', m+1, K-1, work->ZAZ, m+1, IPIV, work->ZBc, m+1, WORK, LWORK); if (status != 0) err("[GenSVM Warning]: Received nonzero " "status from dsysv: %i\n", status); free(WORK); WORK = NULL; free(IPIV); IPIV = NULL; } // the solution is now stored in ZBc, in column-major order. Here we // convert this back to row-major order for (i=0; iZB[i*(K-1)+j] = work->ZBc[j*(m+1)+i]; // copy the old V to Vbar and the new solution to V for (i=0; iVbar, K-1, i, j, matrix_get(model->V, K-1, i, j)); matrix_set(model->V, K-1, i, j, matrix_get(work->ZB, K-1, i, j)); } } } /** * @brief Calculate Z'*A*Z and Z'*B for dense matrices * * @details * This function calculates the matrices Z'*A*Z and Z'*B for the case where Z * is stored as a dense matrix. It calculates the Z'*A*Z product by * constructing a matrix LZ = (A^(1/2) * Z), and calculating (LZ)'*(LZ) with * the BLAS dsyrk function. The matrix Z'*B is calculated with successive * rank-1 updates using the BLAS dger function. These functions came out as * the most efficient way to do these computations in several simulation * studies. * * @param[in] model a GenModel holding the current model * @param[in] data a GenData with the data * @param[in,out] work an allocated GenWork structure, contains * updated ZAZ and ZB matrices on exit. */ void gensvm_get_ZAZ_ZB_dense(struct GenModel *model, struct GenData *data, struct GenWork *work) { long i; double alpha, sqalpha; long n = model->n; long m = model->m; long K = model->K; // generate Z'*A*Z and Z'*B by rank 1 operations for (i=0; ibeta); // calculate row of matrix LZ, which is a scalar // multiplication of sqrt(alpha_i) and row z_i' of Z // Note that we use the fact that the first column of Z is // always 1, by only computing the product for m values and // copying the first element over. sqalpha = sqrt(alpha); work->LZ[i*(m+1)] = sqalpha; cblas_daxpy(m, sqalpha, &data->Z[i*(m+1)+1], 1, &work->LZ[i*(m+1)+1], 1); // rank 1 update of matrix Z'*B // Note: LDA is the second dimension of ZB because of // Row-Major order cblas_dger(CblasRowMajor, m+1, K-1, 1, &data->Z[i*(m+1)], 1, work->beta, 1, work->ZB, K-1); } // calculate Z'*A*Z by symmetric multiplication of LZ with itself // (ZAZ = (LZ)' * (LZ) cblas_dsyrk(CblasRowMajor, CblasUpper, CblasTrans, m+1, n, 1.0, work->LZ, m+1, 0.0, work->ZAZ, m+1); } /** * @brief Calculate Z'*A*Z and Z'*B for sparse matrices * * @details * This function calculates the matrices Z'*A*Z and Z'*B for the case where Z * is stored as a CSR sparse matrix (GenSparse structure). It computes only * the products of the Z'*A*Z matrix that need to be computed, and updates the * Z'*B matrix row-wise for each non-zero element of a row of Z, using a BLAS * daxpy call. * * This function calculates the matrix product Z'*A*Z in separate blocks, * based on the number of rows defined in the GENSVM_BLOCK_SIZE variable. This * is done to improve numerical precision for very large datasets. Due to * rounding errors, precision can become an issue for these large datasets, * when separate blocks are used and added to the result separately, this can * be alleviated a little bit. See also: http://stackoverflow.com/q/40286989 * * @sa * gensvm_get_ZAZ_ZB() * gensvm_get_ZAZ_ZB_dense() * * @param[in] model a GenModel holding the current model * @param[in] data a GenData with the data * @param[in,out] work an allocated GenWork structure, contains * updated ZAZ and ZB matrices on exit. */ void gensvm_get_ZAZ_ZB_sparse(struct GenModel *model, struct GenData *data, struct GenWork *work) { long *Zia = NULL, *Zja = NULL; long b, i, j, k, K, jj, kk, jj_start, jj_end, blk_start, blk_end, rem_size, n_blocks, n_row = data->spZ->n_row, n_col = data->spZ->n_col; double temp, alpha, z_ij, *vals = NULL; K = model->K; Zia = data->spZ->ia; Zja = data->spZ->ja; vals = data->spZ->values; // calculate ZAZ using blocks of rows of Z. This helps avoiding // rounding errors, which increases precision, and in turn helps // convergence of the IM algorithm. // see also: http://stackoverflow.com/q/40286989/ n_blocks = floor(n_row / GENSVM_BLOCK_SIZE); rem_size = n_row % GENSVM_BLOCK_SIZE; for (b=0; b<=n_blocks; b++) { blk_start = b * GENSVM_BLOCK_SIZE; blk_end = blk_start; blk_end += (b == n_blocks) ? rem_size : GENSVM_BLOCK_SIZE; Memset(work->tmpZAZ, double, n_col*n_col); for (i=blk_start; ibeta); jj_start = Zia[i]; jj_end = Zia[i+1]; for (jj=jj_start; jjbeta, 1, &work->ZB[j*(K-1)], 1); z_ij *= alpha; for (kk=jj; kktmpZAZ, n_col, j, Zja[kk], z_ij * vals[kk]); } } } // copy the intermediate results over to the actual ZAZ matrix for (j=0; jtmpZAZ, n_col, j, k); matrix_add(work->ZAZ, n_col, j, k, temp); } } } } /** * @brief Wrapper around calculation of Z'*A*Z and Z'*B for sparse and dense * * @details * This is a wrapper around gensvm_get_ZAZ_ZB_dense() and * gensvm_get_ZAZ_ZB_sparse(). See the documentation of those functions for * more info. * * @param[in] model a GenModel struct * @param[in] data a GenData struct * @param[in] work a GenWork struct * */ void gensvm_get_ZAZ_ZB(struct GenModel *model, struct GenData *data, struct GenWork *work) { gensvm_reset_work(work); if (data->Z == NULL) gensvm_get_ZAZ_ZB_sparse(model, data, work); else gensvm_get_ZAZ_ZB_dense(model, data, work); } /** * @brief Solve AX = B where A is symmetric positive definite. * * @details * Solve a linear system of equations AX = B where A is symmetric positive * definite. This function is a wrapper for the external LAPACK routine * dposv. * * @param[in] UPLO which triangle of A is stored * @param[in] N order of A * @param[in] NRHS number of columns of B * @param[in,out] A double precision array of size (LDA, N). On * exit contains the upper or lower factor of the * Cholesky factorization of A. * @param[in] LDA leading dimension of A * @param[in,out] B double precision array of size (LDB, NRHS). On * exit contains the N-by-NRHS solution matrix X. * @param[in] LDB the leading dimension of B * @returns info parameter which contains the status of the * computation: * - =0: success * - <0: if -i, the i-th argument had * an illegal value * - >0: if i, the leading minor of A * was not positive definite * * See the LAPACK documentation at: * http://www.netlib.org/lapack/explore-html/dc/de9/group__double_p_osolve.html */ int dposv(char UPLO, int N, int NRHS, double *A, int LDA, double *B, int LDB) { extern void dposv_(char *UPLO, int *Np, int *NRHSp, double *A, int *LDAp, double *B, int *LDBp, int *INFOp); int INFO; dposv_(&UPLO, &N, &NRHS, A, &LDA, B, &LDB, &INFO); return INFO; } /** * @brief Solve a system of equations AX = B where A is symmetric. * * @details * Solve a linear system of equations AX = B where A is symmetric. This * function is a wrapper for the external LAPACK routine dsysv. * * @param[in] UPLO which triangle of A is stored * @param[in] N order of A * @param[in] NRHS number of columns of B * @param[in,out] A double precision array of size (LDA, N). On * exit contains the block diagonal matrix D and * the multipliers used to obtain the factor U or * L from the factorization A = U*D*U**T or * A = L*D*L**T. * @param[in] LDA leading dimension of A * @param[in] IPIV integer array containing the details of D * @param[in,out] B double precision array of size (LDB, NRHS). On * exit contains the N-by-NRHS matrix X * @param[in] LDB leading dimension of B * @param[out] WORK double precision array of size max(1,LWORK). On * exit, WORK(1) contains the optimal LWORK * @param[in] LWORK the length of WORK, can be used for determining * the optimal blocksize for dsystrf. * @returns info parameter which contains the status of the * computation: * - =0: success * - <0: if -i, the i-th argument had an * illegal value * - >0: if i, D(i, i) is exactly zero, * no solution can be computed. * * See the LAPACK documentation at: * http://www.netlib.org/lapack/explore-html/d6/d0e/group__double_s_ysolve.html */ int dsysv(char UPLO, int N, int NRHS, double *A, int LDA, int *IPIV, double *B, int LDB, double *WORK, int LWORK) { extern void dsysv_(char *UPLO, int *Np, int *NRHSp, double *A, int *LDAp, int *IPIV, double *B, int *LDBp, double *WORK, int *LWORK, int *INFOp); int INFO; dsysv_(&UPLO, &N, &NRHS, A, &LDA, IPIV, B, &LDB, WORK, &LWORK, &INFO); return INFO; }