/*
 *   -- micMAGMA (version 0.1) --
 *      Univ. of Tennessee, Knoxville
 *      Univ. of California, Berkeley
 *      Univ. of Colorado, Denver
 *      April 2012
 *
 * @author Azzam Haidar 
 * @author Khairul Kabir 
 *
 * @generated s Fri Nov 22 14:21:24 2013
 */

#include <mkl.h>

#include <mkl_trans.h>
#include <scif.h>
#include <fcntl.h>
#include <offload.h>

//#ifdef __MIC__
//#define _M512_TYPED
//#include <lmmintrin.h>
//#endif
#include <immintrin.h>


#include "magma.h"
//#include "magma_s_bindings.h"
#include "magma_s_server.h"



void call_scopy(magma_mic_scopy_param scopy_param)
{
    magma_int_t n, incx, incy;
    float  *x, *y;
    
    n      = scopy_param.n;
    x      = (float *)scopy_param.x;
    incx   = scopy_param.incx;
    y      = (float *)scopy_param.y;
    incy   = scopy_param.incy;
    


    /* sequntial version I think the compiler is optimizing it */
    magma_int_t ix=0, iy=0, ii, chunk=1, nthreads, tid;
    if(incx<0) ix = (-n+1)*incx;
    if(incy<0) iy = (-n+1)*incy;
    for(magma_int_t i=0; i<n; i++){
           y[iy] = x[ix];
           ix += incx;
           iy += incy;
    }

    /* parallel version, where I use openmp to parallelize the copy */
 /* not tested if incx or incy <0
    #pragma omp parallel
    {
        magma_int_t ix=0, iy=0, ii, nthreads, tid;
        tid      = omp_get_thread_num();
        nthreads = omp_get_num_threads();
        if(incx<0) ix = (-n+1)*incx;
        if(incy<0) iy = (-n+1)*incy;
        ix = ix + tid*incx;
        iy = iy + tid*incy;
        for(ii=0; ii<n; ii+=nthreads){
            if((ii+tid)<n){
                y[iy] = x[ix];
                ix = ix + incx * nthreads;
                iy = iy + incy * nthreads;
            }
        }
    }
*/
    // mkl_scopy is not optimized yet and thus giving bad perf. it is better to use the handcode as the compiler is intelligent and optimize the code better. Azzam
    //scopy( &n, (float *)x, &incx, (float *)y, &incy);
}


//=========================================================================================
//magmablas our ssymm only left implemented here Azzam
//=========================================================================================
void magmablas_ssymm_mic(
    magma_side_t side, magma_side_t uplo, magma_int_t n, magma_int_t k, magma_int_t nb,
    float alpha, float *a, magma_int_t lda,
                              float *b, magma_int_t ldb,
    float beta,  float *c, magma_int_t ldc, magma_int_t agboff)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda)
    #define dB(i, j) (b + (i) + (j)*ldb)
    #define dC(i, j) (c + (i) + (j)*ldc)

    const float c_one = MAGMA_S_ONE;
    float newbeta=beta;
    magma_int_t m, ib, ioff,blockoffset;
    // loop over all blocks and does A*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    blockoffset = agboff % nb;
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        /*
        // Symmetrize the upper part of this block
        for(magma_int_t col=0; col<ib; col++){
            for(magma_int_t row=0; row<col; row++){
                *dA(i+row, i+col) =  MAGMA_S_CNJG( *(dA(i+col, i+row)) );
            }
        }
        */
        // C[i:n,0] += alpha * A[i:n,i] * B[i,0]
        /*
        m = n - i -ib;
        ssymm( lapack_const(side), lapack_const(uplo),
              &ib, &k, (const float *)&alpha, (float *)(dA(i, i)), &lda,
              (float *)(dB(i,0)), &ldb, (const float *)&newbeta,
              (float *)(dC(i,0)), &ldc);

        sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaNoTrans), &m, &k, &ib,
               (const float *)&alpha,   (float *)(dA(ioff+ib, ioff)), &lda,
                                                (float *)(dB(i,0)),        &ldb,
               (const float *)&newbeta, (float *)(dC(i+ib,0)),        &ldc );
        */
        m = n - i ;
        sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaNoTrans), &m, &k, &ib,
               (const float *)&alpha,   (float *)(dA(i, i)), &lda,
                                                (float *)(dB(i,0)),        &ldb,
               (const float *)&newbeta, (float *)(dC(i,0)),        &ldc );
        

        sgemm( lapack_const(MagmaTrans), lapack_const(MagmaNoTrans), &i, &k, &ib,
               (const float *)&alpha,   (float *)(dA(i, 0)), &lda,
                                                (float *)(dB(i,0)),     &ldb,
               (const float *)&newbeta, (float *)(dC(0,0)),     &ldc );

        newbeta = c_one;
        blockoffset = 0;
    }
}
#undef dA
#undef dB
#undef dC
//=========================================================================================

void call_ssymm(magma_mic_ssymm_param ssymm_param)
{
    magma_side_t side;
    magma_uplo_t uplo;
    magma_int_t m, n, lda, ldb, ldc, agboff;
    float alpha, beta, *a, *b, *c;

    side = ssymm_param.side;
    uplo = ssymm_param.uplo;
    m = ssymm_param.m;
    n = ssymm_param.n;
    alpha = ssymm_param.alpha;
    a = (float *)ssymm_param.a;
    lda = ssymm_param.lda;
    b = (float *)ssymm_param.b;
    ldb = ssymm_param.ldb;
    beta = ssymm_param.beta;
    c = (float *)ssymm_param.c;
    ldc = ssymm_param.ldc;
    agboff = ssymm_param.agboff;
    
    if((side == MagmaLeft) && (uplo == MagmaLower)){
        //printf("calling AZZAM SYMM m %d   n %d   agboff %d\n",m,n,agboff);
        //Azzam: calling only the Left Lower. need to be developed for all cases 
        magma_int_t nb = 960;
        magmablas_ssymm_mic(side, uplo, 
                         m, n, nb, alpha, a, lda, b, ldb,
                         beta, c, ldc, agboff);
    }else{
        //printf("calling MKL SYMM m %d   n %d \n",m,n);
        ssymm( lapack_const(side), lapack_const(uplo),
              &m, &n, (const float *)&alpha, (float *)a, &lda,
              (float *)b, &ldb, (const float *)&beta,
              (float *)c, &ldc);
    }
}
//=========================================================================================

void call_sgemm(magma_mic_sgemm_param sgemm_param)
{
    magma_trans_t transa, transb;
    magma_int_t m, n, k, lda, ldb, ldc;
    float alpha, beta, *a, *b, *c;
    
    transa = sgemm_param.transa;
    transb = sgemm_param.transb;
    m = sgemm_param.m;
    n = sgemm_param.n;
    k = sgemm_param.k;
    alpha = sgemm_param.alpha;
    a = (float *)sgemm_param.a;
    lda = sgemm_param.lda;
    b = (float *)sgemm_param.b;
    ldb = sgemm_param.ldb;
    beta = sgemm_param.beta;
    c = (float *)sgemm_param.c;
    ldc = sgemm_param.ldc;

    //printf("SGEMM: %c %c %d %d %d %f 0x%lx %d 0x%lx %d %f 0x%lx %d\n", transa, transb, m, n, k, alpha, (off_t)a, lda, (off_t)b, ldb, beta, (off_t)c, ldc);

    sgemm( lapack_const(transa), lapack_const(transb), 
           &m, &n, &k, (const float *)&alpha, (float *)a, &lda, 
           (float *)b, &ldb, (const float *)&beta, 
           (float *)c, &ldc);
}
//=========================================================================================

void call_sgemv(magma_mic_sgemv_param sgemv_param)
{
    magma_trans_t transa;
    magma_int_t m, n, lda, incx, incy;
    float alpha, beta, *a, *x, *y;
    
    transa = sgemv_param.transa;
    m      = sgemv_param.m;
    n      = sgemv_param.n;
    alpha  = sgemv_param.alpha;
    a      = (float *)sgemv_param.a;
    lda    = sgemv_param.lda;
    x      = (float *)sgemv_param.x;
    incx   = sgemv_param.incx;
    beta   = sgemv_param.beta;
    y      = (float *)sgemv_param.y;
    incy   = sgemv_param.incy;
    
    //printf("SGEMV: %c %c %d %d %d %f 0x%lx %d 0x%lx %d %f 0x%lx %d\n", transa, transb, m, n, k, alpha, (off_t)a, lda, (off_t)b, ldb, beta, (off_t)c, ldc);
    
    sgemv( lapack_const(transa), &m, &n, (const float *)&alpha,
           (float *)a, &lda, (float *)x, &incx,
           (const float *)&beta, (float *)y, &incy);
}
//=========================================================================================
void call_ssymv(magma_mic_ssymv_param symv_param)
{
    magma_uplo_t uplo;
    magma_int_t n, lda, incx, incy;
    float alpha, beta, *a, *x, *y;
    
    uplo   = symv_param.uplo;
    n      = symv_param.n;
    alpha  = symv_param.alpha;
    a      = (float *)symv_param.a;
    lda    = symv_param.lda;
    x      = (float *)symv_param.x;
    incx   = symv_param.incx;
    beta   = symv_param.beta;
    y      = (float *)symv_param.y;
    incy   = symv_param.incy;
    
    ssymv( lapack_const(uplo), &n, (const float *)&alpha,
           (float *)a, &lda, (float *)x, &incx,
           (const float *)&beta, (float *)y, &incy);
}
//=========================================================================================
void call_ssymv2(magma_mic_ssymv2_param symv_param)
{
    magma_uplo_t uplo;
    magma_int_t n, lda, incx, incy;
    float alpha, beta, *a, *x, *y;
    
    uplo   = symv_param.uplo;
    n      = symv_param.n;
    alpha  = symv_param.alpha;
    a      = (float *)symv_param.a;
    lda    = symv_param.lda;
    x      = (float *)symv_param.x;
    incx   = symv_param.incx;
    beta   = symv_param.beta;
    y      = (float *)symv_param.y;
    incy   = symv_param.incy;
    
    ssymv( lapack_const(uplo), &n, (const float *)&alpha,
           (float *)a, &lda, (float *)x, &incx,
           (const float *)&beta, (float *)y, &incy);
}
//=========================================================================================
void call_strsm(magma_mic_strsm_param strsm_param)
{
    magma_side_t side;
    magma_uplo_t uplo;
    magma_trans_t transa;
    magma_diag_t diag;
    magma_int_t m, n, lda, ldb;
    float alpha, *a, *b;
    
    side = strsm_param.side;
    uplo = strsm_param.uplo;
    transa = strsm_param.transa;
    diag = strsm_param.diag;
    m = strsm_param.m;
    n = strsm_param.n;
    alpha = strsm_param.alpha;
    a = (float *)strsm_param.a;
    lda = strsm_param.lda;
    b = (float *)strsm_param.b;
    ldb = strsm_param.ldb;
    
    //printf("STRSM: %c %c %c %c %d %d %f 0x%lx %d 0x%lx %d\n", side, uplo, transa, diag, m, n, alpha, (off_t)a, lda, (off_t)b, ldb);
    strsm( lapack_const(side), lapack_const(uplo), lapack_const(transa), lapack_const(diag),
           &m, &n, (const float *)&alpha, (float *)a, &lda, 
           (float *)b, &ldb);
}
//=========================================================================================
void call_strsv(magma_mic_strsv_param strsv_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_diag_t diag;
    magma_int_t n, lda, incx;
    float alpha, *a, *dx;
    
    uplo  = strsv_param.uplo;
    trans = strsv_param.trans;
    diag  = strsv_param.diag;
    n     = strsv_param.n;
    a     = (float *)strsv_param.a;
    lda   = strsv_param.lda;
    dx    = (float *)strsv_param.dx;
    incx  = strsv_param.incx;
    
    strsv( lapack_const(uplo), lapack_const(trans), lapack_const(diag),
           &n, (float *)a, &lda, (float *)dx, &incx);
}
//=========================================================================================

void call_strmv(magma_mic_strmv_param trmv_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_diag_t diag;
    magma_int_t n, lda, incx;
    float alpha, *a, *dx;
    
    uplo  = trmv_param.uplo;
    trans = trmv_param.trans;
    diag  = trmv_param.diag;
    n     = trmv_param.n;
    a     = (float *)trmv_param.a;
    lda   = trmv_param.lda;
    dx    = (float *)trmv_param.dx;
    incx  = trmv_param.incx;
    
    strmv( lapack_const(uplo), lapack_const(trans), lapack_const(diag),
           &n, (float *)a, &lda, (float *)dx, &incx);
}
//=========================================================================================


void call_strmm(magma_mic_strmm_param strmm_param)
{
    magma_side_t side;
    magma_uplo_t uplo;
    magma_trans_t transa;
    magma_diag_t diag;
    magma_int_t m, n, lda, ldb;
    float alpha, *a, *b;

    side = strmm_param.side;
    uplo = strmm_param.uplo;
    transa = strmm_param.transa;
    diag = strmm_param.diag;
    m = strmm_param.m;
    n = strmm_param.n;
    alpha = strmm_param.alpha;
    a = (float *)strmm_param.a;
    lda = strmm_param.lda;
    b = (float *)strmm_param.b;
    ldb = strmm_param.ldb;

    //printf("STRMM: %c %c %c %c %d %d %f 0x%lx %d 0x%lx %d\n", side, uplo, transa, diag, m, n, alpha, (off_t)a, lda, (off_t)b, ldb);
    strmm( lapack_const(side), lapack_const(uplo), lapack_const(transa), lapack_const(diag),
           &m, &n, (const float *)&alpha, (float *)a, &lda, 
           (float *)b, &ldb);
}
//=========================================================================================

//=========================================================================================
//magmablas our ssyrk Azzam
//=========================================================================================
void magmablas_ssyrk_mic0(
    char uplo, char trans, magma_int_t n, magma_int_t k, magma_int_t nb,
    float alpha, float *a, magma_int_t lda, magma_int_t aoff,
    float beta,  float *c, magma_int_t ldc, magma_int_t offset)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    float cbeta  = MAGMA_S_MAKE( beta, 0. );
    float calpha = MAGMA_S_MAKE( alpha, 0. );

    magma_int_t m, ib, ioff;
    magma_int_t blockoffset = offset % nb;

    // loop over all blocks and does A*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        // C[i:n,i] += A[i:n,0] * A[i,0]'
        m = n - i;
        sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &m, &ib, &k,
               (const float *)&calpha, (float *)(dA(i,0)),       &lda,
               (float *)(dA(i,0)),       &lda,
               (const float *)&cbeta, (float *)(dC(ioff,ioff)), &ldc );
        blockoffset = 0;
    }
}
#undef dA
#undef dC
//=========================================================================================



//=========================================================================================
//magmablas our ssyrk version2
//=========================================================================================
void magmablas_ssyrk_mic(
    char uplo, char trans, magma_int_t n, magma_int_t k, magma_int_t nb,
    float alpha, float *a, magma_int_t lda, magma_int_t aoff,
    float beta,  float *c, magma_int_t ldc, magma_int_t offset)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    float cbeta  = MAGMA_S_MAKE( beta, 0. );
    float calpha = MAGMA_S_MAKE( alpha, 0. );

    //if(n <= 3840)
    //        nb = 1200;
    //else 
    //  nb = 960;

    magma_int_t m, ib, ioff;
    magma_int_t blockoffset = offset % nb;

    if(n <= 8640) {
            // loop over all blocks and does A*A'
            // blockoffset is offset within first block; for subsequent blocks it is 0
            for( magma_int_t i = 0; i < n; i += ib ) {
                    ib     = min( nb-blockoffset, n-i );  // block size
                    ioff   = i + offset;                  // global index in parent matrix
                    // C[i:n,i] += A[i:n,0] * A[i,0]'
                    m = n - i;
                    sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &m, &ib, &k,
                                    (const float *)&calpha, (float *)(dA(i,0)),       &lda,
                                    (float *)(dA(i,0)),       &lda,
                                    (const float *)&cbeta, (float *)(dC(ioff,ioff)), &ldc );
                    blockoffset = 0;
            }
    } else {
                    magma_int_t i_960 = n / (2 * 960);
                magma_int_t ngemm = i_960 * 960;;
                magma_int_t nrema = n - ngemm;         
                //gemm for square block
                sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &ngemm, &nrema, &k,
                                (const float *)&calpha, (float *)(dA(nrema, 0)),       &lda,
                                (float *)(dA(0, 0)),       &lda,
                                (const float *)&cbeta, (float *)(dC(nrema, 0)), &ldc );

                //two syrk call
                magmablas_ssyrk_mic('L', 'N', 
                                nrema, k, nb, alpha, a, lda, 0, 
                                beta, c, ldc, 0);

                magmablas_ssyrk_mic('L', 'N', 
                                ngemm, k, nb, alpha, dA(nrema, 0), lda, 0, 
                                beta, dC(nrema, nrema), ldc, 0);

    }
}
#undef dA
#undef dC
//=========================================================================================


//=========================================================================================
void call_ssyrk(magma_mic_ssyrk_param ssyrk_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldc;
    float alpha, beta;
    float *a, *c;

    uplo = ssyrk_param.uplo;
    trans = ssyrk_param.trans;
    n = ssyrk_param.n;
    k = ssyrk_param.k;
    alpha = ssyrk_param.alpha;
    a = (float *)ssyrk_param.a;
    lda = ssyrk_param.lda;
    beta = ssyrk_param.beta;
    c = (float *)ssyrk_param.c;
    ldc = ssyrk_param.ldc;

    //printf("SSYRK: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo, trans, n, k, alpha, (off_t)a, lda, beta, (off_t)c, ldc);

    // call MKL ssyrk
    //ssyrk( lapack_const(uplo), lapack_const(trans), 
    //       &n, &k, &alpha, (float *)a, &lda, 
    //       &beta, (float *)c, &ldc);
   
     
    magma_int_t nb = 960;
    magmablas_ssyrk_mic('L', 'N', 
                     n, k, nb, alpha, a, lda, 0, 
                     beta, c, ldc, 0);
}
//=========================================================================================

//=========================================================================================
void magmablas_ssyrk_mmic(
    char uplo, char trans, magma_int_t n, magma_int_t k,
    float alpha, float *a, magma_int_t lda, magma_int_t aoff,
    float beta,  float *c, magma_int_t ldc, magma_int_t offset,
    magma_int_t nmic, magma_int_t nb, magma_int_t my_id)
{
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    float cbeta  = MAGMA_S_MAKE( beta,  0. );
    float calpha = MAGMA_S_MAKE( alpha, 0. );

    magma_int_t rows, ib, ioff, iblock, idev, di, s;

    // loop over all blocks
    // Faster to have two loops: first does A*B', second does B*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    magma_int_t blockoffset = offset % nb;
    float tt = 0;
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        iblock = (ioff / nb) / nmic;          // local block id
        idev   = (ioff / nb) % nmic;          // device with this block
        di     = iblock*nb + blockoffset;     // local index in parent matrix

        // C[i:n,i] += A[i:n,0] * B[i,0]'
        //printf( "sgemm  n=%4d, ib=%4d, k=%4d, i=%4d\n", n-i, ib, k, i );
        if(idev == my_id) {
            rows = n - i;
            sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &rows, &ib, &k,
                    (const float *)&calpha, (float *)dA(i,0), &lda,
                    (float *)dA(i,0), &lda,
                    (const float *)&cbeta, (float *)dC(ioff,di), &ldc);
        }

        blockoffset = 0;
    }
}
#undef dA
#undef dC
//=========================================================================================
void call_ssyrk_mmic(magma_mic_ssyrk_mmic_param ssyrk_mmic_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldb, ldc, aoff, boff, offset, nb, my_id, nmic;
    float alpha, beta;
    float *a, *b, *c;

    uplo = ssyrk_mmic_param.uplo;
    trans = ssyrk_mmic_param.trans;
    n = ssyrk_mmic_param.n;
    k = ssyrk_mmic_param.k;
    alpha = ssyrk_mmic_param.alpha;
    a = (float *)ssyrk_mmic_param.a;
    lda = ssyrk_mmic_param.lda;
    aoff = ssyrk_mmic_param.aoff;
    beta = ssyrk_mmic_param.beta;
    c = (float *)ssyrk_mmic_param.c;
    ldc = ssyrk_mmic_param.ldc;
    offset = ssyrk_mmic_param.offset;
    nmic = ssyrk_mmic_param.nmic;
    nb = ssyrk_mmic_param.nb;
    my_id = ssyrk_mmic_param.my_id;

    //printf("SSYRK_MMIC: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo, trans, n, k, alpha, (off_t)a, lda, beta, (off_t)c, ldc);

    magmablas_ssyrk_mmic(
        uplo, trans, n, k,
        alpha, a, lda, aoff,
        beta, c, ldc, offset,
        nmic, nb, my_id);
}
//=========================================================================================



//=========================================================================================
//magmablas our ssyr2k Azzam
//=========================================================================================
void magmablas_ssyr2k_mic(
    magma_side_t uplo, magma_side_t trans, magma_int_t n, magma_int_t k, magma_int_t nb,
    float alpha, float *a, magma_int_t lda,
                  float *b, magma_int_t ldb,
    float beta,  float *c, magma_int_t ldc)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda)
    #define dB(i, j) (b + (i) + (j)*ldb)
    #define dC(i, j) (c + (i) + (j)*ldc)

    const float c_one = MAGMA_S_ONE;
    float cbeta  = MAGMA_S_MAKE( beta, 0. );
    float cnjalpha = MAGMA_S_CNJG( alpha );

    magma_int_t m, ib;
    // loop over all blocks and does A*A'
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb, n-i );  // block size
        m = n - i;
        //printf("voici m %d     n %d    k %d \n",m,ib,k);
        // C[i:n,i] += A[i:n,0] * A[i,0]'
        sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &m, &ib, &k,
               (const float *)&alpha, (float *)(dA(i,0)),       &lda,
               (float *)(dB(i,0)),       &ldb,
               (const float *)&cbeta, (float *)(dC(i,i)), &ldc );

        // C[i:n,i] += A[i:n,0] * A[i,0]'
        sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &m, &ib, &k,
               (const float *)&cnjalpha, (float *)(dB(i,0)),       &ldb,
               (float *)(dA(i,0)),       &lda,
               (const float *)&c_one, (float *)(dC(i,i)), &ldc );

    }
}
#undef dA
#undef dB
#undef dC
//=========================================================================================
//=========================================================================================
void call_ssyr2k(magma_mic_ssyr2k_param ssyr2k_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldb, ldc;
    float alpha;
    float beta;
    float *a, *b, *c;

    uplo = ssyr2k_param.uplo;
    trans = ssyr2k_param.trans;
    n = ssyr2k_param.n;
    k = ssyr2k_param.k;
    alpha = ssyr2k_param.alpha;

    a = (float *)ssyr2k_param.a;
    lda = ssyr2k_param.lda;

    b = (float *)ssyr2k_param.b;
    ldb = ssyr2k_param.ldb;

    beta = ssyr2k_param.beta;
    c = (float *)ssyr2k_param.c;
    ldc = ssyr2k_param.ldc;

    //printf("SSYR2K: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo,trans, n,k, alpha, (off_t)a,lda, beta, (off_t)c, ldc);
   
    if(uplo == MagmaUpper) {
           ssyr2k( lapack_const(uplo), lapack_const(trans),
                           &n, &k, (float *)&alpha, (float *)a, &lda,
                           (float *)b, &ldb,
                           &beta, (float *)c, &ldc);
    } else {
           magma_int_t nb = 960;
           magmablas_ssyr2k_mic( 'L', 'N',
                           n, k, nb, alpha, a, lda, 
                           b, ldb, 
                           beta, c, ldc);
    }
}
//=========================================================================================







//=========================================================================================
//magmablas our ssyr2k_mmic Azzam
//=========================================================================================
void magmablas_ssyr2k_mmic(
    char uplo, char trans, magma_int_t n, magma_int_t k,
    float alpha, float *a, magma_int_t lda, magma_int_t aoff,
    float *b, magma_int_t ldb, magma_int_t boff,
    float beta,           float *c, magma_int_t ldc,  magma_int_t offset,
    magma_int_t nmic, magma_int_t nb, magma_int_t my_id)
{
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dB(i, j) (b + (i) + (j)*ldb + (boff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    const float c_one = MAGMA_S_ONE;
    float cbeta = MAGMA_S_MAKE( beta, 0. );


    magma_int_t rows, ib, ioff, iblock, idev, di, s;

    // loop over all blocks
    // Faster to have two loops: first does A*B', second does B*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    magma_int_t blockoffset = offset % nb;
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        iblock = (ioff / nb) / nmic;          // local block id
        idev   = (ioff / nb) % nmic;          // device with this block
        di     = iblock*nb + blockoffset;     // local index in parent matrix

        // C[i:n,i] += A[i:n,0] * B[i,0]'
        //printf( "sgemm  n=%4d, ib=%4d, k=%4d, i=%4d\n", n-i, ib, k, i );
        if(idev == my_id) {
            rows = n - i;
            sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &rows, &ib, &k,
                    (const float *)&alpha, (float *)dA(i,0), &lda,
                    (float *)dB(i,0), &ldb,
                    (const float *)&cbeta, (float *)dC(ioff,di), &ldc);
        }

        blockoffset = 0;
    }

    blockoffset = offset % nb;
    alpha = MAGMA_S_CNJG( alpha );
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        iblock = (ioff / nb) / nmic;          // local block id
        idev   = (ioff / nb) % nmic;          // device with this block
        di     = iblock*nb + blockoffset;     // local index in parent matrix

        // C[i:n,i] += B[i:n,0] * A[i,0]'
        //printf( "sgemm  n=%4d, ib=%4d, k=%4d, i=%4d\n", n-i, ib, k, i );
        if(idev == my_id) {
            rows = n - i;
            sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans), &rows, &ib, &k,
                    (const float *)&alpha, (float *)dB(i,0), &ldb,
                    (float *)dA(i,0), &lda,
                    (const float *)&c_one, (float *)dC(ioff,di), &ldc);
        }
        blockoffset = 0;
    }
}
#undef dA
#undef dB
#undef dC
//=========================================================================================
void call_ssyr2k_mmic(magma_mic_ssyr2k_mmic_param ssyr2k_mmic_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldb, ldc, aoff, boff, offset, nb, my_id, nmic;
    float alpha;
    float beta;
    float *a, *b, *c;

    uplo = ssyr2k_mmic_param.uplo;
    trans = ssyr2k_mmic_param.trans;
    n = ssyr2k_mmic_param.n;
    k = ssyr2k_mmic_param.k;
    alpha = ssyr2k_mmic_param.alpha;
    a = (float *)ssyr2k_mmic_param.a;
    lda = ssyr2k_mmic_param.lda;
    aoff = ssyr2k_mmic_param.aoff;
    b = (float *)ssyr2k_mmic_param.b;
    ldb = ssyr2k_mmic_param.ldb;
    boff = ssyr2k_mmic_param.boff;
    beta = ssyr2k_mmic_param.beta;
    c = (float *)ssyr2k_mmic_param.c;
    ldc = ssyr2k_mmic_param.ldc;
    offset = ssyr2k_mmic_param.offset;
    nmic = ssyr2k_mmic_param.nmic;
    nb = ssyr2k_mmic_param.nb;
    my_id = ssyr2k_mmic_param.my_id;

    //printf("SSYR2K_MMIC: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo, trans, n, k, alpha, (off_t)a, lda, beta, (off_t)c, ldc);

    magmablas_ssyr2k_mmic(
        uplo, trans, n, k,
        alpha, a, lda, aoff,
        b, ldb, boff,
        beta, c, ldc, offset,
        nmic, nb, my_id);
}
//=========================================================================================
void call_slarft(magma_mic_slarft_param slarft_param)
{
    int direct, storev;
    magma_int_t n, k, ldv, ldt; 
    float *v, *tau, *t;
    
    direct = slarft_param.direct;
    storev = slarft_param.storev;
    n = slarft_param.n;
    k = slarft_param.k;
    v = (float *)slarft_param.v;
    ldv = slarft_param.ldv;
    tau = (float *)slarft_param.tau;
    t = (float *)slarft_param.t;
    ldt = slarft_param.ldt;
    
    //printf("SLARFT: %c %c %d %d 0x%lx %d ox%lx 0x%lx %d\n", direct, storev, n, k, (off_t)v, ldv, (off_t)tau, (off_t)t, ldt);
    
    slarft( lapack_const(direct), lapack_const(storev),
            &n, &k, (float *)v, &ldv, 
            (float *)tau, (float *)t, &ldt);
}

//=========================================================================================
void call_slarfb(magma_mic_slarfb_param slarfb_param)
{
    magma_int_t side, trans, direct, storev;
    magma_int_t m, n, k, ldv, ldt, ldc, ldwork;
    float *v, *t, *c, *work;

    side = slarfb_param.side;
    trans = slarfb_param.trans;
    direct = slarfb_param.direct;
    storev = slarfb_param.storev;
    m = slarfb_param.m;
    n = slarfb_param.n;
    k = slarfb_param.k;
    v = (float *)slarfb_param.v;
    ldv = slarfb_param.ldv;
    t = (float *)slarfb_param.t;
    ldt = slarfb_param.ldt;
    c = (float *)slarfb_param.c;
    ldc = slarfb_param.ldc;
    work = (float *)slarfb_param.work;
    ldwork = slarfb_param.ldwork;

    slarfb(lapack_const(side), lapack_const(trans), lapack_const(direct), lapack_const(storev),
                    &m, &n, &k, (float *)v, &ldv, (float *)t, &ldt, 
                    (float *)c, &ldc, (float *)work, &ldwork);
}

//=========================================================================================
//magmablas our SLARFB based on GEMMs Azzam
//=========================================================================================
void call_slarfb_gemm(magma_mic_slarfb_gemm_param slarfb_gemm_param)
{
    magma_int_t side, trans, direct, storev;
    magma_int_t m, n, k, ldv, ldt, ldc, ldwork, ldworkvt;
    float *v, *t, *c, *work, *workvt;
    
    side = slarfb_gemm_param.side;
    trans = slarfb_gemm_param.trans;
    direct = slarfb_gemm_param.direct;
    storev = slarfb_gemm_param.storev;
    m = slarfb_gemm_param.m;
    n = slarfb_gemm_param.n;
    k = slarfb_gemm_param.k;
    v = (float *)slarfb_gemm_param.v;
    ldv = slarfb_gemm_param.ldv;
    t = (float *)slarfb_gemm_param.t;
    ldt = slarfb_gemm_param.ldt;
    c = (float *)slarfb_gemm_param.c;
    ldc = slarfb_gemm_param.ldc;
    work = (float *)slarfb_gemm_param.work;
    ldwork = slarfb_gemm_param.ldwork;
    workvt = (float *)slarfb_gemm_param.workvt;
    ldworkvt = slarfb_gemm_param.ldworkvt;
    
    float c_zero    = MAGMA_S_ZERO;
    float c_one     = MAGMA_S_ONE;
    float c_neg_one = MAGMA_S_NEG_ONE;

    /* Function Body */
    if (m <= 0 || n <= 0) {
        return;
    }

    // opposite of trans
    magma_int_t transt;
    if (trans == MagmaNoTrans)
        transt = MagmaTrans;
    else
        transt = MagmaNoTrans;
    
    // whether T is upper or lower triangular
    magma_int_t uplo;
    if (direct == MagmaForward)
        uplo = MagmaUpper;
    else
        uplo = MagmaLower;
    
    // whether V is stored transposed or not
    magma_int_t notransV, transV;
    if (storev == MagmaColumnwise) {
        notransV = MagmaNoTrans;
        transV   = MagmaTrans;
    }
    else {
        notransV = MagmaTrans;
        transV   = MagmaNoTrans;
    }

    if ( side  == MagmaLeft) {
        // Form H C or H' C
        // Comments assume H C. 
        // When forming H' C, T gets transposed via transt for m>=n or by trans for m<n.
        
        // W = C' V
        sgemm( lapack_const(MagmaTrans), lapack_const(notransV),
                     &n, &k, &m,
                     (const float *)&c_one,  (float *)c, &ldc,
                             (float *)v, &ldv,
                     (const float *)&c_zero, (float *)work, &ldwork);

        if(m<n){
            // W2 = V T 
            sgemm( lapack_const(notransV), lapack_const(trans),
                         &m, &k, &k,
                         (const float *)&c_one,  (float *)v, &ldv,
                                 (float *)t, &ldt,
                         (const float *)&c_zero, (float *)workvt, &ldworkvt);
                        
            // C = C - W2 W' = C - V T V' C = (I - V T V') C = H C 
            sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaTrans),
                         &m, &n, &k,
                         (const float *)&c_neg_one, (float *)workvt, &ldworkvt,
                                    (float *)work, &ldwork,
                         (const float *)&c_one, (float *)c, &ldc);
        }else{
            // W2 = W T' = C' V T'
            sgemm( lapack_const(MagmaNoTrans), lapack_const(transt),
                         &n, &k, &k,
                         (const float *)&c_one,  (float *)work, &ldwork,
                                 (float *)t, &ldt,
                         (const float *)&c_zero, (float *)workvt, &ldworkvt);
                                                 
            // C = C - V W2' = C - V T V' C = (I - V T V') C = H C 
            sgemm( lapack_const(notransV), lapack_const(MagmaTrans),
                         &m, &n, &k,
                         (const float *)&c_neg_one, (float *)v,  &ldv,
                                    (float *)workvt, &ldworkvt,
                         (const float *)&c_one,     (float *)c,       &ldc);
        }
    }
    else {
        // Form C H or C H'
        // Comments assume C H. 
        // When forming C H', T gets transposed via trans.
        
        // W = C V
        sgemm( lapack_const(MagmaNoTrans), lapack_const(notransV),
                     &m, &k, &n,
                     (const float *)&c_one, (float *)c, &ldc,
                             (float *)v, &ldv,
                     (const float *)&c_zero, (float *)work, &ldwork);
        if(m<=n){
            // W2 = W T = C V T
            sgemm( lapack_const(MagmaNoTrans), lapack_const(trans),
                         &m, &k, &k,
                         (const float *)&c_one,  (float *)work, &ldwork,
                                 (float *)t, &ldt,
                         (const float *)&c_zero, (float *)workvt, &ldworkvt);
            
            // C = C - W2 V' = C - C V T V' = C (I - V T V') = C H
            sgemm( lapack_const(MagmaNoTrans), lapack_const(transV),
                         &m, &n, &k,
                         (const float *)&c_neg_one, (float *)work, &ldwork,
                                    (float *)v, &ldv,
                         (const float *)&c_one, (float *)c, &ldc);
        }else{
            // W2 = T V' 
            sgemm( lapack_const(trans), lapack_const(transV),
                         &k, &n, &k,
                         (const float *)&c_one,  (float *)t, &ldt,
                                 (float *)v, &ldv,
                         (const float *)&c_zero, (float *)workvt, &ldworkvt);
            // C = C - W W2 = C - C V T V' = C (I - V T V') = C H
            sgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaNoTrans),
                         &m, &n, &k,
                         (const float *)&c_neg_one, (float *)work, &ldwork,
                                    (float *)workvt, &ldworkvt,
                         (const float *)&c_one, (float *)c, &ldc);

        }
    }
}
//=========================================================================================

void call_slaset(magma_mic_slaset_param slaset_param)
{
    magma_uplo_t uplo;
    magma_int_t m, n, lda;
    float alpha, beta;
    float *dA;

    uplo  = slaset_param.uplo;
    m     = slaset_param.m;
    n     = slaset_param.n;
    alpha = slaset_param.alpha;
    beta  = slaset_param.beta;
    dA    = (float *)slaset_param.dA;
    lda   = slaset_param.lda;

    slaset( lapack_const(uplo),        &m, &n, 
            (const float *)&alpha, (const float *)&beta,
            (float *)dA, &lda);
}
//=========================================================================================


void call_stranspose(magma_mic_stranspose_param stranspose_param)
{
        /******************************************************
         * _mm512_load_pd -- for loading
         * _mm512_i32lo[ext]scatter_pd
         * _mm512_i32loscatter_pd is the simplified version of ...ext...
         * they do the same thing, the latter needs less arguments.
         * **************************************************/
        //        printf("entered stranpose\n");
        //        fflush(stdout);
        /*
           float *odata, *idata;
           int ldo,ldi, m, n;
           int num_threads;
           int tid;
           int index[16], i;
           __M512S _reg;
           __m512i _index;
           float *my_head, *loop_head, *dest_head;
           int num_cols, num_cols_g, col, iter, num_iter, num_rows_g, num_rows;

           odata        =        (float *) stranspose_param.odata;
           ldo                =        stranspose_param.ldo;
           idata        =        (float *) stranspose_param.idata;
           ldi                =        stranspose_param.ldi;
           m                =        stranspose_param.m;
           n                =        stranspose_param.n;

           if (m == 0 || n == 0) {
           return;
           }
           if (m <= n) {
           for (i = 0; i < 16; i++) {
           index[i] = i*SREG_WIDTH*ldo;
           }
        // 62 cores, 4 hardware threads per core. Leave out one core for the OS
        // (62 - 1)*4 = 244

        num_threads = 244;
        num_cols_g =        (n + num_threads - 1)/num_threads; 
        num_iter = (int) m*1.0/SREG_WIDTH;
        _index = _mm512_set_epi32(index[15], index[14], index[13], index[12],
        index[11], index[10], index[9], index[8], index[7], index[6],
        index[5], index[4], index[3], index[2], index[1], index[0]);

#pragma omp parallel num_threads(num_threads) private(tid, col, num_cols, iter,\
my_head, loop_head, dest_head, _reg)
{
tid = omp_get_thread_num();
num_cols = num_cols_g;
if (tid == (num_threads - 1) && (n%num_cols_g != 0)) { 
num_cols = n%num_cols_g;
}
// starting point of my section, constant for each thread
my_head = idata + ldi*(tid*num_cols_g);
for (iter = 0; iter < num_iter; iter++) { // loop inside each column
for (col = 0; col < num_cols; col++) { // loop over each column
if (tid*num_cols_g + col < n) {
        // starting point of the inner loop
        loop_head = my_head + col*ldi + iter*SREG_WIDTH;
        dest_head = odata + (iter*SREG_WIDTH)*ldo + (tid*num_cols_g + col);
        // load 8 real floating point nubmers from address
        // mt into a float64 vector.
        _reg = _MM512_LOAD_PPS( (void const *)loop_head);
        //                                        _mm512_load_pd( (void const *)loop_head);

        // this intrinsic needs int32 indices, but a 512 register
        // contains 16 int32 integers, and there is only one intrinsic
        // to set all 16 positoins of the register. So _index contains
        // 16 int32 integers.
        _MM512_I32LOEXTSCATTER_PPS((void *) dest_head, _index, _reg,
        _MM_DOWNCONV_PS_NONE, 1, _MM_HINT_NONE);
        }
        }
        }        
        }
        } else {
        // init index[], it's for scattering the entries to the dest matrix
        for (i = 0; i < 16; i++) {
        index[i] = i*SREG_WIDTH*ldo;
        }

        // 62 cores, 4 hardware threads per core. Leave out one core for the OS
        // (62 - 1)*4 = 244
        num_threads = 244;
        // number of rows in each group
        num_rows_g = (m/SREG_WIDTH + num_threads - 1)/num_threads*SREG_WIDTH;
        _index = _mm512_set_epi32(index[15], index[14], index[13], index[12],
                        index[11], index[10], index[9], index[8], index[7], index[6],
                        index[5], index[4], index[3], index[2], index[1], index[0]);

#pragma omp parallel num_threads(num_threads) private(tid, col, num_rows, iter,\
                loop_head, dest_head, _reg, num_iter)
        {
                tid = omp_get_thread_num();
                // compute the number of rows I need to work on
                num_rows = num_rows_g;
                if (tid == (num_threads - 1) && (m%num_rows_g != 0)) {
                        num_rows = m%num_rows_g;
                }
                num_iter = (int) num_rows*1.0/SREG_WIDTH;

                for (iter = 0; iter < num_iter; iter++) {
                        if (tid*num_rows_g + iter*SREG_WIDTH < m) {
                                for (col = 0; col < n; col++) {
                                        loop_head = idata + (tid*num_rows_g + iter*SREG_WIDTH) + ldi*col;
                                        dest_head = odata + ldo*(tid*num_rows_g + iter*SREG_WIDTH) + col;

                                        _reg = _MM512_LOAD_PPS((void const *) loop_head);
                                        _MM512_I32LOEXTSCATTER_PPS((void *) dest_head, _index, _reg,
                                                        _MM_DOWNCONV_PS_NONE, 1, _MM_HINT_NONE);
                                }
                        }
                }

        }
}
*/
return;
}

//=========================================================================================
void call_stranspose_mkl(magma_mic_stranspose_param stranspose_param)
{
    float *odata, *idata;
    int ldo,ldi, m, n;
    int num_threads;
    int tid;
    int index[16], i;
    //  _M512 _reg;
    //  _M512I _index;
    float *my_head, *loop_head, *dest_head;
    int num_cols, num_cols_g, col, iter, num_iter;
    
    odata =(float *) stranspose_param.odata;
    ldo   =stranspose_param.ldo;
    idata =(float *) stranspose_param.idata;
    ldi   = stranspose_param.ldi;
    m     = stranspose_param.m;
    n     = stranspose_param.n;
    float alpha = MAGMA_S_MAKE(  1.0, 0.0 );
    
    mkl_somatcopy('C', 'T', m, n, *((float *)(&alpha)), (float *)idata, ldi, 
                    (float *)odata, ldo);
}
//=========================================================================================

void call_slaswp(magma_mic_slaswp_param slaswp_param)
{
    magma_int_t n, k1, k2, lda, incx;
    float *d_A;
    magma_int_t *d_ipiv;
                     
    n      = slaswp_param.n;
    d_A    = (float *) slaswp_param.d_A;
    lda    = slaswp_param.lda;
    k1     = slaswp_param.k1;
    k2     = slaswp_param.k2;
    d_ipiv = (magma_int_t *) slaswp_param.d_ipiv;
    incx   = slaswp_param.incx;
    
    //printf("n: %d lda: %d k1: %d k2: %d incx: %d\n", n, lda, k1, k2, incx);
    slaswp(&n, (float *)d_A, &lda, &k1, &k2, (magma_int_t *)d_ipiv, &incx);
}
//=========================================================================================

void call_sswap(magma_mic_sswap_param sswap_param)
{
    magma_int_t n, lda1, lda2;
    float tmp, *dA1T, *dA2T;
                     
    n       = sswap_param.n;
    dA1T    = (float *) sswap_param.dA1T;
    lda1    = sswap_param.lda1;
    dA2T    = (float *) sswap_param.dA2T;
    lda2    = sswap_param.lda2;
    
    sswap(&n, (float *)dA1T, &lda1, (float *)dA2T, &lda2);
}
//=========================================================================================
void call_strsmgemm(magma_mic_strsmgemm_param strsmgemm_param)
{
    float gtrsmflops, ggemmflops, mic_trsmtime, mic_gemmtime;

    magma_mic_strsm_param trsm_param;
    magma_mic_sgemm_param gemm_param;

    magma_side_t side;
    magma_uplo_t uplo;
    magma_trans_t ttransa;
    magma_diag_t diag;
    magma_int_t tm, tn, tlda, tldb;
    float talpha, *ta, *tb;

    trsm_param = strsmgemm_param.trsm_param;

    side = trsm_param.side;
    uplo = trsm_param.uplo;
    ttransa = trsm_param.transa;
    diag = trsm_param.diag;
    tm = trsm_param.m;
    tn = trsm_param.n;
    talpha = trsm_param.alpha;
    ta = (float *)trsm_param.a;
    tlda = trsm_param.lda;
    tb = (float *)trsm_param.b;
    tldb = trsm_param.ldb;

    mic_trsmtime = dsecnd();

    strsm( lapack_const(side), lapack_const(uplo), lapack_const(ttransa), lapack_const(diag),
           &tm, &tn, (const float *)&talpha, (float *)ta, &tlda, 
           (float *)tb, &tldb);

    mic_trsmtime = dsecnd() - mic_trsmtime;

    gtrsmflops = 0;
    if(lapack_const(side)[0] == 'L')
        gtrsmflops = MICFLOPS_TRSM_L((float)tm, (float)tn) * 1e-9;
    else if(lapack_const(side)[0] == 'R')
        gtrsmflops = MICFLOPS_TRSM_R((float)tm, (float)tn) * 1e-9;

    magma_trans_t transa, transb;
    magma_int_t m, n, k, lda, ldb, ldc;
    float alpha, beta, *a, *b, *c;

    gemm_param = strsmgemm_param.gemm_param;


    transa = gemm_param.transa;
    transb = gemm_param.transb;
    m = gemm_param.m;
    n = gemm_param.n;
    k = gemm_param.k;
    alpha = gemm_param.alpha;
    a = (float *)gemm_param.a;
    lda = gemm_param.lda;
    b = (float *)gemm_param.b;
    ldb = gemm_param.ldb;
    beta = gemm_param.beta;
    c = (float *)gemm_param.c;
    ldc = gemm_param.ldc;

    //printf("SGEMM: %c %c %d %d %d %f 0x%lx %d 0x%lx %d %f 0x%lx %d\n", transa, transb, m, n, k, alpha, (off_t)a, lda, (off_t)b, ldb, beta, (off_t)c, ldc);

    mic_gemmtime = dsecnd();

    sgemm( lapack_const(transa), lapack_const(transb), 
           &m, &n, &k, (const float *)&alpha, (float *)a, &lda, 
           (float *)b, &ldb, (const float *)&beta, 
           (float *)c, &ldc);

    mic_gemmtime = dsecnd() - mic_gemmtime;

    ggemmflops = MICFLOPS_GEMM((float)m, (float)n, (float)k) * 1e-9;

    //printf("DTRSMGEMM: %8.4lf( %8.4lf )\t %8.4lf( %8.4lf )\n", gtrsmflops / mic_trsmtime, mic_trsmtime * 1000, ggemmflops / mic_gemmtime, mic_gemmtime * 1000); 
}
//=========================================================================================

