/*
 *   -- micMAGMA (version 0.1) --
 *      Univ. of Tennessee, Knoxville
 *      Univ. of California, Berkeley
 *      Univ. of Colorado, Denver
 *      April 2012
 *
 * @author Azzam Haidar 
 * @author Khairul Kabir 
 *
 * @precisions normal z -> s d c
 */

#include <mkl.h>

#include <mkl_trans.h>
#include <scif.h>
#include <fcntl.h>
#include <offload.h>

//#ifdef __MIC__
//#define _M512_TYPED
//#include <lmmintrin.h>
//#endif
#include <immintrin.h>


#include "magma.h"
//#include "magma_z_bindings.h"
#include "magma_z_server.h"



void call_zcopy(magma_mic_zcopy_param zcopy_param)
{
    magma_int_t n, incx, incy;
    magmaDoubleComplex  *x, *y;
    
    n      = zcopy_param.n;
    x      = (magmaDoubleComplex *)zcopy_param.x;
    incx   = zcopy_param.incx;
    y      = (magmaDoubleComplex *)zcopy_param.y;
    incy   = zcopy_param.incy;
    


    /* sequntial version I think the compiler is optimizing it */
    magma_int_t ix=0, iy=0, ii, chunk=1, nthreads, tid;
    if(incx<0) ix = (-n+1)*incx;
    if(incy<0) iy = (-n+1)*incy;
    for(magma_int_t i=0; i<n; i++){
           y[iy] = x[ix];
           ix += incx;
           iy += incy;
    }

    /* parallel version, where I use openmp to parallelize the copy */
 /* not tested if incx or incy <0
    #pragma omp parallel
    {
        magma_int_t ix=0, iy=0, ii, nthreads, tid;
        tid      = omp_get_thread_num();
        nthreads = omp_get_num_threads();
        if(incx<0) ix = (-n+1)*incx;
        if(incy<0) iy = (-n+1)*incy;
        ix = ix + tid*incx;
        iy = iy + tid*incy;
        for(ii=0; ii<n; ii+=nthreads){
            if((ii+tid)<n){
                y[iy] = x[ix];
                ix = ix + incx * nthreads;
                iy = iy + incy * nthreads;
            }
        }
    }
*/
    // mkl_zcopy is not optimized yet and thus giving bad perf. it is better to use the handcode as the compiler is intelligent and optimize the code better. Azzam
    //zcopy( &n, (MKL_Complex16 *)x, &incx, (MKL_Complex16 *)y, &incy);
}


//=========================================================================================
//magmablas our zhemm only left implemented here Azzam
//=========================================================================================
void magmablas_zhemm_mic(
    magma_side_t side, magma_side_t uplo, magma_int_t n, magma_int_t k, magma_int_t nb,
    magmaDoubleComplex alpha, magmaDoubleComplex *a, magma_int_t lda,
                              magmaDoubleComplex *b, magma_int_t ldb,
    magmaDoubleComplex beta,  magmaDoubleComplex *c, magma_int_t ldc, magma_int_t agboff)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda)
    #define dB(i, j) (b + (i) + (j)*ldb)
    #define dC(i, j) (c + (i) + (j)*ldc)

    const magmaDoubleComplex c_one = MAGMA_Z_ONE;
    magmaDoubleComplex newbeta=beta;
    magma_int_t m, ib, ioff,blockoffset;
    // loop over all blocks and does A*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    blockoffset = agboff % nb;
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        /*
        // Symmetrize the upper part of this block
        for(magma_int_t col=0; col<ib; col++){
            for(magma_int_t row=0; row<col; row++){
                *dA(i+row, i+col) =  MAGMA_Z_CNJG( *(dA(i+col, i+row)) );
            }
        }
        */
        // C[i:n,0] += alpha * A[i:n,i] * B[i,0]
        /*
        m = n - i -ib;
        zhemm( lapack_const(side), lapack_const(uplo),
              &ib, &k, (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)(dA(i, i)), &lda,
              (MKL_Complex16 *)(dB(i,0)), &ldb, (const MKL_Complex16 *)&newbeta,
              (MKL_Complex16 *)(dC(i,0)), &ldc);

        zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaNoTrans), &m, &k, &ib,
               (const MKL_Complex16 *)&alpha,   (MKL_Complex16 *)(dA(ioff+ib, ioff)), &lda,
                                                (MKL_Complex16 *)(dB(i,0)),        &ldb,
               (const MKL_Complex16 *)&newbeta, (MKL_Complex16 *)(dC(i+ib,0)),        &ldc );
        */
        m = n - i ;
        zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaNoTrans), &m, &k, &ib,
               (const MKL_Complex16 *)&alpha,   (MKL_Complex16 *)(dA(i, i)), &lda,
                                                (MKL_Complex16 *)(dB(i,0)),        &ldb,
               (const MKL_Complex16 *)&newbeta, (MKL_Complex16 *)(dC(i,0)),        &ldc );
        

        zgemm( lapack_const(MagmaConjTrans), lapack_const(MagmaNoTrans), &i, &k, &ib,
               (const MKL_Complex16 *)&alpha,   (MKL_Complex16 *)(dA(i, 0)), &lda,
                                                (MKL_Complex16 *)(dB(i,0)),     &ldb,
               (const MKL_Complex16 *)&newbeta, (MKL_Complex16 *)(dC(0,0)),     &ldc );

        newbeta = c_one;
        blockoffset = 0;
    }
}
#undef dA
#undef dB
#undef dC
//=========================================================================================

void call_zhemm(magma_mic_zhemm_param zhemm_param)
{
    magma_side_t side;
    magma_uplo_t uplo;
    magma_int_t m, n, lda, ldb, ldc, agboff;
    magmaDoubleComplex alpha, beta, *a, *b, *c;

    side = zhemm_param.side;
    uplo = zhemm_param.uplo;
    m = zhemm_param.m;
    n = zhemm_param.n;
    alpha = zhemm_param.alpha;
    a = (magmaDoubleComplex *)zhemm_param.a;
    lda = zhemm_param.lda;
    b = (magmaDoubleComplex *)zhemm_param.b;
    ldb = zhemm_param.ldb;
    beta = zhemm_param.beta;
    c = (magmaDoubleComplex *)zhemm_param.c;
    ldc = zhemm_param.ldc;
    agboff = zhemm_param.agboff;
    
    if((side == MagmaLeft) && (uplo == MagmaLower)){
        //printf("calling AZZAM SYMM m %d   n %d   agboff %d\n",m,n,agboff);
        //Azzam: calling only the Left Lower. need to be developed for all cases 
        magma_int_t nb = 960;
        magmablas_zhemm_mic(side, uplo, 
                         m, n, nb, alpha, a, lda, b, ldb,
                         beta, c, ldc, agboff);
    }else{
        //printf("calling MKL SYMM m %d   n %d \n",m,n);
        zhemm( lapack_const(side), lapack_const(uplo),
              &m, &n, (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)a, &lda,
              (MKL_Complex16 *)b, &ldb, (const MKL_Complex16 *)&beta,
              (MKL_Complex16 *)c, &ldc);
    }
}
//=========================================================================================

void call_zgemm(magma_mic_zgemm_param zgemm_param)
{
    magma_trans_t transa, transb;
    magma_int_t m, n, k, lda, ldb, ldc;
    magmaDoubleComplex alpha, beta, *a, *b, *c;
    
    transa = zgemm_param.transa;
    transb = zgemm_param.transb;
    m = zgemm_param.m;
    n = zgemm_param.n;
    k = zgemm_param.k;
    alpha = zgemm_param.alpha;
    a = (magmaDoubleComplex *)zgemm_param.a;
    lda = zgemm_param.lda;
    b = (magmaDoubleComplex *)zgemm_param.b;
    ldb = zgemm_param.ldb;
    beta = zgemm_param.beta;
    c = (magmaDoubleComplex *)zgemm_param.c;
    ldc = zgemm_param.ldc;

    //printf("ZGEMM: %c %c %d %d %d %f 0x%lx %d 0x%lx %d %f 0x%lx %d\n", transa, transb, m, n, k, alpha, (off_t)a, lda, (off_t)b, ldb, beta, (off_t)c, ldc);

    zgemm( lapack_const(transa), lapack_const(transb), 
           &m, &n, &k, (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)a, &lda, 
           (MKL_Complex16 *)b, &ldb, (const MKL_Complex16 *)&beta, 
           (MKL_Complex16 *)c, &ldc);
}
//=========================================================================================

void call_zgemv(magma_mic_zgemv_param zgemv_param)
{
    magma_trans_t transa;
    magma_int_t m, n, lda, incx, incy;
    magmaDoubleComplex alpha, beta, *a, *x, *y;
    
    transa = zgemv_param.transa;
    m      = zgemv_param.m;
    n      = zgemv_param.n;
    alpha  = zgemv_param.alpha;
    a      = (magmaDoubleComplex *)zgemv_param.a;
    lda    = zgemv_param.lda;
    x      = (magmaDoubleComplex *)zgemv_param.x;
    incx   = zgemv_param.incx;
    beta   = zgemv_param.beta;
    y      = (magmaDoubleComplex *)zgemv_param.y;
    incy   = zgemv_param.incy;
    
    //printf("ZGEMV: %c %c %d %d %d %f 0x%lx %d 0x%lx %d %f 0x%lx %d\n", transa, transb, m, n, k, alpha, (off_t)a, lda, (off_t)b, ldb, beta, (off_t)c, ldc);
    
    zgemv( lapack_const(transa), &m, &n, (const MKL_Complex16 *)&alpha,
           (MKL_Complex16 *)a, &lda, (MKL_Complex16 *)x, &incx,
           (const MKL_Complex16 *)&beta, (MKL_Complex16 *)y, &incy);
}
//=========================================================================================
void call_zhemv(magma_mic_zhemv_param hemv_param)
{
    magma_uplo_t uplo;
    magma_int_t n, lda, incx, incy;
    magmaDoubleComplex alpha, beta, *a, *x, *y;
    
    uplo   = hemv_param.uplo;
    n      = hemv_param.n;
    alpha  = hemv_param.alpha;
    a      = (magmaDoubleComplex *)hemv_param.a;
    lda    = hemv_param.lda;
    x      = (magmaDoubleComplex *)hemv_param.x;
    incx   = hemv_param.incx;
    beta   = hemv_param.beta;
    y      = (magmaDoubleComplex *)hemv_param.y;
    incy   = hemv_param.incy;
    
    zhemv( lapack_const(uplo), &n, (const MKL_Complex16 *)&alpha,
           (MKL_Complex16 *)a, &lda, (MKL_Complex16 *)x, &incx,
           (const MKL_Complex16 *)&beta, (MKL_Complex16 *)y, &incy);
}
//=========================================================================================
void call_zsymv2(magma_mic_zsymv2_param symv_param)
{
    magma_uplo_t uplo;
    magma_int_t n, lda, incx, incy;
    magmaDoubleComplex alpha, beta, *a, *x, *y;
    
    uplo   = symv_param.uplo;
    n      = symv_param.n;
    alpha  = symv_param.alpha;
    a      = (magmaDoubleComplex *)symv_param.a;
    lda    = symv_param.lda;
    x      = (magmaDoubleComplex *)symv_param.x;
    incx   = symv_param.incx;
    beta   = symv_param.beta;
    y      = (magmaDoubleComplex *)symv_param.y;
    incy   = symv_param.incy;
    
    zsymv( lapack_const(uplo), &n, (const MKL_Complex16 *)&alpha,
           (MKL_Complex16 *)a, &lda, (MKL_Complex16 *)x, &incx,
           (const MKL_Complex16 *)&beta, (MKL_Complex16 *)y, &incy);
}
//=========================================================================================
void call_ztrsm(magma_mic_ztrsm_param ztrsm_param)
{
    magma_side_t side;
    magma_uplo_t uplo;
    magma_trans_t transa;
    magma_diag_t diag;
    magma_int_t m, n, lda, ldb;
    magmaDoubleComplex alpha, *a, *b;
    
    side = ztrsm_param.side;
    uplo = ztrsm_param.uplo;
    transa = ztrsm_param.transa;
    diag = ztrsm_param.diag;
    m = ztrsm_param.m;
    n = ztrsm_param.n;
    alpha = ztrsm_param.alpha;
    a = (magmaDoubleComplex *)ztrsm_param.a;
    lda = ztrsm_param.lda;
    b = (magmaDoubleComplex *)ztrsm_param.b;
    ldb = ztrsm_param.ldb;
    
    //printf("ZTRSM: %c %c %c %c %d %d %f 0x%lx %d 0x%lx %d\n", side, uplo, transa, diag, m, n, alpha, (off_t)a, lda, (off_t)b, ldb);
    ztrsm( lapack_const(side), lapack_const(uplo), lapack_const(transa), lapack_const(diag),
           &m, &n, (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)a, &lda, 
           (MKL_Complex16 *)b, &ldb);
}
//=========================================================================================
void call_ztrsv(magma_mic_ztrsv_param ztrsv_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_diag_t diag;
    magma_int_t n, lda, incx;
    magmaDoubleComplex alpha, *a, *dx;
    
    uplo  = ztrsv_param.uplo;
    trans = ztrsv_param.trans;
    diag  = ztrsv_param.diag;
    n     = ztrsv_param.n;
    a     = (magmaDoubleComplex *)ztrsv_param.a;
    lda   = ztrsv_param.lda;
    dx    = (magmaDoubleComplex *)ztrsv_param.dx;
    incx  = ztrsv_param.incx;
    
    ztrsv( lapack_const(uplo), lapack_const(trans), lapack_const(diag),
           &n, (MKL_Complex16 *)a, &lda, (MKL_Complex16 *)dx, &incx);
}
//=========================================================================================

void call_ztrmv(magma_mic_ztrmv_param trmv_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_diag_t diag;
    magma_int_t n, lda, incx;
    magmaDoubleComplex alpha, *a, *dx;
    
    uplo  = trmv_param.uplo;
    trans = trmv_param.trans;
    diag  = trmv_param.diag;
    n     = trmv_param.n;
    a     = (magmaDoubleComplex *)trmv_param.a;
    lda   = trmv_param.lda;
    dx    = (magmaDoubleComplex *)trmv_param.dx;
    incx  = trmv_param.incx;
    
    ztrmv( lapack_const(uplo), lapack_const(trans), lapack_const(diag),
           &n, (MKL_Complex16 *)a, &lda, (MKL_Complex16 *)dx, &incx);
}
//=========================================================================================


void call_ztrmm(magma_mic_ztrmm_param ztrmm_param)
{
    magma_side_t side;
    magma_uplo_t uplo;
    magma_trans_t transa;
    magma_diag_t diag;
    magma_int_t m, n, lda, ldb;
    magmaDoubleComplex alpha, *a, *b;

    side = ztrmm_param.side;
    uplo = ztrmm_param.uplo;
    transa = ztrmm_param.transa;
    diag = ztrmm_param.diag;
    m = ztrmm_param.m;
    n = ztrmm_param.n;
    alpha = ztrmm_param.alpha;
    a = (magmaDoubleComplex *)ztrmm_param.a;
    lda = ztrmm_param.lda;
    b = (magmaDoubleComplex *)ztrmm_param.b;
    ldb = ztrmm_param.ldb;

    //printf("ZTRMM: %c %c %c %c %d %d %f 0x%lx %d 0x%lx %d\n", side, uplo, transa, diag, m, n, alpha, (off_t)a, lda, (off_t)b, ldb);
    ztrmm( lapack_const(side), lapack_const(uplo), lapack_const(transa), lapack_const(diag),
           &m, &n, (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)a, &lda, 
           (MKL_Complex16 *)b, &ldb);
}
//=========================================================================================

//=========================================================================================
//magmablas our zherk Azzam
//=========================================================================================
void magmablas_zherk_mic0(
    char uplo, char trans, magma_int_t n, magma_int_t k, magma_int_t nb,
    double alpha, magmaDoubleComplex *a, magma_int_t lda, magma_int_t aoff,
    double beta,  magmaDoubleComplex *c, magma_int_t ldc, magma_int_t offset)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    magmaDoubleComplex cbeta  = MAGMA_Z_MAKE( beta, 0. );
    magmaDoubleComplex calpha = MAGMA_Z_MAKE( alpha, 0. );

    magma_int_t m, ib, ioff;
    magma_int_t blockoffset = offset % nb;

    // loop over all blocks and does A*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        // C[i:n,i] += A[i:n,0] * A[i,0]'
        m = n - i;
        zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &m, &ib, &k,
               (const MKL_Complex16 *)&calpha, (MKL_Complex16 *)(dA(i,0)),       &lda,
               (MKL_Complex16 *)(dA(i,0)),       &lda,
               (const MKL_Complex16 *)&cbeta, (MKL_Complex16 *)(dC(ioff,ioff)), &ldc );
        blockoffset = 0;
    }
}
#undef dA
#undef dC
//=========================================================================================



//=========================================================================================
//magmablas our zherk version2
//=========================================================================================
void magmablas_zherk_mic(
    char uplo, char trans, magma_int_t n, magma_int_t k, magma_int_t nb,
    double alpha, magmaDoubleComplex *a, magma_int_t lda, magma_int_t aoff,
    double beta,  magmaDoubleComplex *c, magma_int_t ldc, magma_int_t offset)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    magmaDoubleComplex cbeta  = MAGMA_Z_MAKE( beta, 0. );
    magmaDoubleComplex calpha = MAGMA_Z_MAKE( alpha, 0. );

    //if(n <= 3840)
    //        nb = 1200;
    //else 
    //  nb = 960;

    magma_int_t m, ib, ioff;
    magma_int_t blockoffset = offset % nb;

    if(n <= 8640) {
            // loop over all blocks and does A*A'
            // blockoffset is offset within first block; for subsequent blocks it is 0
            for( magma_int_t i = 0; i < n; i += ib ) {
                    ib     = min( nb-blockoffset, n-i );  // block size
                    ioff   = i + offset;                  // global index in parent matrix
                    // C[i:n,i] += A[i:n,0] * A[i,0]'
                    m = n - i;
                    zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &m, &ib, &k,
                                    (const MKL_Complex16 *)&calpha, (MKL_Complex16 *)(dA(i,0)),       &lda,
                                    (MKL_Complex16 *)(dA(i,0)),       &lda,
                                    (const MKL_Complex16 *)&cbeta, (MKL_Complex16 *)(dC(ioff,ioff)), &ldc );
                    blockoffset = 0;
            }
    } else {
                    magma_int_t i_960 = n / (2 * 960);
                magma_int_t ngemm = i_960 * 960;;
                magma_int_t nrema = n - ngemm;         
                //gemm for square block
                zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &ngemm, &nrema, &k,
                                (const MKL_Complex16 *)&calpha, (MKL_Complex16 *)(dA(nrema, 0)),       &lda,
                                (MKL_Complex16 *)(dA(0, 0)),       &lda,
                                (const MKL_Complex16 *)&cbeta, (MKL_Complex16 *)(dC(nrema, 0)), &ldc );

                //two syrk call
                magmablas_zherk_mic('L', 'N', 
                                nrema, k, nb, alpha, a, lda, 0, 
                                beta, c, ldc, 0);

                magmablas_zherk_mic('L', 'N', 
                                ngemm, k, nb, alpha, dA(nrema, 0), lda, 0, 
                                beta, dC(nrema, nrema), ldc, 0);

    }
}
#undef dA
#undef dC
//=========================================================================================


//=========================================================================================
void call_zherk(magma_mic_zherk_param zherk_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldc;
    double alpha, beta;
    magmaDoubleComplex *a, *c;

    uplo = zherk_param.uplo;
    trans = zherk_param.trans;
    n = zherk_param.n;
    k = zherk_param.k;
    alpha = zherk_param.alpha;
    a = (magmaDoubleComplex *)zherk_param.a;
    lda = zherk_param.lda;
    beta = zherk_param.beta;
    c = (magmaDoubleComplex *)zherk_param.c;
    ldc = zherk_param.ldc;

    //printf("ZHERK: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo, trans, n, k, alpha, (off_t)a, lda, beta, (off_t)c, ldc);

    // call MKL zherk
    //zherk( lapack_const(uplo), lapack_const(trans), 
    //       &n, &k, &alpha, (MKL_Complex16 *)a, &lda, 
    //       &beta, (MKL_Complex16 *)c, &ldc);
   
     
    magma_int_t nb = 960;
    magmablas_zherk_mic('L', 'N', 
                     n, k, nb, alpha, a, lda, 0, 
                     beta, c, ldc, 0);
}
//=========================================================================================

//=========================================================================================
void magmablas_zherk_mmic(
    char uplo, char trans, magma_int_t n, magma_int_t k,
    double alpha, magmaDoubleComplex *a, magma_int_t lda, magma_int_t aoff,
    double beta,  magmaDoubleComplex *c, magma_int_t ldc, magma_int_t offset,
    magma_int_t nmic, magma_int_t nb, magma_int_t my_id)
{
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    magmaDoubleComplex cbeta  = MAGMA_Z_MAKE( beta,  0. );
    magmaDoubleComplex calpha = MAGMA_Z_MAKE( alpha, 0. );

    magma_int_t rows, ib, ioff, iblock, idev, di, s;

    // loop over all blocks
    // Faster to have two loops: first does A*B', second does B*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    magma_int_t blockoffset = offset % nb;
    double tt = 0;
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        iblock = (ioff / nb) / nmic;          // local block id
        idev   = (ioff / nb) % nmic;          // device with this block
        di     = iblock*nb + blockoffset;     // local index in parent matrix

        // C[i:n,i] += A[i:n,0] * B[i,0]'
        //printf( "zgemm  n=%4d, ib=%4d, k=%4d, i=%4d\n", n-i, ib, k, i );
        if(idev == my_id) {
            rows = n - i;
            zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &rows, &ib, &k,
                    (const MKL_Complex16 *)&calpha, (MKL_Complex16 *)dA(i,0), &lda,
                    (MKL_Complex16 *)dA(i,0), &lda,
                    (const MKL_Complex16 *)&cbeta, (MKL_Complex16 *)dC(ioff,di), &ldc);
        }

        blockoffset = 0;
    }
}
#undef dA
#undef dC
//=========================================================================================
void call_zherk_mmic(magma_mic_zherk_mmic_param zherk_mmic_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldb, ldc, aoff, boff, offset, nb, my_id, nmic;
    double alpha, beta;
    magmaDoubleComplex *a, *b, *c;

    uplo = zherk_mmic_param.uplo;
    trans = zherk_mmic_param.trans;
    n = zherk_mmic_param.n;
    k = zherk_mmic_param.k;
    alpha = zherk_mmic_param.alpha;
    a = (magmaDoubleComplex *)zherk_mmic_param.a;
    lda = zherk_mmic_param.lda;
    aoff = zherk_mmic_param.aoff;
    beta = zherk_mmic_param.beta;
    c = (magmaDoubleComplex *)zherk_mmic_param.c;
    ldc = zherk_mmic_param.ldc;
    offset = zherk_mmic_param.offset;
    nmic = zherk_mmic_param.nmic;
    nb = zherk_mmic_param.nb;
    my_id = zherk_mmic_param.my_id;

    //printf("ZHERK_MMIC: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo, trans, n, k, alpha, (off_t)a, lda, beta, (off_t)c, ldc);

    magmablas_zherk_mmic(
        uplo, trans, n, k,
        alpha, a, lda, aoff,
        beta, c, ldc, offset,
        nmic, nb, my_id);
}
//=========================================================================================



//=========================================================================================
//magmablas our zher2k Azzam
//=========================================================================================
void magmablas_zher2k_mic(
    magma_side_t uplo, magma_side_t trans, magma_int_t n, magma_int_t k, magma_int_t nb,
    magmaDoubleComplex alpha, magmaDoubleComplex *a, magma_int_t lda,
                  magmaDoubleComplex *b, magma_int_t ldb,
    double beta,  magmaDoubleComplex *c, magma_int_t ldc)
{
    #define min(a, b) ((a) < (b) ? (a) : (b))
    #define dA(i, j) (a + (i) + (j)*lda)
    #define dB(i, j) (b + (i) + (j)*ldb)
    #define dC(i, j) (c + (i) + (j)*ldc)

    const magmaDoubleComplex c_one = MAGMA_Z_ONE;
    magmaDoubleComplex cbeta  = MAGMA_Z_MAKE( beta, 0. );
    magmaDoubleComplex cnjalpha = MAGMA_Z_CNJG( alpha );

    magma_int_t m, ib;
    // loop over all blocks and does A*A'
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb, n-i );  // block size
        m = n - i;
        //printf("voici m %d     n %d    k %d \n",m,ib,k);
        // C[i:n,i] += A[i:n,0] * A[i,0]'
        zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &m, &ib, &k,
               (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)(dA(i,0)),       &lda,
               (MKL_Complex16 *)(dB(i,0)),       &ldb,
               (const MKL_Complex16 *)&cbeta, (MKL_Complex16 *)(dC(i,i)), &ldc );

        // C[i:n,i] += A[i:n,0] * A[i,0]'
        zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &m, &ib, &k,
               (const MKL_Complex16 *)&cnjalpha, (MKL_Complex16 *)(dB(i,0)),       &ldb,
               (MKL_Complex16 *)(dA(i,0)),       &lda,
               (const MKL_Complex16 *)&c_one, (MKL_Complex16 *)(dC(i,i)), &ldc );

    }
}
#undef dA
#undef dB
#undef dC
//=========================================================================================
//=========================================================================================
void call_zher2k(magma_mic_zher2k_param zher2k_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldb, ldc;
    magmaDoubleComplex alpha;
    double beta;
    magmaDoubleComplex *a, *b, *c;

    uplo = zher2k_param.uplo;
    trans = zher2k_param.trans;
    n = zher2k_param.n;
    k = zher2k_param.k;
    alpha = zher2k_param.alpha;

    a = (magmaDoubleComplex *)zher2k_param.a;
    lda = zher2k_param.lda;

    b = (magmaDoubleComplex *)zher2k_param.b;
    ldb = zher2k_param.ldb;

    beta = zher2k_param.beta;
    c = (magmaDoubleComplex *)zher2k_param.c;
    ldc = zher2k_param.ldc;

    //printf("ZHER2K: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo,trans, n,k, alpha, (off_t)a,lda, beta, (off_t)c, ldc);
   
    if(uplo == MagmaUpper) {
           zher2k( lapack_const(uplo), lapack_const(trans),
                           &n, &k, (MKL_Complex16 *)&alpha, (MKL_Complex16 *)a, &lda,
                           (MKL_Complex16 *)b, &ldb,
                           &beta, (MKL_Complex16 *)c, &ldc);
    } else {
           magma_int_t nb = 960;
           magmablas_zher2k_mic( 'L', 'N',
                           n, k, nb, alpha, a, lda, 
                           b, ldb, 
                           beta, c, ldc);
    }
}
//=========================================================================================







//=========================================================================================
//magmablas our zher2k_mmic Azzam
//=========================================================================================
void magmablas_zher2k_mmic(
    char uplo, char trans, magma_int_t n, magma_int_t k,
    magmaDoubleComplex alpha, magmaDoubleComplex *a, magma_int_t lda, magma_int_t aoff,
    magmaDoubleComplex *b, magma_int_t ldb, magma_int_t boff,
    double beta,           magmaDoubleComplex *c, magma_int_t ldc,  magma_int_t offset,
    magma_int_t nmic, magma_int_t nb, magma_int_t my_id)
{
    #define dA(i, j) (a + (i) + (j)*lda + (aoff) )
    #define dB(i, j) (b + (i) + (j)*ldb + (boff) )
    #define dC(i, j) (c + (i) + (j)*ldc)

    const magmaDoubleComplex c_one = MAGMA_Z_ONE;
    magmaDoubleComplex cbeta = MAGMA_Z_MAKE( beta, 0. );


    magma_int_t rows, ib, ioff, iblock, idev, di, s;

    // loop over all blocks
    // Faster to have two loops: first does A*B', second does B*A'
    // blockoffset is offset within first block; for subsequent blocks it is 0
    magma_int_t blockoffset = offset % nb;
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        iblock = (ioff / nb) / nmic;          // local block id
        idev   = (ioff / nb) % nmic;          // device with this block
        di     = iblock*nb + blockoffset;     // local index in parent matrix

        // C[i:n,i] += A[i:n,0] * B[i,0]'
        //printf( "zgemm  n=%4d, ib=%4d, k=%4d, i=%4d\n", n-i, ib, k, i );
        if(idev == my_id) {
            rows = n - i;
            zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &rows, &ib, &k,
                    (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)dA(i,0), &lda,
                    (MKL_Complex16 *)dB(i,0), &ldb,
                    (const MKL_Complex16 *)&cbeta, (MKL_Complex16 *)dC(ioff,di), &ldc);
        }

        blockoffset = 0;
    }

    blockoffset = offset % nb;
    alpha = MAGMA_Z_CNJG( alpha );
    for( magma_int_t i = 0; i < n; i += ib ) {
        ib     = min( nb-blockoffset, n-i );  // block size
        ioff   = i + offset;                  // global index in parent matrix
        iblock = (ioff / nb) / nmic;          // local block id
        idev   = (ioff / nb) % nmic;          // device with this block
        di     = iblock*nb + blockoffset;     // local index in parent matrix

        // C[i:n,i] += B[i:n,0] * A[i,0]'
        //printf( "zgemm  n=%4d, ib=%4d, k=%4d, i=%4d\n", n-i, ib, k, i );
        if(idev == my_id) {
            rows = n - i;
            zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans), &rows, &ib, &k,
                    (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)dB(i,0), &ldb,
                    (MKL_Complex16 *)dA(i,0), &lda,
                    (const MKL_Complex16 *)&c_one, (MKL_Complex16 *)dC(ioff,di), &ldc);
        }
        blockoffset = 0;
    }
}
#undef dA
#undef dB
#undef dC
//=========================================================================================
void call_zher2k_mmic(magma_mic_zher2k_mmic_param zher2k_mmic_param)
{
    magma_uplo_t uplo;
    magma_trans_t trans;
    magma_int_t n, k, lda, ldb, ldc, aoff, boff, offset, nb, my_id, nmic;
    magmaDoubleComplex alpha;
    double beta;
    magmaDoubleComplex *a, *b, *c;

    uplo = zher2k_mmic_param.uplo;
    trans = zher2k_mmic_param.trans;
    n = zher2k_mmic_param.n;
    k = zher2k_mmic_param.k;
    alpha = zher2k_mmic_param.alpha;
    a = (magmaDoubleComplex *)zher2k_mmic_param.a;
    lda = zher2k_mmic_param.lda;
    aoff = zher2k_mmic_param.aoff;
    b = (magmaDoubleComplex *)zher2k_mmic_param.b;
    ldb = zher2k_mmic_param.ldb;
    boff = zher2k_mmic_param.boff;
    beta = zher2k_mmic_param.beta;
    c = (magmaDoubleComplex *)zher2k_mmic_param.c;
    ldc = zher2k_mmic_param.ldc;
    offset = zher2k_mmic_param.offset;
    nmic = zher2k_mmic_param.nmic;
    nb = zher2k_mmic_param.nb;
    my_id = zher2k_mmic_param.my_id;

    //printf("ZHER2K_MMIC: %c %c %d %d %f 0x%lx %d %f 0x%lx %d\n", uplo, trans, n, k, alpha, (off_t)a, lda, beta, (off_t)c, ldc);

    magmablas_zher2k_mmic(
        uplo, trans, n, k,
        alpha, a, lda, aoff,
        b, ldb, boff,
        beta, c, ldc, offset,
        nmic, nb, my_id);
}
//=========================================================================================
void call_zlarft(magma_mic_zlarft_param zlarft_param)
{
    int direct, storev;
    magma_int_t n, k, ldv, ldt; 
    magmaDoubleComplex *v, *tau, *t;
    
    direct = zlarft_param.direct;
    storev = zlarft_param.storev;
    n = zlarft_param.n;
    k = zlarft_param.k;
    v = (magmaDoubleComplex *)zlarft_param.v;
    ldv = zlarft_param.ldv;
    tau = (magmaDoubleComplex *)zlarft_param.tau;
    t = (magmaDoubleComplex *)zlarft_param.t;
    ldt = zlarft_param.ldt;
    
    //printf("ZLARFT: %c %c %d %d 0x%lx %d ox%lx 0x%lx %d\n", direct, storev, n, k, (off_t)v, ldv, (off_t)tau, (off_t)t, ldt);
    
    zlarft( lapack_const(direct), lapack_const(storev),
            &n, &k, (MKL_Complex16 *)v, &ldv, 
            (MKL_Complex16 *)tau, (MKL_Complex16 *)t, &ldt);
}

//=========================================================================================
void call_zlarfb(magma_mic_zlarfb_param zlarfb_param)
{
    magma_int_t side, trans, direct, storev;
    magma_int_t m, n, k, ldv, ldt, ldc, ldwork;
    magmaDoubleComplex *v, *t, *c, *work;

    side = zlarfb_param.side;
    trans = zlarfb_param.trans;
    direct = zlarfb_param.direct;
    storev = zlarfb_param.storev;
    m = zlarfb_param.m;
    n = zlarfb_param.n;
    k = zlarfb_param.k;
    v = (magmaDoubleComplex *)zlarfb_param.v;
    ldv = zlarfb_param.ldv;
    t = (magmaDoubleComplex *)zlarfb_param.t;
    ldt = zlarfb_param.ldt;
    c = (magmaDoubleComplex *)zlarfb_param.c;
    ldc = zlarfb_param.ldc;
    work = (magmaDoubleComplex *)zlarfb_param.work;
    ldwork = zlarfb_param.ldwork;

    zlarfb(lapack_const(side), lapack_const(trans), lapack_const(direct), lapack_const(storev),
                    &m, &n, &k, (MKL_Complex16 *)v, &ldv, (MKL_Complex16 *)t, &ldt, 
                    (MKL_Complex16 *)c, &ldc, (MKL_Complex16 *)work, &ldwork);
}

//=========================================================================================
//magmablas our ZLARFB based on GEMMs Azzam
//=========================================================================================
void call_zlarfb_gemm(magma_mic_zlarfb_gemm_param zlarfb_gemm_param)
{
    magma_int_t side, trans, direct, storev;
    magma_int_t m, n, k, ldv, ldt, ldc, ldwork, ldworkvt;
    magmaDoubleComplex *v, *t, *c, *work, *workvt;
    
    side = zlarfb_gemm_param.side;
    trans = zlarfb_gemm_param.trans;
    direct = zlarfb_gemm_param.direct;
    storev = zlarfb_gemm_param.storev;
    m = zlarfb_gemm_param.m;
    n = zlarfb_gemm_param.n;
    k = zlarfb_gemm_param.k;
    v = (magmaDoubleComplex *)zlarfb_gemm_param.v;
    ldv = zlarfb_gemm_param.ldv;
    t = (magmaDoubleComplex *)zlarfb_gemm_param.t;
    ldt = zlarfb_gemm_param.ldt;
    c = (magmaDoubleComplex *)zlarfb_gemm_param.c;
    ldc = zlarfb_gemm_param.ldc;
    work = (magmaDoubleComplex *)zlarfb_gemm_param.work;
    ldwork = zlarfb_gemm_param.ldwork;
    workvt = (magmaDoubleComplex *)zlarfb_gemm_param.workvt;
    ldworkvt = zlarfb_gemm_param.ldworkvt;
    
    magmaDoubleComplex c_zero    = MAGMA_Z_ZERO;
    magmaDoubleComplex c_one     = MAGMA_Z_ONE;
    magmaDoubleComplex c_neg_one = MAGMA_Z_NEG_ONE;

    /* Function Body */
    if (m <= 0 || n <= 0) {
        return;
    }

    // opposite of trans
    magma_int_t transt;
    if (trans == MagmaNoTrans)
        transt = MagmaConjTrans;
    else
        transt = MagmaNoTrans;
    
    // whether T is upper or lower triangular
    magma_int_t uplo;
    if (direct == MagmaForward)
        uplo = MagmaUpper;
    else
        uplo = MagmaLower;
    
    // whether V is stored transposed or not
    magma_int_t notransV, transV;
    if (storev == MagmaColumnwise) {
        notransV = MagmaNoTrans;
        transV   = MagmaConjTrans;
    }
    else {
        notransV = MagmaConjTrans;
        transV   = MagmaNoTrans;
    }

    if ( side  == MagmaLeft) {
        // Form H C or H' C
        // Comments assume H C. 
        // When forming H' C, T gets transposed via transt for m>=n or by trans for m<n.
        
        // W = C' V
        zgemm( lapack_const(MagmaConjTrans), lapack_const(notransV),
                     &n, &k, &m,
                     (const MKL_Complex16 *)&c_one,  (MKL_Complex16 *)c, &ldc,
                             (MKL_Complex16 *)v, &ldv,
                     (const MKL_Complex16 *)&c_zero, (MKL_Complex16 *)work, &ldwork);

        if(m<n){
            // W2 = V T 
            zgemm( lapack_const(notransV), lapack_const(trans),
                         &m, &k, &k,
                         (const MKL_Complex16 *)&c_one,  (MKL_Complex16 *)v, &ldv,
                                 (MKL_Complex16 *)t, &ldt,
                         (const MKL_Complex16 *)&c_zero, (MKL_Complex16 *)workvt, &ldworkvt);
                        
            // C = C - W2 W' = C - V T V' C = (I - V T V') C = H C 
            zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaConjTrans),
                         &m, &n, &k,
                         (const MKL_Complex16 *)&c_neg_one, (MKL_Complex16 *)workvt, &ldworkvt,
                                    (MKL_Complex16 *)work, &ldwork,
                         (const MKL_Complex16 *)&c_one, (MKL_Complex16 *)c, &ldc);
        }else{
            // W2 = W T' = C' V T'
            zgemm( lapack_const(MagmaNoTrans), lapack_const(transt),
                         &n, &k, &k,
                         (const MKL_Complex16 *)&c_one,  (MKL_Complex16 *)work, &ldwork,
                                 (MKL_Complex16 *)t, &ldt,
                         (const MKL_Complex16 *)&c_zero, (MKL_Complex16 *)workvt, &ldworkvt);
                                                 
            // C = C - V W2' = C - V T V' C = (I - V T V') C = H C 
            zgemm( lapack_const(notransV), lapack_const(MagmaConjTrans),
                         &m, &n, &k,
                         (const MKL_Complex16 *)&c_neg_one, (MKL_Complex16 *)v,  &ldv,
                                    (MKL_Complex16 *)workvt, &ldworkvt,
                         (const MKL_Complex16 *)&c_one,     (MKL_Complex16 *)c,       &ldc);
        }
    }
    else {
        // Form C H or C H'
        // Comments assume C H. 
        // When forming C H', T gets transposed via trans.
        
        // W = C V
        zgemm( lapack_const(MagmaNoTrans), lapack_const(notransV),
                     &m, &k, &n,
                     (const MKL_Complex16 *)&c_one, (MKL_Complex16 *)c, &ldc,
                             (MKL_Complex16 *)v, &ldv,
                     (const MKL_Complex16 *)&c_zero, (MKL_Complex16 *)work, &ldwork);
        if(m<=n){
            // W2 = W T = C V T
            zgemm( lapack_const(MagmaNoTrans), lapack_const(trans),
                         &m, &k, &k,
                         (const MKL_Complex16 *)&c_one,  (MKL_Complex16 *)work, &ldwork,
                                 (MKL_Complex16 *)t, &ldt,
                         (const MKL_Complex16 *)&c_zero, (MKL_Complex16 *)workvt, &ldworkvt);
            
            // C = C - W2 V' = C - C V T V' = C (I - V T V') = C H
            zgemm( lapack_const(MagmaNoTrans), lapack_const(transV),
                         &m, &n, &k,
                         (const MKL_Complex16 *)&c_neg_one, (MKL_Complex16 *)work, &ldwork,
                                    (MKL_Complex16 *)v, &ldv,
                         (const MKL_Complex16 *)&c_one, (MKL_Complex16 *)c, &ldc);
        }else{
            // W2 = T V' 
            zgemm( lapack_const(trans), lapack_const(transV),
                         &k, &n, &k,
                         (const MKL_Complex16 *)&c_one,  (MKL_Complex16 *)t, &ldt,
                                 (MKL_Complex16 *)v, &ldv,
                         (const MKL_Complex16 *)&c_zero, (MKL_Complex16 *)workvt, &ldworkvt);
            // C = C - W W2 = C - C V T V' = C (I - V T V') = C H
            zgemm( lapack_const(MagmaNoTrans), lapack_const(MagmaNoTrans),
                         &m, &n, &k,
                         (const MKL_Complex16 *)&c_neg_one, (MKL_Complex16 *)work, &ldwork,
                                    (MKL_Complex16 *)workvt, &ldworkvt,
                         (const MKL_Complex16 *)&c_one, (MKL_Complex16 *)c, &ldc);

        }
    }
}
//=========================================================================================

void call_zlaset(magma_mic_zlaset_param zlaset_param)
{
    magma_uplo_t uplo;
    magma_int_t m, n, lda;
    magmaDoubleComplex alpha, beta;
    magmaDoubleComplex *dA;

    uplo  = zlaset_param.uplo;
    m     = zlaset_param.m;
    n     = zlaset_param.n;
    alpha = zlaset_param.alpha;
    beta  = zlaset_param.beta;
    dA    = (magmaDoubleComplex *)zlaset_param.dA;
    lda   = zlaset_param.lda;

    zlaset( lapack_const(uplo),        &m, &n, 
            (const MKL_Complex16 *)&alpha, (const MKL_Complex16 *)&beta,
            (MKL_Complex16 *)dA, &lda);
}
//=========================================================================================


void call_ztranspose(magma_mic_ztranspose_param ztranspose_param)
{
        /******************************************************
         * _mm512_load_pd -- for loading
         * _mm512_i32lo[ext]scatter_pd
         * _mm512_i32loscatter_pd is the simplified version of ...ext...
         * they do the same thing, the latter needs less arguments.
         * **************************************************/
        //        printf("entered stranpose\n");
        //        fflush(stdout);
        /*
           double *odata, *idata;
           int ldo,ldi, m, n;
           int num_threads;
           int tid;
           int index[16], i;
           __M512Z _reg;
           __m512i _index;
           double *my_head, *loop_head, *dest_head;
           int num_cols, num_cols_g, col, iter, num_iter, num_rows_g, num_rows;

           odata        =        (double *) ztranspose_param.odata;
           ldo                =        ztranspose_param.ldo;
           idata        =        (double *) ztranspose_param.idata;
           ldi                =        ztranspose_param.ldi;
           m                =        ztranspose_param.m;
           n                =        ztranspose_param.n;

           if (m == 0 || n == 0) {
           return;
           }
           if (m <= n) {
           for (i = 0; i < 16; i++) {
           index[i] = i*ZREG_WIDTH*ldo;
           }
        // 62 cores, 4 hardware threads per core. Leave out one core for the OS
        // (62 - 1)*4 = 244

        num_threads = 244;
        num_cols_g =        (n + num_threads - 1)/num_threads; 
        num_iter = (int) m*1.0/ZREG_WIDTH;
        _index = _mm512_set_epi32(index[15], index[14], index[13], index[12],
        index[11], index[10], index[9], index[8], index[7], index[6],
        index[5], index[4], index[3], index[2], index[1], index[0]);

#pragma omp parallel num_threads(num_threads) private(tid, col, num_cols, iter,\
my_head, loop_head, dest_head, _reg)
{
tid = omp_get_thread_num();
num_cols = num_cols_g;
if (tid == (num_threads - 1) && (n%num_cols_g != 0)) { 
num_cols = n%num_cols_g;
}
// starting point of my section, constant for each thread
my_head = idata + ldi*(tid*num_cols_g);
for (iter = 0; iter < num_iter; iter++) { // loop inside each column
for (col = 0; col < num_cols; col++) { // loop over each column
if (tid*num_cols_g + col < n) {
        // starting point of the inner loop
        loop_head = my_head + col*ldi + iter*ZREG_WIDTH;
        dest_head = odata + (iter*ZREG_WIDTH)*ldo + (tid*num_cols_g + col);
        // load 8 double precision floating point nubmers from address
        // mt into a float64 vector.
        _reg = _MM512_LOAD_PPZ( (void const *)loop_head);
        //                                        _mm512_load_pd( (void const *)loop_head);

        // this intrinsic needs int32 indices, but a 512 register
        // contains 16 int32 integers, and there is only one intrinsic
        // to set all 16 positoins of the register. So _index contains
        // 16 int32 integers.
        _MM512_I32LOEXTSCATTER_PPZ((void *) dest_head, _index, _reg,
        _MM_DOWNCONV_PZ_NONE, 1, _MM_HINT_NONE);
        }
        }
        }        
        }
        } else {
        // init index[], it's for scattering the entries to the dest matrix
        for (i = 0; i < 16; i++) {
        index[i] = i*ZREG_WIDTH*ldo;
        }

        // 62 cores, 4 hardware threads per core. Leave out one core for the OS
        // (62 - 1)*4 = 244
        num_threads = 244;
        // number of rows in each group
        num_rows_g = (m/ZREG_WIDTH + num_threads - 1)/num_threads*ZREG_WIDTH;
        _index = _mm512_set_epi32(index[15], index[14], index[13], index[12],
                        index[11], index[10], index[9], index[8], index[7], index[6],
                        index[5], index[4], index[3], index[2], index[1], index[0]);

#pragma omp parallel num_threads(num_threads) private(tid, col, num_rows, iter,\
                loop_head, dest_head, _reg, num_iter)
        {
                tid = omp_get_thread_num();
                // compute the number of rows I need to work on
                num_rows = num_rows_g;
                if (tid == (num_threads - 1) && (m%num_rows_g != 0)) {
                        num_rows = m%num_rows_g;
                }
                num_iter = (int) num_rows*1.0/ZREG_WIDTH;

                for (iter = 0; iter < num_iter; iter++) {
                        if (tid*num_rows_g + iter*ZREG_WIDTH < m) {
                                for (col = 0; col < n; col++) {
                                        loop_head = idata + (tid*num_rows_g + iter*ZREG_WIDTH) + ldi*col;
                                        dest_head = odata + ldo*(tid*num_rows_g + iter*ZREG_WIDTH) + col;

                                        _reg = _MM512_LOAD_PPZ((void const *) loop_head);
                                        _MM512_I32LOEXTSCATTER_PPZ((void *) dest_head, _index, _reg,
                                                        _MM_DOWNCONV_PZ_NONE, 1, _MM_HINT_NONE);
                                }
                        }
                }

        }
}
*/
return;
}

//=========================================================================================
void call_ztranspose_mkl(magma_mic_ztranspose_param ztranspose_param)
{
    magmaDoubleComplex *odata, *idata;
    int ldo,ldi, m, n;
    int num_threads;
    int tid;
    int index[16], i;
    //  _M512 _reg;
    //  _M512I _index;
    magmaDoubleComplex *my_head, *loop_head, *dest_head;
    int num_cols, num_cols_g, col, iter, num_iter;
    
    odata =(magmaDoubleComplex *) ztranspose_param.odata;
    ldo   =ztranspose_param.ldo;
    idata =(magmaDoubleComplex *) ztranspose_param.idata;
    ldi   = ztranspose_param.ldi;
    m     = ztranspose_param.m;
    n     = ztranspose_param.n;
    magmaDoubleComplex alpha = MAGMA_Z_MAKE(  1.0, 0.0 );
    
    mkl_zomatcopy('C', 'T', m, n, *((MKL_Complex16 *)(&alpha)), (MKL_Complex16 *)idata, ldi, 
                    (MKL_Complex16 *)odata, ldo);
}
//=========================================================================================

void call_zlaswp(magma_mic_zlaswp_param zlaswp_param)
{
    magma_int_t n, k1, k2, lda, incx;
    magmaDoubleComplex *d_A;
    magma_int_t *d_ipiv;
                     
    n      = zlaswp_param.n;
    d_A    = (magmaDoubleComplex *) zlaswp_param.d_A;
    lda    = zlaswp_param.lda;
    k1     = zlaswp_param.k1;
    k2     = zlaswp_param.k2;
    d_ipiv = (magma_int_t *) zlaswp_param.d_ipiv;
    incx   = zlaswp_param.incx;
    
    //printf("n: %d lda: %d k1: %d k2: %d incx: %d\n", n, lda, k1, k2, incx);
    zlaswp(&n, (MKL_Complex16 *)d_A, &lda, &k1, &k2, (magma_int_t *)d_ipiv, &incx);
}
//=========================================================================================

void call_zswap(magma_mic_zswap_param zswap_param)
{
    magma_int_t n, lda1, lda2;
    magmaDoubleComplex tmp, *dA1T, *dA2T;
                     
    n       = zswap_param.n;
    dA1T    = (magmaDoubleComplex *) zswap_param.dA1T;
    lda1    = zswap_param.lda1;
    dA2T    = (magmaDoubleComplex *) zswap_param.dA2T;
    lda2    = zswap_param.lda2;
    
    zswap(&n, (MKL_Complex16 *)dA1T, &lda1, (MKL_Complex16 *)dA2T, &lda2);
}
//=========================================================================================
void call_ztrsmgemm(magma_mic_ztrsmgemm_param ztrsmgemm_param)
{
    double gtrsmflops, ggemmflops, mic_trsmtime, mic_gemmtime;

    magma_mic_ztrsm_param trsm_param;
    magma_mic_zgemm_param gemm_param;

    magma_side_t side;
    magma_uplo_t uplo;
    magma_trans_t ttransa;
    magma_diag_t diag;
    magma_int_t tm, tn, tlda, tldb;
    magmaDoubleComplex talpha, *ta, *tb;

    trsm_param = ztrsmgemm_param.trsm_param;

    side = trsm_param.side;
    uplo = trsm_param.uplo;
    ttransa = trsm_param.transa;
    diag = trsm_param.diag;
    tm = trsm_param.m;
    tn = trsm_param.n;
    talpha = trsm_param.alpha;
    ta = (magmaDoubleComplex *)trsm_param.a;
    tlda = trsm_param.lda;
    tb = (magmaDoubleComplex *)trsm_param.b;
    tldb = trsm_param.ldb;

    mic_trsmtime = dsecnd();

    ztrsm( lapack_const(side), lapack_const(uplo), lapack_const(ttransa), lapack_const(diag),
           &tm, &tn, (const MKL_Complex16 *)&talpha, (MKL_Complex16 *)ta, &tlda, 
           (MKL_Complex16 *)tb, &tldb);

    mic_trsmtime = dsecnd() - mic_trsmtime;

    gtrsmflops = 0;
    if(lapack_const(side)[0] == 'L')
        gtrsmflops = MICFLOPS_TRSM_L((double)tm, (double)tn) * 1e-9;
    else if(lapack_const(side)[0] == 'R')
        gtrsmflops = MICFLOPS_TRSM_R((double)tm, (double)tn) * 1e-9;

    magma_trans_t transa, transb;
    magma_int_t m, n, k, lda, ldb, ldc;
    magmaDoubleComplex alpha, beta, *a, *b, *c;

    gemm_param = ztrsmgemm_param.gemm_param;


    transa = gemm_param.transa;
    transb = gemm_param.transb;
    m = gemm_param.m;
    n = gemm_param.n;
    k = gemm_param.k;
    alpha = gemm_param.alpha;
    a = (magmaDoubleComplex *)gemm_param.a;
    lda = gemm_param.lda;
    b = (magmaDoubleComplex *)gemm_param.b;
    ldb = gemm_param.ldb;
    beta = gemm_param.beta;
    c = (magmaDoubleComplex *)gemm_param.c;
    ldc = gemm_param.ldc;

    //printf("ZGEMM: %c %c %d %d %d %f 0x%lx %d 0x%lx %d %f 0x%lx %d\n", transa, transb, m, n, k, alpha, (off_t)a, lda, (off_t)b, ldb, beta, (off_t)c, ldc);

    mic_gemmtime = dsecnd();

    zgemm( lapack_const(transa), lapack_const(transb), 
           &m, &n, &k, (const MKL_Complex16 *)&alpha, (MKL_Complex16 *)a, &lda, 
           (MKL_Complex16 *)b, &ldb, (const MKL_Complex16 *)&beta, 
           (MKL_Complex16 *)c, &ldc);

    mic_gemmtime = dsecnd() - mic_gemmtime;

    ggemmflops = MICFLOPS_GEMM((double)m, (double)n, (double)k) * 1e-9;

    //printf("DTRSMGEMM: %8.4lf( %8.4lf )\t %8.4lf( %8.4lf )\n", gtrsmflops / mic_trsmtime, mic_trsmtime * 1000, ggemmflops / mic_gemmtime, mic_gemmtime * 1000); 
}
//=========================================================================================

