/*
 *   -- micMAGMA (version 0.1) --
 *      Univ. of Tennessee, Knoxville
 *      Univ. of California, Berkeley
 *      Univ. of Colorado, Denver
 *      April 2012
 *
 *   @author Azzam Haidar
 *   @author Khairul Kabir 
 *
 *   @precisions normal z -> s d c
 */

#include <stdio.h>
#include <stdlib.h>
#include "magma.h"

#ifdef HAVE_MIC

#include "magma_mic_bindings.h"


// ========================================
// copying sub-matrices (contiguous columns)
magma_err_t
magma_zsetmatrix(
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex const* hA_src, size_t hA_offset, magma_int_t ldha,
    magmaDoubleComplex_ptr    dA_dst, size_t dA_offset, magma_int_t ldda,
    magma_queue_t queue )
{
    magma_event_t event;
    magma_mic_synchronize_kernels();
    magma_mic_memcpy2D_async(
        dA_dst + dA_offset, ldda*sizeof(magmaDoubleComplex),
        hA_src + hA_offset, ldha*sizeof(magmaDoubleComplex),
        m*sizeof(magmaDoubleComplex), n,
        WRITETO_DEV, queue, &event );
    magma_mic_fence_wait( event );
    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zgetmatrix(
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex_const_ptr dA_src, size_t dA_offset, magma_int_t ldda,
    magmaDoubleComplex*          hA_dst, size_t hA_offset, magma_int_t ldha,
    magma_queue_t queue )
{
    magma_event_t event;
    magma_mic_synchronize_kernels();
    magma_mic_memcpy2D_async(
        hA_dst + hA_offset, ldha*sizeof(magmaDoubleComplex),
        dA_src + dA_offset, ldda*sizeof(magmaDoubleComplex),
        m*sizeof(magmaDoubleComplex), n,
        READFROM_DEV, queue, &event );
    magma_mic_fence_wait( event );
    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zsetmatrix_async(
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex const* hA_src, size_t hA_offset, magma_int_t ldha,
    magmaDoubleComplex_ptr    dA_dst, size_t dA_offset, magma_int_t ldda,
    magma_queue_t queue, magma_event_t *event )
{
    magma_mic_memcpy2D_async(
        dA_dst + dA_offset, ldda*sizeof(magmaDoubleComplex),
        hA_src + hA_offset, ldha*sizeof(magmaDoubleComplex),
        m*sizeof(magmaDoubleComplex), n,
        WRITETO_DEV, queue, event );
    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zgetmatrix_async(
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex_const_ptr dA_src, size_t dA_offset, magma_int_t ldda,
    magmaDoubleComplex*          hA_dst, size_t hA_offset, magma_int_t ldha,
    magma_queue_t queue, magma_event_t *event )
{
    magma_mic_memcpy2D_async(
        hA_dst + hA_offset, ldha*sizeof(magmaDoubleComplex),
        dA_src + dA_offset, ldda*sizeof(magmaDoubleComplex),
        m*sizeof(magmaDoubleComplex), n,
        READFROM_DEV, queue, event );
    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zsetintarray_async(
    magma_int_t lda, magma_int_t const* hA_src, size_t hA_offset, 
    magma_int_t *dA_dst, size_t dA_offset, magma_queue_t queue, magma_event_t *event )
{
    magma_mic_memcpy2D_async((void *)(dA_dst + dA_offset), lda * sizeof(int), (void *)(hA_src + hA_offset), lda * sizeof(int), lda * sizeof(int), 1, WRITETO_DEV, queue, event);
    return MAGMA_SUCCESS;
}




// ========================================
// BLAS functions
// --------------------
magma_err_t
magma_zcopy(
    magma_int_t n,
    magmaDoubleComplex_ptr dx, size_t dx_offset, magma_int_t incx,
    magmaDoubleComplex_ptr dy, size_t dy_offset, magma_int_t incy,
    magma_queue_t handle )
{
    int err;
    magma_mic_zcopy_param copy_param;

    copy_param.n      = n;
    copy_param.x      = dx + dx_offset;
    copy_param.incx   = incx;
    copy_param.y      = dy + dy_offset;
    copy_param.incy    = incy;

    int control_msg = magma_mic_ZCOPY;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &copy_param, sizeof(copy_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zgemm(
    magma_trans_t transA, magma_trans_t transB,
    magma_int_t m, magma_int_t n, magma_int_t k,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dB, size_t dB_offset, magma_int_t ldb,
    magmaDoubleComplex beta,  magmaDoubleComplex_ptr       dC, size_t dC_offset, magma_int_t ldc,
    magma_queue_t handle )
{
    int err;
    magma_mic_zgemm_param gemm_param;

    gemm_param.transa = transA;
    gemm_param.transb = transB;
    gemm_param.m      = m;
    gemm_param.n      = n;
    gemm_param.k      = k;
    gemm_param.alpha  = alpha;
    gemm_param.a      = dA + dA_offset;
    gemm_param.lda    = lda;
    gemm_param.b      = dB + dB_offset;
    gemm_param.ldb    = ldb;
    gemm_param.beta   = beta;
    gemm_param.c      = dC + dC_offset;
    gemm_param.ldc    = ldc;

    int control_msg = magma_mic_ZGEMM;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &gemm_param, sizeof(gemm_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zgemv(
    magma_trans_t transA,
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dx, size_t dx_offset, magma_int_t incx,
    magmaDoubleComplex beta,  magmaDoubleComplex_ptr       dy, size_t dy_offset, magma_int_t incy,
    magma_queue_t handle )
{
    int err;
    magma_mic_zgemv_param gemv_param;

    gemv_param.transa = transA;
    gemv_param.m      = m;
    gemv_param.n      = n;
    gemv_param.alpha  = alpha;
    gemv_param.a      = dA + dA_offset;
    gemv_param.lda    = lda;
    gemv_param.x      = dx + dx_offset;
    gemv_param.incx   = incx;
    gemv_param.beta   = beta;
    gemv_param.y      = dy + dy_offset;
    gemv_param.incy    = incy;

    int control_msg = magma_mic_ZGEMV;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &gemv_param, sizeof(gemv_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------
// --------------------
magma_err_t
magma_zhemm(
    magma_side_t side, magma_uplo_t uplo,
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dB, size_t dB_offset, magma_int_t ldb,
    magmaDoubleComplex beta,  magmaDoubleComplex_ptr       dC, size_t dC_offset, magma_int_t ldc,
    magma_queue_t handle )
{
    int err;
    magma_mic_zhemm_param hemm_param;

    hemm_param.side   = side;
    hemm_param.uplo   = uplo;
    hemm_param.m      = m;
    hemm_param.n      = n;
    hemm_param.alpha  = alpha;
    hemm_param.a      = dA + dA_offset;
    hemm_param.lda    = lda;
    hemm_param.b      = dB + dB_offset;
    hemm_param.ldb    = ldb;
    hemm_param.beta   = beta;
    hemm_param.c      = dC + dC_offset;
    hemm_param.ldc    = ldc;
    hemm_param.agboff = 0;

    int control_msg = magma_mic_ZHEMM;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &hemm_param, sizeof(hemm_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}
// --------------------
magma_err_t
magma_zhemm_ours(
    magma_side_t side, magma_uplo_t uplo,
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dB, size_t dB_offset, magma_int_t ldb,
    magmaDoubleComplex beta,  magmaDoubleComplex_ptr       dC, size_t dC_offset, magma_int_t ldc,
    magma_int_t globaldAoffset,
    magma_queue_t handle )
{
    int err;
    magma_mic_zhemm_param hemm_param;

    hemm_param.side   = side;
    hemm_param.uplo   = uplo;
    hemm_param.m      = m;
    hemm_param.n      = n;
    hemm_param.alpha  = alpha;
    hemm_param.a      = dA + dA_offset;
    hemm_param.lda    = lda;
    hemm_param.b      = dB + dB_offset;
    hemm_param.ldb    = ldb;
    hemm_param.beta   = beta;
    hemm_param.c      = dC + dC_offset;
    hemm_param.ldc    = ldc;
    hemm_param.ldc    = ldc;
    hemm_param.agboff = globaldAoffset;


    int control_msg = magma_mic_ZHEMM;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &hemm_param, sizeof(hemm_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zsymv2(
    magma_uplo_t uplo,
    magma_int_t n,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dx, size_t dx_offset, magma_int_t incx,
    magmaDoubleComplex beta,  magmaDoubleComplex_ptr       dy, size_t dy_offset, magma_int_t incy,
    magma_queue_t handle )
{
    int err;
    magma_mic_zsymv2_param symv2_param;

    symv2_param.uplo   = uplo;
    symv2_param.n      = n;
    symv2_param.alpha  = alpha;
    symv2_param.a      = dA + dA_offset;
    symv2_param.lda    = lda;
    symv2_param.x      = dx + dx_offset;
    symv2_param.incx   = incx;
    symv2_param.beta   = beta;
    symv2_param.y      = dy + dy_offset;
    symv2_param.incy    = incy;

    int control_msg = magma_mic_ZSYMV2;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &symv2_param, sizeof(symv2_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zhemv(
    magma_uplo_t uplo,
    magma_int_t n,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dx, size_t dx_offset, magma_int_t incx,
    magmaDoubleComplex beta,  magmaDoubleComplex_ptr       dy, size_t dy_offset, magma_int_t incy,
    magma_queue_t handle )
{
    int err;
    magma_mic_zhemv_param hemv_param;

    hemv_param.uplo   = uplo;
    hemv_param.n      = n;
    hemv_param.alpha  = alpha;
    hemv_param.a      = dA + dA_offset;
    hemv_param.lda    = lda;
    hemv_param.x      = dx + dx_offset;
    hemv_param.incx   = incx;
    hemv_param.beta   = beta;
    hemv_param.y      = dy + dy_offset;
    hemv_param.incy    = incy;

    int control_msg = magma_mic_ZHEMV;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &hemv_param, sizeof(hemv_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zherk(
    magma_uplo_t uplo,
    magma_trans_t trans,
    magma_int_t n, magma_int_t k,
    double alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
    double beta,  magmaDoubleComplex_ptr       dC, size_t dC_offset, magma_int_t ldc,
    magma_queue_t handle )
{
    int err;
    magma_mic_zherk_param syrk_param;

    syrk_param.uplo  = uplo;
    syrk_param.trans = trans;
    syrk_param.n     = n;
    syrk_param.k     = k;
    syrk_param.alpha = alpha;
    syrk_param.a     = dA + dA_offset;
    syrk_param.lda   = lda;
    syrk_param.beta  = beta;
    syrk_param.c     = dC + dC_offset;
    syrk_param.ldc   = ldc;

    int control_msg = magma_mic_ZHERK;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &syrk_param, sizeof(syrk_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}


// --------------------
magma_err_t
magma_zherk_mmic(
    magma_uplo_t uplo, magma_trans_t trans,
    magma_int_t n, magma_int_t k,
    double alpha, magmaDoubleComplex_const_ptr dA, magma_int_t lda, magma_int_t dAoff, 
    double beta,  magmaDoubleComplex_const_ptr dC, magma_int_t ldc, magma_int_t offset, 
    magma_int_t nmic, magma_int_t nb, magma_int_t my_id )
{
    int err;
    magma_mic_zherk_mmic_param syrk_mmic_param;

    syrk_mmic_param.uplo  = uplo;
    syrk_mmic_param.trans = trans;
    syrk_mmic_param.n     = n;
    syrk_mmic_param.k     = k;
    syrk_mmic_param.alpha = alpha;
    syrk_mmic_param.a     = dA;
    syrk_mmic_param.lda   = lda;
    syrk_mmic_param.aoff     = dAoff;
    syrk_mmic_param.beta  = beta;
    syrk_mmic_param.c     = dC;
    syrk_mmic_param.ldc   = ldc;
    syrk_mmic_param.offset     = offset;
    syrk_mmic_param.nb   = nb;
    syrk_mmic_param.nmic   = nmic;
    syrk_mmic_param.my_id   = my_id;

    int control_msg = magma_mic_ZHERK_MMIC;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }


    if ((err = scif_send(gEpd, &syrk_mmic_param, sizeof(syrk_mmic_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}



// --------------------
magma_err_t
magma_zher2k(
    magma_uplo_t uplo, magma_trans_t trans,
    magma_int_t n, magma_int_t k,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dB, size_t dB_offset, magma_int_t ldb,
    double              beta, magmaDoubleComplex_ptr       dC, size_t dC_offset, magma_int_t ldc,
    magma_queue_t queue )
{
    int err;
    magma_mic_zher2k_param syr2k_param;

    syr2k_param.uplo  = uplo;
    syr2k_param.trans = trans;
    syr2k_param.n     = n;
    syr2k_param.k     = k;
    syr2k_param.alpha = alpha;
    syr2k_param.a     = dA + dA_offset;
    syr2k_param.lda   = lda;
    syr2k_param.b     = dB + dB_offset;
    syr2k_param.ldb   = ldb;
    syr2k_param.beta  = beta;
    syr2k_param.c     = dC + dC_offset;
    syr2k_param.ldc   = ldc;

    int control_msg = magma_mic_ZHER2K;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }


    if ((err = scif_send(gEpd, &syr2k_param, sizeof(syr2k_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------
magma_err_t
magma_zher2k_mmic(
    magma_uplo_t uplo, magma_trans_t trans, magma_int_t n, magma_int_t k,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, magma_int_t lda, magma_int_t dAoff,
                              magmaDoubleComplex_const_ptr dB, magma_int_t ldb, magma_int_t dBoff,
    double beta,              magmaDoubleComplex_const_ptr dC, magma_int_t ldc,  magma_int_t offset, 
    magma_int_t nmic, magma_int_t nb, magma_int_t my_id)
{
    int err;
    magma_mic_zher2k_mmic_param syr2k_mmic_param;

    syr2k_mmic_param.uplo  = uplo;
    syr2k_mmic_param.trans = trans;
    syr2k_mmic_param.n     = n;
    syr2k_mmic_param.k     = k;
    syr2k_mmic_param.alpha = alpha;
    syr2k_mmic_param.a     = dA;
    syr2k_mmic_param.lda   = lda;
    syr2k_mmic_param.aoff     = dAoff;
    syr2k_mmic_param.b     = dB;
    syr2k_mmic_param.ldb   = ldb;
    syr2k_mmic_param.boff     = dBoff;
    syr2k_mmic_param.beta  = beta;
    syr2k_mmic_param.c     = dC;
    syr2k_mmic_param.ldc   = ldc;
    syr2k_mmic_param.offset     = offset;
    syr2k_mmic_param.nb   = nb;
    syr2k_mmic_param.nmic   = nmic;
    syr2k_mmic_param.my_id   = my_id;

    int control_msg = magma_mic_ZHER2K_MMIC;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }


    if ((err = scif_send(gEpd, &syr2k_mmic_param, sizeof(syr2k_mmic_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}


// --------------------
magma_err_t
magma_ztrsm(
    magma_side_t  side,
    magma_uplo_t  uplo,
    magma_trans_t trans,
    magma_diag_t  diag,
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_ptr       dB, size_t dB_offset, magma_int_t ldb,
    magma_queue_t handle )
{
    int err;
    magma_mic_ztrsm_param trsm_param;

    trsm_param.side   = side;
    trsm_param.uplo   = uplo;
    trsm_param.transa = trans;
    trsm_param.diag   = diag;
    trsm_param.m      = m;
    trsm_param.n      = n;
    trsm_param.alpha  = alpha;
    trsm_param.a      = dA + dA_offset;
    trsm_param.lda    = lda;
    trsm_param.b      = dB + dB_offset;
    trsm_param.ldb    = ldb;

    int control_msg = magma_mic_ZTRSM;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &trsm_param, sizeof(trsm_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------                                                                                                           
magma_err_t
magma_ztrsv(magma_uplo_t uplo, magma_trans_t trans, magma_diag_t diag,
    magma_int_t n,
    magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
    magmaDoubleComplex_ptr dx, size_t dx_offset, magma_int_t incx,
    magma_queue_t queue )
{
    int err;
    magma_mic_ztrsv_param trsv_param;
    
    trsv_param.uplo   = uplo;
    trsv_param.trans  = trans;
    trsv_param.diag   = diag;
    trsv_param.n      = n;
    trsv_param.a      = dA + dA_offset;
    trsv_param.lda    = lda;
    trsv_param.dx     = dx + dx_offset;
    trsv_param.incx   = incx;
    
    int control_msg = magma_mic_ZTRSV;
    
    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }
    
    if ((err = scif_send(gEpd, &trsv_param, sizeof(trsv_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }
    
    return MAGMA_SUCCESS;
}

// --------------------                                                                                                                                      
magma_err_t
magma_ztrmm(
    magma_side_t side, magma_uplo_t uplo, magma_trans_t trans, magma_diag_t diag,
    magma_int_t m, magma_int_t n,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_ptr       dB, size_t dB_offset, magma_int_t ldb,
    magma_queue_t queue )
{
    int err;
    magma_mic_ztrmm_param trmm_param;

    trmm_param.side   = side;
    trmm_param.uplo   = uplo;
    trmm_param.transa = trans;
    trmm_param.diag   = diag;
    trmm_param.m      = m;
    trmm_param.n      = n;
    trmm_param.alpha  = alpha;
    trmm_param.a      = dA + dA_offset;
    trmm_param.lda    = lda;
    trmm_param.b      = dB + dB_offset;
    trmm_param.ldb    = ldb;

    int control_msg = magma_mic_ZTRMM;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &trmm_param, sizeof(trmm_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

// --------------------                                                                                                           
magma_err_t
magma_ztrmv(magma_uplo_t uplo, magma_trans_t trans, magma_diag_t diag,
    magma_int_t n,
    magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
    magmaDoubleComplex_ptr dx, size_t dx_offset, magma_int_t incx,
    magma_queue_t queue )
{
    int err;
    magma_mic_ztrmv_param trmv_param;
    
    trmv_param.uplo   = uplo;
    trmv_param.trans  = trans;
    trmv_param.diag   = diag;
    trmv_param.n      = n;
    trmv_param.a      = dA + dA_offset;
    trmv_param.lda    = lda;
    trmv_param.dx     = dx + dx_offset;
    trmv_param.incx   = incx;
    
    int control_msg = magma_mic_ZTRMV;
    
    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }
    
    if ((err = scif_send(gEpd, &trmv_param, sizeof(trmv_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }
    
    return MAGMA_SUCCESS;
}




// ========================================
// Auxuliary routines
magma_err_t
magma_ztranspose(magmaDoubleComplex_ptr odata, int offo, int ldo,
    magmaDoubleComplex_ptr idata, int offi, int ldi,
    int m, int n,
    magma_queue_t queue) 
{
    int err;
    magma_mic_ztranspose_param transpose_param;

    transpose_param.odata = odata + offo;
    transpose_param.ldo   = ldo;
    transpose_param.idata = idata + offi;
    transpose_param.ldi   = ldi;
    transpose_param.m     = m;
    transpose_param.n     = n;

    int control_msg = magma_mic_ZTRANSPOSE;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &transpose_param, sizeof(transpose_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

magma_err_t
magma_zlaswp(
    magma_int_t n, magmaDoubleComplex_ptr d_A, magma_int_t lda,
    magma_int_t k1, magma_int_t k2, magma_int_t *d_ipiv, magma_int_t incx,
    magma_queue_t queue)
{
    magma_event_t event;
    int err;
    magma_mic_zlaswp_param zlaswp_param;
    zlaswp_param.n      = n;
    zlaswp_param.d_A    = d_A;
    zlaswp_param.lda    = lda;
    zlaswp_param.k1     = k1;
    zlaswp_param.k2     = k2;
    zlaswp_param.d_ipiv = d_ipiv;
    zlaswp_param.incx   = incx;

    int control_msg = magma_mic_ZLASWP;
    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);    
    }

    if ((err = scif_send(gEpd, &zlaswp_param, sizeof(zlaswp_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);    
    }
    return MAGMA_SUCCESS;
}

magma_err_t
magma_zswap(
    magma_int_t n, magmaDoubleComplex *dA1T, magma_int_t lda1,
    magmaDoubleComplex *dA2T, magma_int_t lda2,
    magma_queue_t queue)
{
    magma_event_t event;
    int err;
    magma_mic_zswap_param zswap_param;
    zswap_param.n      = n;
    zswap_param.dA1T    = dA1T;
    zswap_param.lda1    = lda1;
    zswap_param.dA2T    = dA2T;
    zswap_param.lda2    = lda2;

    int control_msg = magma_mic_ZSWAP;
    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);    
    }

    if ((err = scif_send(gEpd, &zswap_param, sizeof(zswap_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);    
    }
    return MAGMA_SUCCESS;
}



// This version sends a larfb request to the MIC. The difference with the version in
// 'src' is that here there is only one request and control is passed immediately back
// to the CPU. 
magma_err_t
magma_zlarfb(
    magma_int_t side, magma_int_t trans, magma_int_t direct, magma_int_t storev,
    magma_int_t m, magma_int_t n, magma_int_t k,
    magmaDoubleComplex_ptr dV, size_t dV_offset, magma_int_t ldv,
    magmaDoubleComplex_ptr dT, size_t dT_offset, magma_int_t ldt,
    magmaDoubleComplex_ptr dC, size_t dC_offset, magma_int_t ldc,
    magmaDoubleComplex_ptr dwork, size_t dwork_offset, magma_int_t ldwork,
    magma_queue_t queue)
{
    int err;
    magma_mic_zlarfb_param larfb_param;

    larfb_param.side   = side;
    larfb_param.trans  = trans;
    larfb_param.direct = direct;
    larfb_param.storev = storev;
    larfb_param.m      = m;
    larfb_param.n      = n;
    larfb_param.k      = k;
    larfb_param.v      = dV + dV_offset;
    larfb_param.ldv    = ldv;
    larfb_param.t      = dT + dT_offset;
    larfb_param.ldt    = ldt;
    larfb_param.c      = dC + dC_offset;
    larfb_param.ldc    = ldc;
    larfb_param.work   = dwork + dwork_offset;
    larfb_param.ldwork = ldwork;

    int control_msg = magma_mic_ZLARFB;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &larfb_param, sizeof(larfb_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

magma_err_t
magma_zlarfb_gemm(
    magma_int_t side, magma_int_t trans, magma_int_t direct, magma_int_t storev,
    magma_int_t m, magma_int_t n, magma_int_t k,
    magmaDoubleComplex_ptr dV, size_t dV_offset, magma_int_t ldv,
    magmaDoubleComplex_ptr dT, size_t dT_offset, magma_int_t ldt,
    magmaDoubleComplex_ptr dC, size_t dC_offset, magma_int_t ldc,
    magmaDoubleComplex_ptr dwork, size_t dwork_offset, magma_int_t ldwork,
    magmaDoubleComplex_ptr dworkvt, size_t dworkvt_offset, magma_int_t ldworkvt,
    magma_queue_t queue)
{
    int err;
    magma_mic_zlarfb_gemm_param larfb_gemm_param;

    larfb_gemm_param.side   = side;
    larfb_gemm_param.trans  = trans;
    larfb_gemm_param.direct = direct;
    larfb_gemm_param.storev = storev;
    larfb_gemm_param.m      = m;
    larfb_gemm_param.n      = n;
    larfb_gemm_param.k      = k;
    larfb_gemm_param.v      = dV + dV_offset;
    larfb_gemm_param.ldv    = ldv;
    larfb_gemm_param.t      = dT + dT_offset;
    larfb_gemm_param.ldt    = ldt;
    larfb_gemm_param.c      = dC + dC_offset;
    larfb_gemm_param.ldc    = ldc;
    larfb_gemm_param.work   = dwork + dwork_offset;
    larfb_gemm_param.ldwork = ldwork;
    larfb_gemm_param.workvt   = dworkvt + dworkvt_offset;
    larfb_gemm_param.ldworkvt = ldworkvt;

    int control_msg = magma_mic_ZLARFB_GEMM;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &larfb_gemm_param, sizeof(larfb_gemm_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

magma_err_t
magma_zlaset(magma_uplo_t uplo,
             magma_int_t m, magma_int_t n,
             magmaDoubleComplex alpha, magmaDoubleComplex beta,
             magmaDoubleComplex_ptr dA, size_t dA_offset, magma_int_t lda,
             magma_queue_t queue)
{
     int err;
     magma_mic_zlaset_param laset_param;
     int control_msg = magma_mic_ZLASET;

     laset_param.uplo   = uplo;
     laset_param.m      = m;
     laset_param.n      = n;
     laset_param.alpha  = alpha;
     laset_param.beta   = beta;
     laset_param.dA     = dA + dA_offset;
     laset_param.lda    = lda;

     if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
         err = errno;
         printf("scif_send failed with err %d\n", errno);
         fflush(stdout);
     }

     if ((err = scif_send(gEpd, &laset_param, sizeof(laset_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
     }

     return MAGMA_SUCCESS;
}


// ========================================
// BLAS functions - combined blas for MIC
magma_err_t
magma_ztrsmgemm(
    magma_side_t side, magma_uplo_t uplo,
    magma_trans_t trans, magma_diag_t diag,
    magma_int_t tm, magma_int_t tn,
    magmaDoubleComplex talpha, magmaDoubleComplex_const_ptr tdA, size_t tdA_offset, magma_int_t tlda,
                               magmaDoubleComplex_ptr       tdB, size_t tdB_offset, magma_int_t tldb,
    
    magma_trans_t transA, magma_trans_t transB,
    magma_int_t m, magma_int_t n, magma_int_t k,
    magmaDoubleComplex alpha, magmaDoubleComplex_const_ptr dA, size_t dA_offset, magma_int_t lda,
                              magmaDoubleComplex_const_ptr dB, size_t dB_offset, magma_int_t ldb,
    magmaDoubleComplex beta,  magmaDoubleComplex_ptr       dC, size_t dC_offset, magma_int_t ldc,
    magma_queue_t handle )
{
    int err;
    magma_mic_ztrsmgemm_param trsmgemm_param;

    magma_mic_ztrsm_param trsm_param;
    magma_mic_zgemm_param gemm_param;

    //trsm parameter
    trsm_param.side   = side;
    trsm_param.uplo   = uplo;
    trsm_param.transa = trans;
    trsm_param.diag   = diag;
    trsm_param.m      = tm;
    trsm_param.n      = tn;
    trsm_param.alpha  = talpha;
    trsm_param.a      = tdA + tdA_offset;
    trsm_param.lda    = tlda;
    trsm_param.b      = tdB + tdB_offset;
    trsm_param.ldb    = tldb;

    //gemm parameter
    gemm_param.transa = transA;
    gemm_param.transb = transB;
    gemm_param.m      = m;
    gemm_param.n      = n;
    gemm_param.k      = k;
    gemm_param.alpha  = alpha;
    gemm_param.a      = dA + dA_offset;
    gemm_param.lda    = lda;
    gemm_param.b      = dB + dB_offset;
    gemm_param.ldb    = ldb;
    gemm_param.beta   = beta;
    gemm_param.c      = dC + dC_offset;
    gemm_param.ldc    = ldc;

    trsmgemm_param.trsm_param = trsm_param;
    trsmgemm_param.gemm_param = gemm_param;

    int control_msg = magma_mic_ZTRSM_GEMM;

    if ((err = scif_send(gEpd, &control_msg, sizeof(control_msg), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    if ((err = scif_send(gEpd, &trsmgemm_param, sizeof(trsmgemm_param), 1)) <= 0) {
        err = errno;
        printf("scif_send failed with err %d\n", errno);
        fflush(stdout);
    }

    return MAGMA_SUCCESS;
}

#endif // HAVE_MIC
