/*
 *   -- micMAGMA (version 0.1) --
 *      Univ. of Tennessee, Knoxville
 *      Univ. of California, Berkeley
 *      Univ. of Colorado, Denver
 *      April 2012
 *
 */

#include <mkl.h>
#include <sys/time.h>
#include <sys/mman.h>

#include "magma.h"
#include "magma_mic_bindings.h"
#include "magma_z_bindings.h"
#include "magma_c_bindings.h"
#include "magma_d_bindings.h"
#include "magma_s_bindings.h"
#include "../../testing/flops.h"

#include "magma_s_server.h"
#include "magma_d_server.h"
#include "magma_c_server.h"
#include "magma_z_server.h"

#include "magma_zc_server.h"
#include "magma_ds_server.h"

// Flops formula
/*#define MICFMULS_GEMM(m, n, k) ((m) * (n) * (k))
#define MICFADDS_GEMM(m, n, k) ((m) * (n) * (k))
#define MICFMULS_SYMM(side, m, n) (((side) == MagmaLeft) ? MICFMULS_GEMM(m, m, n) : MICFMULS_GEMM(m, n, n))
#define MICFADDS_SYMM(side, m, n) (((side) == MagmaLeft) ? MICFADDS_GEMM(m, m, n) : MICFADDS_GEMM(m, n, n))
#define MICFMULS_GEMV(m, n)   ((m) * (n))
#define MICFADDS_GEMV(m, n)   ((m) * (n))
#define MICFMULS_TRMM_L(m, n) (0.5 * (n) * (m) * ((m)+1))
#define MICFADDS_TRMM_L(m, n) (0.5 * (n) * (m) * ((m)-1))
#define MICFMULS_TRMM_R(m, n) (0.5 * (m) * (n) * ((n)+1))
#define MICFADDS_TRMM_R(m, n) (0.5 * (m) * (n) * ((n)-1))
#define MICFMULS_TRSM_L(m, n) (0.5 * (n) * (m) * ((m)+1))
#define MICFADDS_TRSM_L(m, n) (0.5 * (n) * (m) * ((m)-1))
#define MICFMULS_TRSM_R(m, n) (0.5 * (m) * (n) * ((n)+1))
#define MICFADDS_TRSM_R(m, n) (0.5 * (m) * (n) * ((n)-1))
#define MICFMULS_SYRK(k, n) (0.5 * (k) * (n) * ((n)+1))
#define MICFADDS_SYRK(k, n) (0.5 * (k) * (n) * ((n)+1))
#define MICFMULS_SYR2K(k, n) ((k) * (n) * (n)        )
#define MICFADDS_SYR2K(k, n) ((k) * (n) * (n) + (n)  )

#define MICFLOPS_GEMM(m, n, k) (MICFMULS_GEMM(m, n, k) + MICFADDS_GEMM(m, n, k))
#define MICFLOPS_SYMM(side, m, n) (MICFMULS_SYMM(side, m, n) + MICFADDS_SYMM(side, m, n))
#define MICFLOPS_GEMV(m, n)    (MICFMULS_GEMV(m, n) + MICFADDS_GEMV(m, n))
#define MICFLOPS_TRSM_L(m, n)  (MICFMULS_TRSM_L(m, n) + MICFADDS_TRSM_L(m, n))
#define MICFLOPS_TRSM_R(m, n)  (MICFMULS_TRSM_R(m, n) + MICFADDS_TRSM_R(m, n))

#define MICFLOPS_SYRK(k, n)    (MICFMULS_SYRK(k, n) + MICFADDS_SYRK(k, n))
#define MICFLOPS_SYR2K(k, n)   (MICFMULS_SYR2K(k, n) + MICFADDS_SYR2K(k, n))
*/
/* //////////////////////////////////////////////////////////////////////////
   -- Return time in seconds since arbitrary point (e.g., unix epoch).
*/


#define MB_PAGE ((2 * 1024 * 1024))

#include "../../control/constants.cpp"

scif_epd_t mic_epd;
scif_epd_t epd;

int accept_connection()
{
    int req_pn = 2050, con_pn;
    struct scif_portID portID;
    
    if ((epd = scif_open()) < 0) {
        printf("scif_open failed with error %d\n", (int)epd);
        return MIC_FAILURE;
    }

    if ((con_pn = scif_bind(epd, req_pn)) < 0) {
        printf("scif_bind failed with error %d\n", con_pn);
        return MIC_FAILURE;
    }

    if ((scif_listen(epd, 5)) != 0) {
        printf("scif_listen failed with error %d\n", errno);
        return MIC_FAILURE;
    }

    if (scif_accept(epd, &portID, &mic_epd, SCIF_ACCEPT_SYNC) != 0) {
        printf("scif_accept failed with errno %d\n", errno);
        return MIC_FAILURE;
    }

    printf("accepted connection request from node:%d port:%d\n", portID.node, portID.port);
    return MAGMA_SUCCESS;
}


int close_connection()
{
    if ((scif_close(epd) != 0) || (scif_close(mic_epd) != 0)) {
        printf("scif_close failed with error %d\n", errno);
        return MIC_FAILURE;
    }

    return MAGMA_SUCCESS;
}


int mic_send(void *msg, int len)
{
    int err;
    if ((err = scif_send(mic_epd, msg, len, 1)) <= 0) {
        err = errno;
        printf("scif_recv failed with err %d\n", errno);
        fflush(stdout);
        return MIC_FAILURE;
    }
    return MAGMA_SUCCESS;
}


int mic_recv(void *msg, int len)
{
    int err;
    if ((err = scif_recv(mic_epd, msg, len, 1)) <= 0) {
        err = errno;
        printf("scif_recv failed with err %d\n", errno);
        fflush(stdout);
        return MIC_FAILURE;
    }
    return MAGMA_SUCCESS;
}


int mic_free(void *ptr, off_t offset, size_t nbyte)
{
    int err;
    
    //size_t real_size = ((nbyte + MB_PAGE - 1) / MB_PAGE + 1) * MB_PAGE;
    size_t real_size = nbyte + MB_PAGE & ~(MB_PAGE - 1);

    if ((err = scif_unregister(mic_epd, offset, real_size)) < 0) {
        err = errno;
        printf("mic_free: scif_unregister failed with err %d\n", errno);
        fflush(stdout);
        return MIC_FAILURE;
    }

    err = munmap(ptr, real_size);
    if(err != 0) {
        printf("mic_free: %d munmap failed: %d\n", EINVAL, err);
        return MIC_FAILURE;
    }

    //free(ptr);
    //printf("mic_free: 0x%lx offset: 0x%lx nby: %ld\n", (off_t)ptr, offset, nbyte);
    return MAGMA_SUCCESS;
}


int mic_allocate(void **ptr, size_t nbyte, off_t *offset)
{
    int err = 0;
    off_t suggested_offset;
    off_t address;

    //size_t real_size = ((nbyte + MB_PAGE - 1) / MB_PAGE + 1) * MB_PAGE;
    size_t real_size = nbyte + MB_PAGE & ~(MB_PAGE - 1);

    (*ptr) = mmap( NULL, real_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED | MAP_HUGETLB | MAP_POPULATE, -1, 0 );
    if (*ptr == MAP_FAILED || *ptr == NULL) {
        printf("mmap  failed %d\n", err);
        fflush(0);
        return MIC_FAILURE;
    }

    //err = posix_memalign(ptr, PAGE_SIZE, nbyte);
    //if (*ptr == NULL) {
    //    printf("posix_memalign  failed %d\n", err);
    //    fflush(0);
    //    return MIC_FAILURE;
    //}

    suggested_offset = (off_t)(*ptr);//0x0;
    address = (off_t)*ptr;

    if((*offset = scif_register(mic_epd, *ptr, real_size, suggested_offset,
                    SCIF_PROT_READ | SCIF_PROT_WRITE, 0)) < 0) {
        err = errno;
        printf("scif_register failed with err %d\n", errno);
        fflush(stdout);
        return MIC_FAILURE;
    }

    //printf("mic_malloc: 0x%lx offset: 0x%lx nbyte: %ld pagesize: %ld\n", address, *offset, nbyte, (size_t)sysconf(_SC_PAGESIZE));
    
    return MAGMA_SUCCESS;
}


void do_work() {
    int err = 0, task_ID, m, n, k;
    magma_side_t side;
    magma_trans_t transA, transB;

    double gflops, alpha, beta, mic_time;

    void *buf_address;
    off_t offset = 0;
    size_t nbyte = 4096;
    
    magma_mic_mem_info mmm_info;

    while(1) {
        if (mic_recv(&task_ID, sizeof(task_ID)) == MIC_FAILURE)
            return;

        switch(task_ID) {
            #include "magma_z_kernels.h"
            #include "magma_c_kernels.h"
            #include "magma_d_kernels.h"
            #include "magma_s_kernels.h"

            #include "magma_zc_kernels.h"
            #include "magma_ds_kernels.h"
            
            case magma_mic_SYNC:
                //printf("sync\n");
                if (mic_send(&task_ID, sizeof(task_ID)) == MIC_FAILURE) {
                    printf("scif_send failed for STRMM\n");
                    return;
                }
                break;


            case ALLOCATE_MEM:
                //printf("allocate mem\n");
                if(mic_recv(&mmm_info, sizeof(mmm_info)) == MIC_FAILURE) {
                    printf("scif_recv failed for ALLOCATE_MEM\n");
                    return;
                }

                //allocate
                if(mic_allocate(&buf_address, mmm_info.nbyte, &offset) == MIC_FAILURE) {
                    mmm_info.address = (off_t)(NULL);
                    mmm_info.offset = -1;
                } else {

                    mmm_info.address = (off_t)(buf_address);
                    mmm_info.offset = offset;
                }

                if(mic_send(&mmm_info, sizeof(mmm_info)) == MIC_FAILURE) {
                    printf("scif_send failed for ALLOCATE_MEM\n");
                    return;
                }
                break;

            case FREE_MEM:
                //printf("free mem\n");
                if (mic_recv(&mmm_info, sizeof(mmm_info)) == MIC_FAILURE) {
                    printf("scif_recv failed for FREE_MEM\n");
                    return;
                }

                buf_address = (void *)mmm_info.address;
                offset = mmm_info.offset;
                nbyte = mmm_info.nbyte;
                mic_free(buf_address, offset, nbyte);
                if(mic_send(&mmm_info, sizeof(mmm_info)) == MIC_FAILURE) {
                    printf("scif_send failed for FREE_MEM\n");
                    return;
                }

                break;

            case CLOSE_MIC:
                return;

            default:
                printf("Unknown task\n");
        }

    }
}


int main(int argc, char **argv)
{
    while(1) {
        if(accept_connection() != MAGMA_SUCCESS)
            return 0;

        do_work();

        if(close_connection() != MAGMA_SUCCESS)
            return 0;
    }

    return 0;
}
