/*
 *  -- micMAGMA (version 1.3.0) --
 *     Univ. of Tennessee, Knoxville
 *     Univ. of California, Berkeley
 *     Univ. of Colorado, Denver
 *     April 2012
 *
 *     @precisions mixed zc -> ds
 *
 **/

// includes, system
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

// includes, project
#include "flops.h"
#include "magma.h"
#include "magma_lapack.h"
#include "testings.h"

#define PRECISION_z

int main(int argc , char **argv)
{
    real_Double_t time;
    double      gflopsF, gflopsS, gpu_perf;
    double      gpu_perfdf, gpu_perfds;
    double      gpu_perfsf, gpu_perfss;
    double      Rnorm, Anorm;
    magmaDoubleComplex c_one     = MAGMA_Z_ONE;
    magmaDoubleComplex c_neg_one = MAGMA_Z_NEG_ONE;
    magmaDoubleComplex *h_A, *h_B, *h_X;
    magmaDoubleComplex_ptr d_A, d_B, d_X, d_WORKD;
    magmaFloatComplex_ptr  d_As, d_Bs, d_WORKS;
    double          *h_workd;
    magma_int_t *h_ipiv, *d_ipiv;
    magma_int_t lda, ldb, ldx;
    magma_int_t ldda, lddb, lddx;
    magma_int_t i, iter, info, size;
    magma_int_t N        = 0;
    magma_int_t ione     = 1;
    magma_int_t NRHS     = 1;
    magma_int_t ISEED[4] = {0,0,0,1};
    magma_int_t sizetest[10] = {1024,2048,3072,4032,5184,6016,7040,7520,8064,8192};

    char trans = MagmaNoTrans;
    //char trans = MagmaConjTrans;

    if (argc != 1){
        for(i = 1; i<argc; i++){        
            if (strcmp("-N", argv[i])==0)
                N = atoi(argv[++i]);
            else if (strcmp("-nrhs", argv[i])==0)
                NRHS = atoi(argv[++i]);
        }
        if (N>0) sizetest[0] = sizetest[9] = N;
        else exit(1);
    }
    else {
        printf("\nUsage: \n");
        printf("  testing_zcgesv_mic -nrhs %d -N %d\n\n", (int) NRHS, 1024);
    }

    /* Initialize */
    magma_queue_t  queue;
    magma_device_t device;
    int num = 0;
    magma_err_t err;

    magma_init();
    err = magma_get_devices( &device, 1, &num );
    if ( err != MAGMA_SUCCESS or num < 1 ) {
      fprintf( stderr, "magma_get_devices failed: %d\n", err );
      exit(-1);
    }
    err = magma_queue_create( device, &queue );
    if ( err != MAGMA_SUCCESS ) {
      fprintf( stderr, "magma_queue_create failed: %d\n", err );
      exit(-1);
    }

    
    printf("Epsilon(double): %8.6e\n"
           "Epsilon(single): %8.6e\n\n", 
           lapackf77_dlamch("Epsilon"), lapackf77_slamch("Epsilon") );

    N = sizetest[9];
    ldb  = ldx = lda = N;
    ldda = ((N+31)/32)*32;
    lddb = lddx = ldda;

    TESTING_MALLOC_HOST( h_A, magmaDoubleComplex, lda*N    );
    TESTING_MALLOC_HOST( h_B, magmaDoubleComplex, ldb*NRHS );
    TESTING_MALLOC_HOST( h_X, magmaDoubleComplex, ldx*NRHS );
    TESTING_MALLOC_HOST( h_ipiv, magma_int_t,    N        );
    TESTING_MALLOC_HOST( h_workd, double, N );
    
    TESTING_MALLOC_DEV( d_A,     magmaDoubleComplex, ldda*N        );
    TESTING_MALLOC_DEV( d_B,     magmaDoubleComplex, lddb*NRHS     );
    TESTING_MALLOC_DEV( d_X,     magmaDoubleComplex, lddx*NRHS     );
    TESTING_MALLOC_DEV( d_ipiv,  magma_int_t,     N             );
    TESTING_MALLOC_DEV( d_WORKS, magmaFloatComplex,  ldda*(N+NRHS) );
    TESTING_MALLOC_DEV( d_WORKD, magmaDoubleComplex, N*NRHS        );

    printf("  N   DP-Factor  DP-Solve  SP-Factor  SP-Solve  MP-Solve  ||b-Ax||/||A||  NumIter\n");
    printf("==================================================================================\n");
    for(i=0; i<10; i++){
        N = sizetest[i] ;
        
        gflopsF = FLOPS_ZGETRF( N, N ) / 1e9;
        gflopsS = gflopsF + ( FLOPS_ZGETRS( N, NRHS ) / 1e9 );

        ldb  = ldx = lda = N;
        ldda = ((N+31)/32)*32;
        lddb = lddx = N;//ldda;

        /* Initialize matrices */
        size = lda * N;
        lapackf77_zlarnv( &ione, ISEED, &size, h_A );
        size = ldb * NRHS;
        lapackf77_zlarnv( &ione, ISEED, &size, h_B );
        lapackf77_zlacpy( MagmaUpperLowerStr, &N, &NRHS, h_B, &ldb, h_X, &ldx);

        printf("%5d  ",(int) N);

        magma_zsetmatrix( N, N,    h_A, 0, lda, d_A, 0, ldda, queue );
        magma_zsetmatrix( N, NRHS, h_B, 0, ldb, d_B, 0, lddb, queue );

        //=====================================================================
        //              MIXED - GPU
        //=====================================================================
	magma_kernel_sync();

        time = magma_wtime();
        magma_zcgesv_mic( trans, N, NRHS, 
                          d_A, 0, ldda, h_ipiv, d_ipiv, 
                          d_B, 0, lddb, d_X, 0, lddx, 
                          d_WORKD, d_WORKS, &iter, &info, queue );
	magma_kernel_sync();
        time = magma_wtime() - time;
        if (info < 0)
            printf("Argument %d of magma_zcgesv had an illegal value.\n", (int) -info);
        gpu_perf = gflopsS / time;

        //=====================================================================
        //              ERROR DP vs MIXED  - GPU
        //=====================================================================
        magma_zgetmatrix( N, NRHS, d_X, 0, lddx, h_X, 0, ldx, queue );

        Anorm = lapackf77_zlange("I", &N, &N, h_A, &lda, h_workd);
        blasf77_zgemm( lapack_const(trans), MagmaNoTransStr, 
                       &N, &NRHS, &N, 
                       &c_one,     h_A, &lda,
                                   h_X, &ldx,
                       &c_neg_one, h_B, &ldb);
        Rnorm = lapackf77_zlange("I", &N, &NRHS, h_B, &ldb, h_workd);

        //=====================================================================
        //                 Double Precision Factor 
        //=====================================================================
        magma_zsetmatrix( N, N, h_A, 0, lda, d_A, 0, ldda, queue );
        
	magma_kernel_sync();
        time = magma_wtime();
        magma_zgetrf_mic(N, N, d_A, 0, ldda, h_ipiv, &info, queue );
	magma_kernel_sync();
        time = magma_wtime() - time;
        if (info < 0)
            printf("Argument %d of magma_zgetrf had an illegal value.\n", (int) -info);
        gpu_perfdf = gflopsF / time;

        printf("%6.2f    ", gpu_perfdf); fflush(stdout);

        //=====================================================================
        //                 Double Precision Solve 
        //=====================================================================
        magma_zsetmatrix( N, N,    h_A, 0, lda, d_A, 0, ldda, queue );
        magma_zsetmatrix( N, NRHS, h_B, 0, ldb, d_B, 0, lddb, queue );

	magma_kernel_sync();
        time = magma_wtime();
        magma_zgetrf_mic(N, N, d_A, 0, ldda, h_ipiv, &info, queue );
        magma_zgetrs_mic( trans, N, NRHS, d_A, 0, ldda, h_ipiv, d_B, 0, lddb, &info, queue );
	magma_kernel_sync();
	magma_kernel_sync();
        time = magma_wtime() - time;
        if (info < 0)
            printf("Argument %d of magma_zgetrs had an illegal value.\n", (int) -info);

        gpu_perfds = gflopsS / time;

        printf("%6.2f    ", gpu_perfds); fflush(stdout);

        //=====================================================================
        //                 Single Precision Factor 
        //=====================================================================
        d_As = d_WORKS;
        d_Bs = d_WORKS + ldda*N;
        magma_zsetmatrix( N, N,    h_A, 0, lda,  d_A, 0, ldda, queue );
        magma_zsetmatrix( N, NRHS, h_B, 0, ldb,  d_B, 0, lddb, queue );

        magma_zlag2c( N, N,    d_A, 0, ldda, d_As, 0, ldda, queue ); 
        magma_zlag2c( N, NRHS, d_B, 0, lddb, d_Bs, 0, lddb, queue );

	magma_kernel_sync();
        time = magma_wtime();
        magma_cgetrf_mic(N, N, d_As, 0, ldda, h_ipiv, &info, queue );
	magma_kernel_sync();
        time = magma_wtime() - time;
        if (info < 0)
            printf("Argument %d of magma_cgetrf had an illegal value.\n", (int) -info);
        
        gpu_perfsf = gflopsF / time;
        printf("%6.2f     ", gpu_perfsf); fflush(stdout);

        //=====================================================================
        //                 Single Precision Solve 
        //=====================================================================
        magma_zlag2c(N, N,    d_A, 0, ldda, d_As, 0, ldda, queue ); 
        magma_zlag2c(N, NRHS, d_B, 0, lddb, d_Bs, 0, lddb, queue );

	magma_kernel_sync();
        time = magma_wtime();
        magma_cgetrf_mic( N, N,    d_As, 0, ldda, h_ipiv, &info, queue );
        magma_cgetrs_mic( trans, N, NRHS, d_As, 0, ldda, h_ipiv, 
                          d_Bs, 0, lddb, &info, queue );
	magma_kernel_sync();
        time = magma_wtime() - time;
        if (info < 0)
            printf("Argument %d of magma_cgetrs had an illegal value.\n", (int) -info);

        gpu_perfss = gflopsS / time;
        printf("%6.2f     ", gpu_perfss); fflush(stdout);

        printf("%6.2f     ", gpu_perf);
        printf("%e    %3d\n", Rnorm/Anorm, (int) iter); fflush(stdout);

        if( argc != 1 ) break ;        
    }

    /* Memory clean up */
    TESTING_FREE_HOST( h_A );
    TESTING_FREE_HOST( h_B );
    TESTING_FREE_HOST( h_X );
    TESTING_FREE_HOST( h_ipiv );
    TESTING_FREE_HOST( h_workd );
    
    TESTING_FREE_DEV( d_A );
    TESTING_FREE_DEV( d_B );
    TESTING_FREE_DEV( d_X );
    TESTING_FREE_DEV( d_ipiv );
    TESTING_FREE_DEV( d_WORKS );
    TESTING_FREE_DEV( d_WORKD );

    /* Shutdown */
    magma_queue_destroy( queue );
    magma_finalize();
}
