/*
    -- MAGMA (version 1.1) --
       Univ. of Tennessee, Knoxville
       Univ. of California, Berkeley
       Univ. of Colorado, Denver
       April 2012

       @precisions normal z -> s d c
*/

// includes, system
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

// includes, project
#include "flops.h"
#include "magma.h"
#include "magma_lapack.h"
#include "testings.h"

/* ////////////////////////////////////////////////////////////////////////////
   -- Testing magma_zher2k_mmic
*/
int main( int argc, char** argv)
{
    /* Initialize */
    magma_queue_t  queue;
    magma_device_t device;
    int num = 0;
    magma_err_t err;
    magma_init();
    err = magma_get_devices( &device, 1, &num );
    if ( err != MAGMA_SUCCESS or num < 1 ) {
      fprintf( stderr, "magma_get_devices failed: %d\n", err );
      exit(-1);
    }
    err = magma_queue_create( device, &queue );
    if ( err != MAGMA_SUCCESS ) {
      fprintf( stderr, "magma_queue_create failed: %d\n", err );
      exit(-1);
    } 
 
    magmaDoubleComplex c_neg_one = MAGMA_Z_NEG_ONE;
    double alpha = MAGMA_D_MAKE(  0.29, -0.86 );
    double beta  = MAGMA_D_MAKE( -0.48,  0.38 );
    double d_one = 1.0;
    
    real_Double_t    gflops, mic_perf=0., cpu_perf=0., mic_time=0., cpu_time=0.;
    real_Double_t    mic_perf2=0., mic_time2=0.;
    double           error=0., error2=0., work[1];
    magmaDoubleComplex *hA, *hR, *hV;
    magma_int_t j, nk, MagmaMaxMICs = 4;
    magmaDoubleComplex *dV[MagmaMaxMICs], *dW[MagmaMaxMICs], *dA[MagmaMaxMICs];

    /* Matrix size */
    magma_int_t n, size, lda, ldda, count;
    const int MAXTESTS = 10;
    // sizes are 1024*N - 32
    magma_int_t nsize[MAXTESTS] = { 1024, 2048, 3072, 4032, 5184, 6016, 7040, 8064, 9088, 10112 };
    magma_int_t k       = 64;
    magma_int_t nb      = 64;
    magma_int_t nmic    = 1;//magma_num_mics();
    
    magma_int_t ione     = 1;
    magma_int_t iseed[4] = {0,0,0,1};
    count = 1;
        
    printf( "Usage: %s -N n -K k -nb nb -nstream nstream -nmic nmic -count count -c\n"
            "    -N can be repeated %d times.\n"
            "    -nmic or setting $MAGMA_NUM_MICS sets number of MICs to use.\n"
            "    -c or setting $MAGMA_TESTINGS_CHECK runs LAPACK and checks result.\n\n",
            argv[0], MAXTESTS );

    int checkres = 0;

    int ntest = 0;
    int nmax = 0;
    for( int i = 1; i < argc; i++ ) {
        if ( strcmp("-N", argv[i]) == 0 && i+1 < argc ) {
            magma_assert( ntest < MAXTESTS, "error: -N repeated more than maximum %d tests\n", MAXTESTS );
            nsize[ntest] = atoi( argv[++i] );
            magma_assert( nsize[ntest] > 0, "error: -N %s is invalid; must be > 0.\n", argv[i] );
            nmax = max( nmax, nsize[ntest] );
            ntest++;
        }
        else if ( strcmp("-K", argv[i]) == 0 && i+1 < argc ) {
            k = atoi( argv[++i] );
            magma_assert( k > 0, "error: -K %s is invalid; must be > 0.\n", argv[i] );
        }
        else if ( strcmp("-NB", argv[i]) == 0 && i+1 < argc ) {
            nb = atoi( argv[++i] );
            magma_assert( nb > 0, "error: -NB %s is invalid; must be > 0.\n", argv[i] );
        }
        else if ( strcmp("-count", argv[i]) == 0 && i+1 < argc ) {
            count = atoi( argv[++i] );
            magma_assert( count > 0, "error: -count %s is invalid; must be > 0.\n", argv[i] );
        }
        else if ( strcmp("-nmic", argv[i]) == 0 && i+1 < argc ) {
            nmic = atoi( argv[++i] );
            magma_assert( nmic > 0, "error: -nmic %s is invalid; must be > 0.\n", argv[i] );
        }
        else if ( strcmp("-c", argv[i]) == 0 ) {
            checkres = true;
        }
        else {
            printf( "invalid argument: %s\n", argv[i] );
            exit(1);
        }
    }

    if ( ntest == 0 ) {
        ntest = MAXTESTS;
        nmax = nsize[ntest-1];
    }
    assert( nmax > 0 );
    
    // allocate memory for largest problem
    n = nmax;
    lda    = n;
    ldda   = ((n + 31)/32)*32;

    TESTING_MALLOC_HOST( hA,  magmaDoubleComplex, lda*n );
    TESTING_MALLOC_HOST( hR,  magmaDoubleComplex, lda*n );
    TESTING_MALLOC_HOST( hV,  magmaDoubleComplex, lda*k );
    
    for( int d = 0; d < nmic; ++d ) {
        magma_int_t nlocal = ((n / nb) / nmic + 1) * nb;
        magma_setdevice( d );
        TESTING_MALLOC_DEV( dA[d], magmaDoubleComplex, ldda*nlocal );
        TESTING_MALLOC_DEV( dV[d], magmaDoubleComplex, ldda*k*2      );
    }
    
    printf( "k %d, nb %d, nmic %d\n", (int) k, (int) nb, (int) nmic);
    printf("    n     k    nb offset  CPU GFlop/s (sec)   MIC GFlop/s (sec)   Ichi's code (sec)  |R|/|A|   Ichi |R|/|A|\n");
    printf("==========================================================================================\n");
    for( int i = 0; i < ntest; ++i ) {
        for( int offset = 0; offset < min(n,4*nb); offset += min(k,nb) ) {
            for( int cnt = 0; cnt < count; ++cnt ) {
                n = nsize[i];
                assert( n > 0 && k > 0 );
                
                lda  = n;
                ldda = ((n + 31)/32)*32;
                gflops = FLOPS_ZHER2K( (double)k, (double)n-offset ) / 1e9;
        
                size = lda*n;
                lapackf77_zlarnv( &ione, iseed, &size, hA );
                size = lda*k;
                lapackf77_zlarnv( &ione, iseed, &size, hV );
                
                /* ====================================================================
                   Performs operation using MAGMA
                   =================================================================== */
                //magmablas_zsetmatrix_1D_bcyclic( n, n, hA, lda, dA, ldda, nmic, nb );
		
		for( j = 0; j < n; j += nb){
			int d = (j / nb) % nmic;
			magma_setdevice(d );
			nk = min(nb, n - j);
			magma_zsetmatrix( n, nk,
					hA + j * lda, 0,                      lda,
					dA[d] + j / (nb * nmic) * nb * ldda, 0, ldda, queue);
		}


                for( int d = 0; d < nmic; ++d ) {
                    magma_setdevice( d );
                    magma_zsetmatrix( n, k, hV, 0, lda, dV[d], 0, ldda, queue );
                }
                
                mic_time = magma_wtime();
 
               magmablas_zherk_mmic2(
                    MagmaLower, MagmaNoTrans, n-offset, k,
                    alpha, dV, ldda,0,
                    beta,     dA, ldda, offset,
                    nmic, nb, queue );

                mic_time = magma_wtime() - mic_time;
                mic_perf = gflops / mic_time;
                
                // Get dA back to the CPU to compare with the CPU result.
                //magmablas_zgetmatrix_1D_bcyclic( n, n, dA, ldda, hR, lda, nmic, nb );
		for(j = 0; j < n; j +=nb){
			int d = (j / nb) % nmic;
			magma_setdevice(d );
			nk = min(nb, n - j);
			magma_zgetmatrix( n, nk,
					dA[d]+j/(nb*nmic)*nb*ldda, 0, ldda,
					hR+j*lda,  0,                   lda, queue );
		}

		/* =====================================================================
                   Performs operation using LAPACK
                   =================================================================== */
                if ( checkres ) {
                    // store ||A||
                    //error = lapackf77_zlange("f", &n, &n, hA, &lda, work );
                    error = lapackf77_zlanhe("f", "L", &n, hA, &lda, work );
                    
                    cpu_time = magma_wtime();
                    n -= offset;
                    blasf77_zherk( "Lower", "NoTrans", &n, &k,
                                    &alpha, hV, &lda,
                                    &beta,     &hA[ offset + offset*lda ], &lda );
                    n += offset;
                    cpu_time = magma_wtime() - cpu_time;
                    cpu_perf = gflops / cpu_time;
                    
                    // compute relative error ||R||/||A||, where R := A_magma - A_lapack = R - A
                    size = lda*n;
                    
                    blasf77_zaxpy( &size, &c_neg_one, hA, &ione, hR, &ione );
                    error = lapackf77_zlanhe("fro", "Lower", &n, hR, &lda, work) / error;
                    
                    printf( "%5d %5d %5d %5d   %7.1f (%7.4f)   %7.1f (%7.4f)	%8.2e\n",
                            (int) n, (int) k, (int) nb, (int) offset,
                            cpu_perf, cpu_time, mic_perf, mic_time, error);
                }
                else {
                    printf( "%5d %5d %5d %5d     ---   (  ---  )   %7.1f (%7.4f)    ---        ---\n",
                            (int) n, (int) k, (int) nb, (int) offset,
                            mic_perf, mic_time);
                }
            } // count
        } // offset
        printf( "\n" );
    }
    
    /* Memory clean up */
    for( int d = 0; d < nmic; ++d ) {
        magma_setdevice( d );
        TESTING_FREE_DEV( dA[d] );
        TESTING_FREE_DEV( dV[d] );
    }
    
    TESTING_FREE_HOST( hA );
    TESTING_FREE_HOST( hR );
    TESTING_FREE_HOST( hV );
    
	/* Shutdown */
	magma_queue_destroy(queue);
	magma_finalize();
	return 0;
}
