/*
 *  -- MAGMA (version 1.2.1) --
 *     Univ. of Tennessee, Knoxville
 *     Univ. of California, Berkeley
 *     Univ. of Colorado, Denver
 *     June 2012
 *
 * @precisions normal z -> c d s
 *
 **/
/* includes, system */
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

/* includes, project */
#include "flops.h"
#include "magma.h"
#include "magma_lapack.h"
#include "magma_types.h"
#include "testings.h"

/* ////////////////////////////////////////////////////////////////////////////
   -- Testing zpotrf_mmic
*/
int main( int argc, char** argv) 
{
	/* Initialize */
	magma_queue_t  queue;
	magma_device_t device;
	magma_int_t num = 0;
	magma_err_t err;

	magma_init();
	err = magma_get_devices( &device, 1, &num );
	if ( err != MAGMA_SUCCESS or num < 1 ) {
		fprintf( stderr, "magma_get_devices failed: %d\n", err );
		exit(-1);
	}
	err = magma_queue_create( device, &queue );
	if ( err != MAGMA_SUCCESS ) {
		fprintf( stderr, "magma_queue_create failed: %d\n", err );
		exit(-1);
	}

	real_Double_t   gflops, mic_perf, mic_time, cpu_perf, cpu_time;
	double      work[1], error;
	magmaDoubleComplex *h_A, *h_R;
	magmaDoubleComplex *d_lA[4] = {NULL, NULL, NULL, NULL};
	magma_int_t N, n2, lda, ldda, info;
	magma_int_t i, j, k, num_mics0 = 1, num_mics;
	magmaDoubleComplex c_neg_one = MAGMA_Z_NEG_ONE;
	magma_int_t ione     = 1;
	magma_int_t ISEED[4] = {0,0,0,1};
	magma_int_t nb, nk, n_local, ldn_local;

	magma_opts opts;
	parse_opts( argc, argv, &opts );
	num_mics0 = opts.nmic;

	printf("  N     CPU GFlop/s (sec)   MAGMA GFlop/s (sec)   ||R_magma - R_lapack||_F / ||R_lapack||_F\n");
	printf("=============================================================================================\n");
	for(i = 0; i < opts.ntest; ++i ) {
		for( int iter = 0; iter < opts.niter; ++iter ) {
			N   = opts.nsize[i];
			lda = N;
			n2  = lda*N;
			ldda = ((N+31)/32)*32;
			gflops = FLOPS_ZPOTRF( N ) / 1e9;

			TESTING_MALLOC_HOST(    h_A, magmaDoubleComplex, n2     );
			TESTING_MALLOC_HOST( h_R, magmaDoubleComplex, n2     );

			//TESTING_MALLOC_DEV(  d_A, magmaDoubleComplex, ldda*N );
			nb = magma_get_zpotrf_nb(N);
			if( num_mics0 > N / nb ) {
				num_mics = N / nb;
				if( N % nb != 0 ) num_mics++;
				printf( " * too many mics for the matrix size, using %d mics\n", (int) num_mics );
			} else {
				num_mics = num_mics0;
			}

			for(j = 0; j < num_mics; j++){
				n_local = ((N / nb) / num_mics) * nb;
				if (j < (N / nb) % num_mics)
					n_local += nb;
				else if (j == (N / nb) % num_mics)
					n_local += N % nb;

				ldn_local = (n_local + 31);

				magma_setdevice(j);
				TESTING_MALLOC_DEV( d_lA[j], magmaDoubleComplex, ldda * ldn_local );
			}

			/* Initialize the matrix */
			if(opts.check) {
				lapackf77_zlarnv( &ione, ISEED, &n2, h_A );
				magma_zmake_hpd( N, h_A, lda );
				lapackf77_zlacpy( MagmaUpperLowerStr, &N, &N, h_A, &lda, h_R, &lda );
			} else {
				lapackf77_zlarnv( &ione, ISEED, &n2, h_A );
				magma_zmake_hpd( N, h_A, lda );
			}

			/* ====================================================================
			   Performs operation using MAGMA 
			   =================================================================== */
			/* distribute matrix to mics */
			//if( opts.uplo == MagmaUpper) {
				for(j = 0; j < N; j += nb){
					k = (j / nb) % num_mics;
					magma_setdevice(k);
					nk = min(nb, N - j);
					magma_zsetmatrix( N, nk,
							h_A + j * lda,  0,                     lda,
							d_lA[k] + j / (nb * num_mics) * nb * ldda, 0, ldda, queue);
				}
			/*} else {
				for(j = 0; j < N; j += nb){
					k = (j / nb) % num_mics;
					n_local = ((N / nb) / num_mics) * nb;
					if (k < (N / nb) % num_mics)
						n_local += nb;
					else if (k == (N / nb) % num_mics)
						n_local += N % nb;
					ldn_local = ((n_local + 31) / 32) * 32;
					ldn_local = (ldn_local % 256 == 0) ? ldn_local + 32 : ldn_local;

					magma_setdevice(k);
					nk = min(nb, N - j);
					magma_zsetmatrix( nk, N,
							h_A + j, 0,                     lda,
							d_lA[k] + j / (nb * num_mics) * nb, 0, ldn_local, queue );
				}
			}*/

			mic_time = magma_wtime();
			magma_zpotrf_mmic(num_mics, opts.uplo, N, d_lA, ldda, &info, queue);
			mic_time = magma_wtime() - mic_time;
			if (info != 0)
				printf("magma_zpotrf_mic returned error %d.\n", (int) info);
			mic_perf = gflops / mic_time;

			if ( opts.check ) {
				/* =====================================================================
				   Performs operation using LAPACK
				   =================================================================== */
				cpu_time = magma_wtime();
				lapackf77_zpotrf( lapack_const(opts.uplo), &N, h_A, &lda, &info );
				cpu_time = magma_wtime() - cpu_time;
				cpu_perf = gflops / cpu_time;
				if (info != 0)
					printf("lapackf77_zpotrf returned error %d.\n", (int) info);

				/* =====================================================================
				   Check the result compared to LAPACK
				   =================================================================== */
				//magma_zgetmatrix( N, N, d_A, 0, ldda, h_R, 0, lda, queue );
				/* gather matrix from mics */
				//if( opts.uplo == MagmaUpper ) {
					for(j = 0; j < N; j += nb){
						k = (j / nb) % num_mics;
						magma_setdevice(k);
						nk = min(nb, N - j);
						magma_zgetmatrix( N, nk,
								d_lA[k] + j / (nb * num_mics) * nb * ldda, 0, ldda,
								h_R + j * lda, 0,                       lda, queue );
					}
				/*} else {
					for(j = 0; j < N; j += nb){
						k = (j / nb) % num_mics;
						n_local = ((N / nb) / num_mics) * nb;
						if (k < (N / nb) % num_mics)
							n_local += nb;
						else if (k == (N / nb) % num_mics)
							n_local += N % nb;
						ldn_local = ((n_local + 31) / 32) * 32;
						ldn_local = (ldn_local % 256 == 0) ? ldn_local + 32 : ldn_local;

						magma_setdevice(k);
						nk = min(nb, N - j);
						magma_zgetmatrix( nk, N,
								d_lA[k] + j / (nb * num_mics) * nb, 0, ldn_local,
								h_R + j, 0,                      lda, queue );
					}
				}*/

				magma_setdevice(0);

				//error = lapackf77_zlange("f", &N, &N, h_A, &lda, work);
				error = lapackf77_zlanhe("f", "L", &N, h_A, &lda, work);
				blasf77_zaxpy(&n2, &c_neg_one, h_A, &ione, h_R, &ione);
				//error = lapackf77_zlange("f", &N, &N, h_R, &lda, work) / error;
				error = lapackf77_zlanhe("f", "L", &N, h_R, &lda, work) / error;

				printf("%5d   %7.2f (%7.2f)   %7.2f (%7.2f)   %8.2e\n",
						(int) N, cpu_perf, cpu_time, mic_perf, mic_time, error );
			}
			else {
				printf("%5d     ---   (  ---  )   %7.2f (%7.2f)     ---  \n",
						(int) N, mic_perf, mic_time );
			}

			fflush(0);

			TESTING_FREE_HOST(     h_A );
			TESTING_FREE_HOST( h_R );
			for(j = 0; j < num_mics; j++) {
				magma_setdevice(j);
				TESTING_FREE_DEV( d_lA[j] );
			}
		}
		if ( opts.niter > 1 ) {
			printf( "\n" );
		}
	}

	/* Shutdown */
	magma_queue_destroy( queue );
	magma_finalize();

	return 0;
}

