/*
   -- MAGMA (version 1.2.1) --
   Univ. of Tennessee, Knoxville
   Univ. of California, Berkeley
   Univ. of Colorado, Denver
   June 2012

   @generated d Fri Nov 22 14:21:35 2013

*/

// includes, system
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

// includes, project
#include "flops.h"
#include "magma.h"
#include "magma_lapack.h"
#include "testings.h"

#define Multimics

/* ////////////////////////////////////////////////////////////////////////////
   -- Testing dgeqrf
   */

int main( int argc, char** argv)
{
	/* Initialize */
	magma_queue_t  queue;
	magma_device_t device;
	magma_int_t num = 0;
	magma_err_t err;

	magma_init();
	err = magma_get_devices( &device, 1, &num );
	if ( err != MAGMA_SUCCESS or num < 1 ) {
		fprintf( stderr, "magma_get_devices failed: %d\n", err );
		exit(-1);
	}
	err = magma_queue_create( device, &queue );
	if ( err != MAGMA_SUCCESS ) {
		fprintf( stderr, "magma_queue_create failed: %d\n", err );
		exit(-1);
	}

	real_Double_t    gflops, mic_perf, mic_time, cpu_perf, cpu_time;
	double           error, work[1];
	double  c_neg_one = MAGMA_D_NEG_ONE;
	double *h_A, *h_R, *tau, *h_work, tmp[1];
	double *d_lA[4] = {NULL, NULL, NULL, NULL};

	magma_int_t M, N, n2, lda, ldda, lwork, info, min_mn;
	int num_mics, num_mics0 = 1, n_local;
	magma_int_t i, j, d, k, nk;
	magma_int_t ione     = 1;
	magma_int_t ISEED[4] = {0,0,0,1};
	
	magma_opts opts;
	parse_opts( argc, argv, &opts );
	num_mics0 = opts.nmic;

	printf("  M     N     CPU GFlop/s (sec)   MAGMA GFlop/s (sec)   ||R||_F / ||A||_F\n");
	printf("=======================================================================\n");

	for(i = 0; i < opts.ntest; ++i ) {
		for( int iter = 0; iter < opts.niter; ++iter ) {
			M = opts.msize[i];
			N = opts.nsize[i];
			min_mn = min(M, N);
			lda    = M;
			n2     = lda*N;
			ldda   = ((M+31)/32)*32;
			gflops = FLOPS_DGEQRF( M, N ) / 1e9;

			lwork = -1;
			lapackf77_dgeqrf(&M, &N, h_A, &M, tau, tmp, &lwork, &info);
			lwork = (magma_int_t)MAGMA_D_REAL( tmp[0] );

			TESTING_MALLOC_HOST(    tau, double, min_mn );
			TESTING_MALLOC_HOST(    h_A, double, n2     );
			TESTING_MALLOC_HOST(    h_R, double, n2     );
			TESTING_MALLOC_HOST(    h_work, double, lwork  );

			//TESTING_MALLOC_DEV (    d_A, double, ldda*N );
			magma_int_t nb  = magma_get_dgeqrf_nb(N);
			if( num_mics0 > N / nb ) {
				num_mics = N / nb;
				if( N % nb != 0 ) num_mics ++;
				printf( " * too many MICs for the matrix size, using %d MICs\n", (int) num_mics );
			} else {
				num_mics = num_mics0;
			}

			for(j = 0; j < num_mics; j++){      
				n_local = ((N / nb) / num_mics) * nb;
				if (j < (N / nb) % num_mics)
					n_local += nb;
				else if (j == (N / nb) % num_mics)
					n_local += N % nb;

				d_lA[j] = 0;
				magma_setdevice(j);
				if(n_local > 0)
					TESTING_MALLOC_DEV(  d_lA[j], double, ldda * n_local);
			}

			/* Initialize the matrix */
			if(opts.check) {
				lapackf77_dlarnv( &ione, ISEED, &n2, h_A );
				lapackf77_dlacpy( MagmaUpperLowerStr, &M, &N, h_A, &lda, h_R, &lda );
			} 
			else
				lapackf77_dlarnv( &ione, ISEED, &n2, h_R );

			/* ====================================================================
			   Performs operation using MAGMA
			   =================================================================== */
			for(j = 0; j < N; j += nb ) {
				d = (j / nb) % num_mics;
				magma_setdevice(d);
				nk = min(nb, N - j);
				magma_dsetmatrix( M, nk,
						h_R + j * lda, 0, lda,
						d_lA[d] + j / (nb * num_mics) * nb * ldda, 0, ldda, queue);
			}

			magma_setdevice(0);

			mic_time = magma_wtime();
			magma_dgeqrf2_mmic( num_mics, M, N, d_lA, ldda, tau, &info, queue);
			mic_time = magma_wtime() - mic_time;
			mic_perf = gflops / mic_time;

			if (info != 0)
				printf("magma_dgeqrf returned error %d.\n", (int) info);

			if ( opts.check ) {
				/* =====================================================================
				   Performs operation using LAPACK
				   =================================================================== */
				cpu_time = magma_wtime();
				lapackf77_dgeqrf(&M, &N, h_A, &lda, tau, h_work, &lwork, &info);
				cpu_time = magma_wtime() - cpu_time;
				cpu_perf = gflops / cpu_time;
				if (info != 0)
					printf("lapackf77_dgeqrf returned error %d.\n", (int) info);

				/* =====================================================================
				   Check the result compared to LAPACK
				   =================================================================== */
				for( j = 0; j < N; j += nb ) {
					d = (j / nb) % num_mics;
					magma_setdevice( d );
					nk = min(nb, N - j);
					magma_dgetmatrix( M, nk,
							d_lA[d] + j / (nb * num_mics) * nb * ldda, 0, ldda,
							h_R + j * lda, 0, lda, queue);
				}

				magma_setdevice(0);

				error = lapackf77_dlange("f", &M, &N, h_A, &lda, work);
				blasf77_daxpy(&n2, &c_neg_one, h_A, &ione, h_R, &ione);
				error = lapackf77_dlange("f", &M, &N, h_R, &lda, work) / error;

				printf("%5d %5d   %7.2f (%7.2f)   %7.2f (%7.2f)   %8.2e\n",
						(int) M, (int) N, cpu_perf, cpu_time, mic_perf, mic_time, error );
			}
			else {
				printf("%5d %5d     ---   (  ---  )   %7.2f (%7.2f)     ---  \n",
						(int) M, (int) N, mic_perf, mic_time);
			}
			fflush(0);

			TESTING_FREE_HOST( tau );
			TESTING_FREE_HOST( h_A );
			TESTING_FREE_HOST( h_work );
			TESTING_FREE_HOST( h_R );

			for(j = 0; j < num_mics; j++){
				magma_setdevice(j);
				if(d_lA[j] != NULL)
					TESTING_FREE_DEV( d_lA[j] );
			}
		}
		if ( opts.niter > 1 ) {
			printf( "\n" );
		}
	}

	/* Shutdown */
	magma_queue_destroy( queue );
	magma_finalize();

	return 0;
}
