/*
 *  -- MAGMA (version 1.2.1) --
 *     Univ. of Tennessee, Knoxville
 *     Univ. of California, Berkeley
 *     Univ. of Colorado, Denver
 *     November 2012
 *
 * @generated s Fri Nov 22 14:21:34 2013
 *
 **/

// includes, system
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

// includes, project
#include "flops.h"
#include "magma.h"
#include "magma_lapack.h"
#include "testings.h"

#define PRECISION_s

float get_LU_error(magma_int_t M, magma_int_t N, 
                    float *A,  magma_int_t lda, 
                    float *LU, magma_int_t *IPIV)
{
    magma_int_t min_mn = min(M,N);
    magma_int_t ione   = 1;
    magma_int_t i, j;
    float alpha = MAGMA_S_ONE;
    float beta  = MAGMA_S_ZERO;
    float *L, *U;
    float work[1], matnorm, residual;
                       
    TESTING_MALLOC( L, float, M*min_mn);
    TESTING_MALLOC( U, float, min_mn*N);
    memset( L, 0, M*min_mn*sizeof(float) );
    memset( U, 0, min_mn*N*sizeof(float) );

    lapackf77_slaswp( &N, A, &lda, &ione, &min_mn, IPIV, &ione);
    lapackf77_slacpy( MagmaLowerStr, &M, &min_mn, LU, &lda, L, &M      );
    lapackf77_slacpy( MagmaUpperStr, &min_mn, &N, LU, &lda, U, &min_mn );

    for(j=0; j<min_mn; j++)
        L[j+j*M] = MAGMA_S_MAKE( 1., 0. );
    
    matnorm = lapackf77_slange("f", &M, &N, A, &lda, work);

    blasf77_sgemm("N", "N", &M, &N, &min_mn,
                  &alpha, L, &M, U, &min_mn, &beta, LU, &lda);

    for( j = 0; j < N; j++ ) {
        for( i = 0; i < M; i++ ) {
            LU[i+j*lda] = MAGMA_S_SUB( LU[i+j*lda], A[i+j*lda] );
        }
    }
    residual = lapackf77_slange("f", &M, &N, LU, &lda, work);

    TESTING_FREE(L);
    TESTING_FREE(U);

    return residual / (matnorm * N);
}

/* ////////////////////////////////////////////////////////////////////////////
   -- Testing sgetrf
   */
int main( int argc, char** argv)
{
	/* Initialize */
	magma_queue_t  queue;
	magma_device_t device;
	magma_int_t num = 0;
	magma_err_t err;

	magma_init();
	err = magma_get_devices( &device, 1, &num );
	if ( err != MAGMA_SUCCESS or num < 1 ) {
		fprintf( stderr, "magma_get_devices failed: %d\n", err );
		exit(-1);
	}
	err = magma_queue_create( device, &queue );
	if ( err != MAGMA_SUCCESS ) {
		fprintf( stderr, "magma_queue_create failed: %d\n", err );
		exit(-1);
	}

	real_Double_t   gflops, mic_perf, mic_time, cpu_perf, cpu_time;
	float          error;
	float *h_A, *h_R;
	float *d_lA[4] = {NULL, NULL, NULL, NULL};
	magma_int_t     *ipiv;
	magma_int_t M, N, n2, lda, ldda, info, min_mn;
	magma_int_t num_mics, num_mics0 = 1, n_local;
	magma_int_t i, j, k, nb0, nb, nk;
	magma_int_t ione     = 1;
	magma_int_t ISEED[4] = {0,0,0,1};

	magma_opts opts;
	parse_opts( argc, argv, &opts );
	num_mics0 = opts.nmic;

	printf("  M     N     CPU GFlop/s (sec)   MAGMA GFlop/s (sec)   ||PA-LU||/(||A||*N)\n");
	printf("=========================================================================\n");
	for(i = 0; i < opts.ntest; ++i ) {
		for( int iter = 0; iter < opts.niter; ++iter ) {
			M = opts.msize[i];
			N = opts.nsize[i];
			min_mn = min(M, N);
			lda    = M;
			n2     = lda*N;
			ldda   = ((M+31)/32)*32;
			gflops = FLOPS_SGETRF( M, N ) / 1e9;

			TESTING_MALLOC_HOST(    ipiv, magma_int_t,     min_mn );
			TESTING_MALLOC_HOST(    h_A,  float, n2     );
			TESTING_MALLOC_HOST( h_R,  float, n2     );

			//TESTING_MALLOC_DEV (  d_A,  float, ldda*N );
			/* == distributing the matrix == */
			nb = magma_get_sgetrf_nb(N);
			
			if( num_mics0 > N / nb ) {
				num_mics = N / nb;
				if( N % nb != 0 ) num_mics ++;
				printf( " * too many MICs for the matrix size, using %d MICs\n", (int) num_mics );
			} else {
				num_mics = num_mics0;
			}

			/* allocate device memory, assuming fixed nb and num_mics */
			for(j = 0; j < num_mics; j++){
				n_local = ((N / nb) / num_mics) * nb;
				if (j < ((N / nb) % num_mics))
					n_local += nb;
				else if (j == (N / nb) % num_mics)
					n_local += N % nb;
				magma_setdevice(j);

				d_lA[j] = 0;
				n_local += 32;
				if(n_local > 0) {
					TESTING_MALLOC_DEV( d_lA[j], float, ldda * n_local );
				}
			}

			/* Initialize the matrix */
			if ( opts.check ) {
				lapackf77_slarnv( &ione, ISEED, &n2, h_A );
				lapackf77_slacpy( MagmaUpperLowerStr, &M, &N, h_A, &lda, h_R, &lda );
			}
			else
				lapackf77_slarnv( &ione, ISEED, &n2, h_R );

			/* =====================================================================
			   Performs operation using LAPACK
			   =================================================================== */
			if ( opts.check ) {
				cpu_time = magma_wtime();
				lapackf77_sgetrf(&M, &N, h_A, &lda, ipiv, &info);
				cpu_time = magma_wtime() - cpu_time;
				cpu_perf = gflops / cpu_time;
				if (info != 0)
					printf("lapackf77_sgetrf returned error %d.\n", (int) info);
			}

			/* ====================================================================
			   Performs operation using MAGMA
			   =================================================================== */
			for( j = 0; j < N; j += nb){
				k = (j / nb) % num_mics;
				magma_setdevice(k);
				nk = min(nb, N - j);
				magma_ssetmatrix( M, nk,
						h_R + j * lda, 0,                      lda,
						d_lA[k] + j / (nb * num_mics) * nb * ldda, 0, ldda, queue);
			}

			magma_setdevice(0);

			/* == calling MAGMA with multiple MICs == */
			mic_time = magma_wtime();
			magma_sgetrf_mmic( num_mics, M, N, d_lA, ldda, ipiv, &info, queue);
			mic_time = magma_wtime() - mic_time;
			mic_perf = gflops / mic_time;
			if (info < 0) {
				printf("Argument %d of magma_sgetrf_mmic had an illegal value.\n", (int) -info);
				break;
			} else if (info != 0 ) {
				printf("magma_sgetrf_mmic returned info=%d.\n", (int) info);
				break;
			}

			/* =====================================================================
			   Check the factorization
			   =================================================================== */
			if ( opts.check ) {
				/* == download the matrix from MICs == */
				for(j = 0; j < N; j +=nb){
					k = (j / nb) % num_mics;
					magma_setdevice(k);
					nk = min(nb, N - j);
					magma_sgetmatrix( M, nk,
							d_lA[k]+j/(nb*num_mics)*nb*ldda, 0, ldda,
							h_A+j*lda,  0,                   lda, queue );
				}
				magma_setdevice(0);


				error = get_LU_error(M, N, h_R, lda, h_A, ipiv);

				printf("%5d %5d   %7.2f (%7.2f)   %7.2f (%7.2f)   %8.2e\n",
						(int) M, (int) N, cpu_perf, cpu_time, mic_perf, mic_time, error);
			}
			else {
				printf("%5d %5d     ---   (  ---  )   %7.2f (%7.2f)     ---  \n",
						(int) M, (int) N, mic_perf, mic_time);
			}

			fflush(0);
			/* Memory clean up */
			TESTING_FREE_HOST( ipiv );
			TESTING_FREE_HOST( h_A );
			TESTING_FREE_HOST( h_R );
			//TESTING_FREE_DEV ( d_A );
			for(j = 0; j < num_mics0; j++){
				magma_setdevice(j);
				if(d_lA[j] != NULL)
					TESTING_FREE_DEV( d_lA[j] );
			}


		}
		if ( opts.niter > 1 ) {
			printf( "\n" );
		}
	}


	/* Shutdown */
	magma_queue_destroy(queue);
	magma_finalize();
	return 0;
}
