/*
    -- MAGMA (version 1.2.1) --
       Univ. of Tennessee, Knoxville
       Univ. of California, Berkeley
       Univ. of Colorado, Denver
       June 2012

       @precisions normal z -> s d c

*/
#include "common_magma.h"

#define Multimics

extern "C" magma_int_t
magma_zgeqrf2_mmic( magma_int_t num_mics, magma_int_t m, magma_int_t n,
                    magmaDoubleComplex **dlA, magma_int_t ldda,
                    magmaDoubleComplex *tau, 
                    magma_int_t *info, magma_queue_t queue)
{
/*  -- MAGMA (version 1.2.1) --
       Univ. of Tennessee, Knoxville
       Univ. of California, Berkeley
       Univ. of Colorado, Denver
       June 2012

    Purpose
    =======
    ZGEQRF2_Mmic computes a QR factorization of a complex M-by-N matrix A:
    A = Q * R. This is a mic interface of the routine.

    Arguments
    =========
    M       (input) INTEGER
            The number of rows of the matrix A.  M >= 0.

    N       (input) INTEGER
            The number of columns of the matrix A.  N >= 0.

    dA      (input/output) COMPLEX_16 array on the mic, dimension (LDDA,N)
            On entry, the M-by-N matrix dA.
            On exit, the elements on and above the diagonal of the array
            contain the min(M,N)-by-N upper trapezoidal matrix R (R is
            upper triangular if m >= n); the elements below the diagonal,
            with the array TAU, represent the orthogonal matrix Q as a
            product of min(m,n) elementary reflectors (see Further
            Details).

    LDDA    (input) INTEGER
            The leading dimension of the array dA.  LDDA >= max(1,M).
            To benefit from coalescent memory accesses LDDA must be
            dividable by 16.

    TAU     (output) COMPLEX_16 array, dimension (min(M,N))
            The scalar factors of the elementary reflectors (see Further
            Details).

    INFO    (output) INTEGER
            = 0:  successful exit
            < 0:  if INFO = -i, the i-th argument had an illegal value
                  or another error occured, such as memory allocation failed.

    Further Details
    ===============

    The matrix Q is represented as a product of elementary reflectors

       Q = H(1) H(2) . . . H(k), where k = min(m,n).

    Each H(i) has the form

       H(i) = I - tau * v * v'

    where tau is a complex scalar, and v is a complex vector with
    v(1:i-1) = 0 and v(i) = 1; v(i+1:m) is stored on exit in A(i+1:m,i),
    and tau in TAU(i).
    =====================================================================    */

    	#define dlA(mic, a_1, a_2) ( dlA[mic] + (a_2) * (ldda) + (a_1))
        #define htmpR(i_)       (htmpR + (i_)*(nb)*(nb))
        #define tmppanel(i_)    (tmppanel + (i_))
        #define tmpnextpanel(i_)    (tmpnextpanel + (i_))
        #define tmpprevpanel(i_)    (tmpprevpanel + (i_))

        magmaDoubleComplex c_one  = MAGMA_Z_ONE ;
        magmaDoubleComplex c_zero = MAGMA_Z_ZERO;
	magmaDoubleComplex *dwork[4], *dworkvt[4], *panel[4];

	magma_int_t i, j, k, ldwork, lddwork, lddworkvt, old_i, old_nb, rows, old_rows;
	magma_int_t ib, nb, numblk;
	magma_int_t panel_micnum, i_local, n_local[4], la_mic; 

        magmaDoubleComplex  *work, *hwork, *hT, *htmpR, *dT[4], *dT0[4], *dT1[4], *dPanel0[4], *dPanel1[4], *dPanel[4];
        magmaDoubleComplex  *tmppanel, *tmppanel0, *tmppanel1, *tmpprevpanel;

        magma_int_t blkid, crosspoint;
        magma_int_t lwork, lhtmppanel, lhqrwork, lhT, lhR;

	*info = 0;
	if (m < 0) {
		*info = -1;
	} else if (n < 0) {
		*info = -2;
	} else if (ldda < max(1,m)) {
		*info = -4;
	}
	if (*info != 0) {
		magma_xerbla( __func__, -(*info) );
		return *info;
	}

	k = min(m, n);
	if (k == 0)
		return *info;

	magma_event_t event[4][2];
	for(i = 0; i < num_mics; i++){
		event[i][0] = NULL;
		event[i][1] = NULL;
	}  

	nb = magma_get_zgeqrf_nb(m);
	numblk = k / nb;

        // CPU WORK will be storing two consecutive temporaly 
        // panel (size=2*m*nb) followed by T (nb*nb) followed 
        // by temporary storing of the all upper part of Vs which is diag(R),
        // we will have temp R (n*nb)
        // followed  by work for QR and DLARFB (n*nb)        
        lhtmppanel  = m * nb;//2 * m * nb;
        lhT         = nb * nb; // size for storing T
        lhR         = 2 * nb * nb;//min(m,n)*nb;
        lhqrwork    = n * nb;
        lwork       = lhtmppanel + lhqrwork + lhT + lhR;
        //lhwork      = n * nb;

	//allocate host memory
	if (MAGMA_SUCCESS != magma_zmalloc_pinned( &work, lwork )) {
		*info = -9;
		for(i = 0; i < num_mics; i++){
			magma_setdevice(i);
			magma_free( dwork[i] );
		}

		*info = MAGMA_ERR_HOST_ALLOC;
		return *info;
	}

	//allocate device memory
	for(i = 0; i < num_mics; i++){
		magma_setdevice(i);
		if ( MAGMA_SUCCESS != magma_zmalloc( &dwork[i], (n*nb + 2*ldda*nb + 2*nb*nb + min(m, n)*nb ) )) {
			*info = MAGMA_ERR_DEVICE_ALLOC;
			return *info;
		}

		// setup pointer in device side
		dPanel0[i] = dwork[i] + n*nb;
		dPanel1[i] = dPanel0[i] + ldda*nb;
		dT0[i]       = dPanel1[i] + ldda*nb;
		dT1[i]       = dT0[i] + nb*nb;
		dworkvt[i] = dT1[i] + nb*nb;
	}

	// setup pointer in host side
        tmppanel0 = work;
        tmppanel1 = tmppanel0;//work + m*nb;
        hT        = work + lhtmppanel;
        htmpR     = work + lhtmppanel + lhT;
        hwork     = work + lhtmppanel + lhT + lhR;

	//set lda
	ldwork = m;
	lddwork = n;
	lddworkvt = ldwork;

	//Set the number of local n for each mic
	for(i = 0; i < num_mics; i++){
		n_local[i] = ((n / nb) / num_mics) * nb;
		if (i < (n / nb) % num_mics)
			n_local[i] += nb;
		else if (i == (n / nb) % num_mics)
			n_local[i] += n % nb;
	}

	if (k >= (numblk * nb)) {
		//#define ENABLE_TIMER
		
		#if defined (ENABLE_TIMER)
		double trfb[4], tmtc, tqr, trft, tctmr, tctmvt, tmnp, tcnp;
		#endif

                #if defined (ENABLE_TIMER)
                printf("\n\n %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s\n", 
                                "i", "nb", "row", "mtc", "tcnp", "panel", "dlarft", "QR+T", "ctmvt", "CPU", "dlarfb0", "dlarfb1", "dlarfb2", "dlarfb3", "MIC_np", "ctm r");
                printf("     ========================================================================================================================\n");
                #endif 

		//Use blocked code initially
		old_i = 0; old_nb = nb;
		blkid = -1;
		
		crosspoint = n;//0;

		for (i = 0; (i + nb) < k; i += nb) 
		{
			#if defined (ENABLE_TIMER)
			trfb[0] = trfb[1] = trfb[2] = trfb[3] = tmtc = tqr = trft = tctmr = tctmvt = tmnp = tcnp = 0.0;
			#endif

			blkid += 1;
			tmppanel = blkid%2 == 0 ? tmppanel0 : tmppanel1;

			//Set the mic number that holds the current panel
			panel_micnum = (i / nb) % num_mics;

			//Set the local index where the current panel is
			i_local = i / (nb * num_mics) * nb;

			rows = m - i;
			magma_setdevice(panel_micnum);

			if (i > 0 && old_rows > crosspoint){

				#if defined (ENABLE_TIMER)
				tmtc = magma_wtime();
				#endif

				//Apply H' to A(i:m, i + 2 * nb:n) from the left; this is the look-ahead
				//application to the trailing matrix                                    
				tmpprevpanel = (blkid - 1)%2 == 0 ? tmppanel0 : tmppanel1;
                                
				// CPU do the next panel, bring it from the MIC
                                magma_zgetmatrix_async(old_rows, nb, dlA(panel_micnum, old_i, i_local), 0, ldda, tmppanel(old_i), 0, ldwork, queue, &event[panel_micnum][0]);
                                magma_event_sync(event[panel_micnum][0]);

				#if defined (ENABLE_TIMER)
				tmtc = magma_wtime() - tmtc;
				#endif

				#if defined (ENABLE_TIMER)
				tcnp = magma_wtime();
				#endif

                                lapackf77_zlarfb( "L", "C", "F", "C", 
                                                   &old_rows, &nb, &nb, 
                                                   tmpprevpanel(old_i), &ldwork,
                                                   hT, &nb,
                                                   tmppanel(old_i), &ldwork,
                                                   hwork, &nb);

                                // send the top nb*nb of the updated panel to the MIC 
                                magma_zsetmatrix_async( nb, nb, tmppanel(old_i), 0, ldwork,
                                                        dlA(panel_micnum, old_i, i_local), 0, ldda, queue, &event[panel_micnum][1]); 
                                magma_event_sync(event[panel_micnum][1]);

				#if defined (ENABLE_TIMER)
				tcnp = magma_wtime() - tcnp;
				#endif
			}

			if(i == 0 || old_rows <= crosspoint) {
				#if defined (ENABLE_TIMER)
				tmtc = magma_wtime();
				#endif

				//Send current panel to the CPU
				magma_zgetmatrix_async( rows, nb,
						dlA(panel_micnum, i, i_local), 0, ldda,
						tmppanel(i),    0, ldwork, queue, &event[panel_micnum][1]);

				magma_event_sync( event[panel_micnum][1] );

				#if defined (ENABLE_TIMER)
				tmtc = magma_wtime() - tmtc;
				#endif
			}

			#if defined (ENABLE_TIMER)
			tqr = magma_wtime();
			#endif

			lapackf77_zgeqrf(&rows, &nb, tmppanel(i), &ldwork, tau + i, hwork, &lhqrwork, info);

			#if defined (ENABLE_TIMER)
			tqr = magma_wtime() - tqr;
			#endif

			#if defined (ENABLE_TIMER)
			trft = magma_wtime();
			#endif

			// ======================================================
                        //            SEND V to MIC
                        // ======================================================
                        // store the upper part of V (the R) then put zero 
                        // on the upper part of V then send it to MIC
                        // keep in mind that R need to be send it alsolater when
                        // MIC finish the dlarfb that use the V with zero on upper
                        //lapackf77_zlacpy("A", &nb, &nb, tmppanel(i), &ldwork, htmpR(blkid), &nb);
                        lapackf77_zlacpy("A", &nb, &nb, tmppanel(i), &ldwork, htmpR(blkid%2), &nb);
                        lapackf77_zlaset("U", &nb, &nb, &c_zero, &c_one, tmppanel(i), &ldwork);
		        	
			// Send the current panel back to the mics 
			// Has to be done with asynchronous copies
			for(j = 0; j < num_mics; j++)
			{  
				magma_setdevice(j);
				dPanel[j] = blkid%2 == 0 ? dPanel0[j] : dPanel1[j];
				if (j == panel_micnum) {
					panel[j] = dlA(j, i, i_local);
				}
				else
					panel[j] = dPanel[j];
				
				//send V
				magma_zsetmatrix_async(rows, nb, tmppanel(i), 0, ldwork, panel[j], 0, ldda, queue, &event[j][0]);
			}

			memset(hT, 0, nb * nb * sizeof(magmaDoubleComplex));
			// Form the triangular factor of the block reflector
			// H = H(i) H(i+1) . . . H(i+nb-1) 
			lapackf77_zlarft( MagmaForwardStr, MagmaColumnwiseStr,
					&rows, &nb,
					tmppanel(i), &ldwork, tau + i, hT, &nb);

			#if defined (ENABLE_TIMER)
			trft = magma_wtime() - trft;
			#endif
		
			#if defined (ENABLE_TIMER)
			tctmvt = magma_wtime();
			#endif
			
			// ======================================================
                        //            SEND T to MIC
                        // ======================================================
			// Send the current panel back to the mics 
			// Has to be done with asynchronous copies
			for(j = 0; j < num_mics; j++)
			{  
				magma_setdevice(j);
				dPanel[j] = blkid%2 == 0 ? dPanel0[j] : dPanel1[j];
				if (j == panel_micnum) {
					panel[j] = dlA(j, i, i_local);
				}
				else
					panel[j] = dPanel[j];
				
				//send T
				dT[j] = blkid%2 == 0 ? dT0[j] : dT1[j];
				magma_zsetmatrix_async(nb, nb, hT, 0, nb, dT[j], 0, nb, queue, &event[j][0]);
				
			}


			#if defined (ENABLE_TIMER)
			for(j = 0; j < num_mics; j++)
			{
				magma_setdevice(j);
				magma_event_sync( event[j][0] );
			}
			tctmvt = magma_wtime() - tctmvt;
			#endif

			//#if defined (ENABLE_TIMER)
			//tctmr = magma_wtime();
			//#endif

			//send previous R
			//if(i > 0) {
			//	int prev_mic = ((i - nb) / nb) % num_mics;
			//	int i_loc = (i - nb) / (nb * num_mics) * nb;
			//	magma_setdevice(prev_mic);
			//	magma_kernel_sync( );
				//send R
			//	magma_zsetmatrix_async( old_nb, old_nb,
			//			htmpR((blkid - 1)%2), 0, nb,
			//			dlA(prev_mic, i - nb, i_loc),   0, ldda, queue, &event[prev_mic][0] );

			//}

			//sync sending V&T
			for(j = 0; j < num_mics; j++)
			{
				magma_setdevice(j);
				magma_event_sync( event[j][0] );
			}
			//#if defined (ENABLE_TIMER)
			//tctmr = magma_wtime() - tctmr;
			//#endif

			if ((i + nb + nb) < k) 
			{
				la_mic = (panel_micnum + 1) % num_mics;
				int i_loc = (i + nb) / (nb * num_mics) * nb;
				int nmic = 0;

				//Apply H' to A(i:m, i + 2 * nb:n) from the left; this is the look-ahead
				//application to the trailing matrix       
				if(rows <= crosspoint) { 
					
					#if defined (ENABLE_TIMER)
					tmnp = magma_wtime();
					#endif

					magma_setdevice(la_mic);
					magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
							rows, nb, nb,
							panel[la_mic], 0, ldda, dT[la_mic],  0, nb,
							dlA(la_mic, i, i_loc), 0, ldda, dwork[la_mic], 0, lddwork, 
							dworkvt[la_mic], 0, lddworkvt, queue);
					
					#if defined (ENABLE_TIMER)
					magma_kernel_sync();
					tmnp = magma_wtime() - tmnp;
					#endif
				}

	
				//Apply H' to A(i:m,i+nb:i+2*nb) from the left;
				//This is update for the next panel; part of the look-ahead  
				for(j = (la_mic + 1) % num_mics; nmic < num_mics; nmic++, j = (j + 1) % num_mics){

					#if defined (ENABLE_TIMER)
					trfb[j] = magma_wtime();
					#endif

					magma_setdevice(j);
					magma_event_sync( event[j][0] );
					magma_kernel_sync( );
					
					magma_int_t ncols, j_mic;

					if (j == la_mic) {
						ncols = n_local[j] - i_loc - nb;
						j_mic = i_loc + nb;
					}else if (j <= panel_micnum){
						ncols = n_local[j] - i_local - nb;
						j_mic = i_local + nb;
					}else {
						ncols = n_local[j] - i_local;
						j_mic = i_local;
					}

					if(ncols > 0) {
						magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
								rows, ncols, nb,
								panel[j], 0, ldda, dT[j],  0, nb,
								dlA(j, i, j_mic), 0, ldda, dwork[j], 0, lddwork, 
								dworkvt[j], 0, lddworkvt, queue);
					}

					#if defined (ENABLE_TIMER)
					magma_kernel_sync();
					trfb[j] = magma_wtime() - trfb[j];
					#endif
				}     
			}
			else {
				//do the entire update as we exit and there would be no lookahead
				la_mic = (panel_micnum + 1) % num_mics;
				int i_loc = (i + nb) / (nb * num_mics) * nb;

				magma_setdevice(la_mic);
				magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
						rows, n_local[la_mic] - i_loc, nb,
						panel[la_mic], 0, ldda, dT[la_mic],    0, nb,
						dlA(la_mic, i, i_loc), 0, ldda, dwork[la_mic], 0, lddwork, 
						dworkvt[la_mic], 0, lddworkvt, queue);

				magma_setdevice(panel_micnum);
				magma_zsetmatrix( nb, nb,
						htmpR(blkid%2), 0, nb,
						dlA(panel_micnum, i, i_local), 0, ldda, queue );
			}

			#if defined (ENABLE_TIMER)
			tctmr = magma_wtime();
			#endif

			//send previous R
			if(i > 0) {
				int prev_mic = ((i - nb) / nb) % num_mics;
				int i_loc = (i - nb) / (nb * num_mics) * nb;
				magma_setdevice(prev_mic);
				//send R
				magma_zsetmatrix_async( old_nb, old_nb,
						htmpR((blkid - 1)%2), 0, nb,
						dlA(prev_mic, i - nb, i_loc),   0, ldda, queue, &event[prev_mic][0] );
				magma_event_sync( event[prev_mic][0] );

			}

			#if defined (ENABLE_TIMER)
			tctmr = magma_wtime() - tctmr;
			#endif


			old_i  = i;
			old_rows = rows;
			old_nb = nb;
	
			#if defined (ENABLE_TIMER)
			printf("%10d %10d %10d %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf(%d) %10.3lf\n",
					i, nb, rows, tmtc * 1000, tcnp * 1000, tqr * 1000, trft*1000, (trft+tqr)*1000, tctmvt * 1000, (trft+tqr+tctmvt+tmtc+tcnp)*1000,  
					trfb[0] * 1000, trfb[1] * 1000, trfb[2] * 1000, trfb[3] * 1000, tmnp * 1000, la_mic, tctmr * 1000);
			fflush(0);
			#endif


		}
	} else {
		i = 0;
	}

	for(j = 0; j < num_mics; j++){
		magma_setdevice(j);
		magma_free( dwork[j] );
	}

	// Use unblocked code to factor the last or only block 
	if (i < k) {
		ib   = n - i;
		rows = m - i;
		//lhwork = lwork - rows * ib;

		panel_micnum = (panel_micnum + 1) % num_mics;
		int i_loc = (i) / (nb * num_mics) * nb;

		magma_setdevice(panel_micnum);
		magma_zgetmatrix( rows, ib,
				dlA(panel_micnum, i, i_loc), 0, ldda,
				tmppanel0, 0, rows, queue);

		lhqrwork = lwork - rows * ib;
		lapackf77_zgeqrf(&rows, &ib, tmppanel0, &rows, tau + i, tmppanel0 + ib * rows, &lhqrwork, info);

		magma_zsetmatrix( rows, ib,
				tmppanel0, 0,                      rows,
				dlA(panel_micnum, i, i_loc), 0, ldda, queue);
	}

	magma_setdevice(0);
	magma_free_pinned(work);

	return *info;
} /* magma_zgeqrf2_mmic */
