/*
    -- MAGMA (version 1.2.1) --
       Univ. of Tennessee, Knoxville
       Univ. of California, Berkeley
       Univ. of Colorado, Denver
       June 2012

       @precisions normal z -> s d c

*/
#include "common_magma.h"

#define Multimics

extern "C" magma_int_t
magma_zgeqrf2_mmic( magma_int_t num_mics, magma_int_t m, magma_int_t n,
                    magmaDoubleComplex **dlA, magma_int_t ldda,
                    magmaDoubleComplex *tau, 
                    magma_int_t *info, magma_queue_t queue)
{
/*  -- MAGMA (version 1.2.1) --
       Univ. of Tennessee, Knoxville
       Univ. of California, Berkeley
       Univ. of Colorado, Denver
       June 2012

    Purpose
    =======
    ZGEQRF2_Mmic computes a QR factorization of a complex M-by-N matrix A:
    A = Q * R. This is a mic interface of the routine.

    Arguments
    =========
    M       (input) INTEGER
            The number of rows of the matrix A.  M >= 0.

    N       (input) INTEGER
            The number of columns of the matrix A.  N >= 0.

    dA      (input/output) COMPLEX_16 array on the mic, dimension (LDDA,N)
            On entry, the M-by-N matrix dA.
            On exit, the elements on and above the diagonal of the array
            contain the min(M,N)-by-N upper trapezoidal matrix R (R is
            upper triangular if m >= n); the elements below the diagonal,
            with the array TAU, represent the orthogonal matrix Q as a
            product of min(m,n) elementary reflectors (see Further
            Details).

    LDDA    (input) INTEGER
            The leading dimension of the array dA.  LDDA >= max(1,M).
            To benefit from coalescent memory accesses LDDA must be
            dividable by 16.

    TAU     (output) COMPLEX_16 array, dimension (min(M,N))
            The scalar factors of the elementary reflectors (see Further
            Details).

    INFO    (output) INTEGER
            = 0:  successful exit
            < 0:  if INFO = -i, the i-th argument had an illegal value
                  or another error occured, such as memory allocation failed.

    Further Details
    ===============

    The matrix Q is represented as a product of elementary reflectors

       Q = H(1) H(2) . . . H(k), where k = min(m,n).

    Each H(i) has the form

       H(i) = I - tau * v * v'

    where tau is a complex scalar, and v is a complex vector with
    v(1:i-1) = 0 and v(i) = 1; v(i+1:m) is stored on exit in A(i+1:m,i),
    and tau in TAU(i).
    =====================================================================    */

    	#define dlA(mic, a_1, a_2) ( dlA[mic] + (a_2) * (ldda) + (a_1))
        #define htmpR(i_)       (htmpR + (i_)*(pnb)*(pnb))
        #define tmppanel(i_)    (tmppanel + (i_))
        #define tmpnextpanel(i_)    (tmpnextpanel + (i_))
        #define tmpprevpanel(i_)    (tmpprevpanel + (i_))

        magmaDoubleComplex c_one  = MAGMA_Z_ONE ;
        magmaDoubleComplex c_zero = MAGMA_Z_ZERO;
	magmaDoubleComplex *dwork[4], *dworkvt[4], *panel[4];

	magma_int_t i, j, k, ldwork, lddwork, lddworkvt, old_i, old_pnb, rows, old_rows;
	magma_int_t ib, nb, numblk;
	magma_int_t panel_micnum, i_local, n_local[4], la_mic; 
        magma_int_t pnb, blockoffset, myblkid;

        magmaDoubleComplex  *work, *hwork, *hT, *htmpR, *dT[4], *dT0[4], *dT1[4], *dPanel0[4], *dPanel1[4], *dPanel[4];
        magmaDoubleComplex  *tmppanel, *tmppanel0, *tmppanel1, *tmpprevpanel;

        magma_int_t blkid, crosspoint;
        magma_int_t lhwork, lwork, lhtmppanel, lhqrwork, lhT, lhR;

	*info = 0;
	if (m < 0) {
		*info = -1;
	} else if (n < 0) {
		*info = -2;
	} else if (ldda < max(1,m)) {
		*info = -4;
	}
	if (*info != 0) {
		magma_xerbla( __func__, -(*info) );
		return *info;
	}

	k = min(m, n);
	if (k == 0)
		return *info;

	magma_event_t event[4][2];
	for(i = 0; i < num_mics; i++){
		event[i][0] = NULL;
		event[i][1] = NULL;
	}  

	nb = magma_get_zgeqrf_nb(m);
        pnb = 512; // 256;//nb;

	printf("hello here is pnb %d and distnb %d \n",pnb,nb);
	numblk = k / pnb;


        // CPU WORK will be storing two consecutive temporaly 
        // panel (size=2*m*nb) followed by T (nb*nb) followed 
        // by temporary storing of the all upper part of Vs which is diag(R),
        // we will have temp R (n*nb)
        // followed  by work for QR and DLARFB (n*nb)        
        lhtmppanel  = 2 * m * pnb;
        lhT         = pnb * pnb; // size for storing T
        lhR         = min(m,n)*pnb;
        lhqrwork    = n * pnb;
        lwork       = lhtmppanel + lhqrwork + lhT + lhR;
        lhwork      = n * pnb;

	//allocate host memory
	if (MAGMA_SUCCESS != magma_zmalloc_pinned( &work, lwork )) {
		*info = -9;
		for(i = 0; i < num_mics; i++){
			magma_setdevice(i);
			magma_free( dwork[i] );
		}

		*info = MAGMA_ERR_HOST_ALLOC;
		return *info;
	}

	//allocate device memory
	for(i = 0; i < num_mics; i++){
		magma_setdevice(i);
		if ( MAGMA_SUCCESS != magma_zmalloc( &dwork[i], (n*pnb + 2*ldda*pnb + 2*pnb*pnb + min(m, n)*pnb ) )) {
			*info = MAGMA_ERR_DEVICE_ALLOC;
			return *info;
		}

		// setup pointer in device side
		dPanel0[i] = dwork[i] + n*pnb;
		dPanel1[i] = dPanel0[i] + ldda*pnb;
		dT0[i]       = dPanel1[i] + ldda*pnb;
		dT1[i]       = dT0[i] + pnb*pnb;
		dworkvt[i] = dT1[i] + pnb*pnb;
	}

	// setup pointer in host side
        tmppanel0 = work;
        tmppanel1 = work + m*pnb;
        hT        = work + lhtmppanel;
        htmpR     = work + lhtmppanel + lhT;
        hwork     = work + lhtmppanel + lhT + lhR;

	//set lda
	ldwork = m;
	lddwork = n;
	lddworkvt = ldwork;

	//Set the number of local n for each mic
	for(i = 0; i < num_mics; i++){
		n_local[i] = ((n / nb) / num_mics) * nb;
		if (i < (n / nb) % num_mics)
			n_local[i] += nb;
		else if (i == (n / nb) % num_mics)
			n_local[i] += n % nb;
	}

	if (k >= (numblk * pnb)) {
		//Use blocked code initially
		old_i = 0; old_pnb = pnb;
		blkid = -1;
		for (i = 0; (i + pnb) < k; i += pnb) 
		{
			blkid += 1;
			tmppanel = blkid%2 == 0 ? tmppanel0 : tmppanel1;

			//Set the mic number that holds the current panel
			panel_micnum = (i / nb) % num_mics;

			//Set the local index where the current panel is
			blockoffset = i%nb;
			//i_local = i / (nb * num_mics) * nb;
			myblkid = (i / nb) / num_mics;
			i_local = myblkid * nb + blockoffset;

			rows = m - i;
			magma_setdevice(panel_micnum);
			//Send current panel to the CPU
			magma_zgetmatrix_async( rows, pnb,
					dlA(panel_micnum, i, i_local), 0, ldda,
					tmppanel(i),    0, ldwork, queue, &event[panel_micnum][1]);

			magma_event_sync( event[panel_micnum][1] );

			lapackf77_zgeqrf(&rows, &pnb, tmppanel(i), &ldwork, tau + i, hwork, &lhqrwork, info);

			memset(hT, 0, pnb * pnb * sizeof(magmaDoubleComplex));
			// Form the triangular factor of the block reflector
			// H = H(i) H(i+1) . . . H(i+nb-1) 
			lapackf77_zlarft( MagmaForwardStr, MagmaColumnwiseStr,
					&rows, &pnb,
					tmppanel(i), &ldwork, tau + i, hT, &pnb);

		
			// ======================================================
                        //            SEND V and T to MIC
                        // ======================================================
                        // store the upper part of V (the R) then put zero 
                        // on the upper part of V then send it to MIC
                        // keep in mind that R need to be send it alsolater when
                        // MIC finish the dlarfb that use the V with zero on upper
                        lapackf77_zlacpy("A", &pnb, &pnb, tmppanel(i), &ldwork, htmpR(blkid), &pnb);
                        lapackf77_zlaset("U", &pnb, &pnb, &c_zero, &c_one, tmppanel(i), &ldwork);
		        	
			// Send the current panel back to the mics 
			// Has to be done with asynchronous copies
			for(j = 0; j < num_mics; j++)
			{  
				magma_setdevice(j);
				dPanel[j] = blkid%2 == 0 ? dPanel0[j] : dPanel1[j];
				if (j == panel_micnum) {
					panel[j] = dlA(j, i, i_local);
				}
				else
					panel[j] = dPanel[j];
				
				//send V
				magma_zsetmatrix_async(rows, pnb, tmppanel(i), 0, ldwork, panel[j], 0, ldda, queue, &event[j][0]);

				//send T
				dT[j] = blkid%2 == 0 ? dT0[j] : dT1[j];
				magma_zsetmatrix_async(pnb, pnb, hT, 0, pnb, dT[j], 0, pnb, queue, &event[j][0]);
				
			}

			
			for(j = 0; j < num_mics; j++)
			{
				magma_setdevice(j);
				magma_event_sync( event[j][0] );
			}


			//send previous R
			if(i > 0) {
   			        int previ = i-pnb;
				la_mic = (previ / nb) % num_mics;
			        int i_loc = ((previ / nb) / num_mics)*nb + previ%nb;
				magma_setdevice(la_mic);
				magma_kernel_sync( );
				//send R
				magma_zsetmatrix_async( old_pnb, old_pnb,
						htmpR(blkid - 1), 0, pnb,
						dlA(la_mic, previ, i_loc),   0, ldda, queue, &event[la_mic][0] );

				magma_event_sync( event[la_mic][0] );
			}

			if ((i + pnb + pnb) < k) 
			{
				la_mic = ((i + pnb) / nb) % num_mics;
				int nexti = i+pnb;
			        int i_loc = ((nexti / nb) / num_mics)*nb + nexti%nb;
				int nmic = 0;

				//Apply H' to A(i:m, i + 2 * nb:n) from the left; this is the look-ahead
				//application to the trailing matrix        
				magma_setdevice(la_mic);
				magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
						rows, pnb, pnb,
						panel[la_mic], 0, ldda, dT[la_mic],  0, pnb,
						dlA(la_mic, i, i_loc), 0, ldda, dwork[la_mic], 0, lddwork, 
						dworkvt[la_mic], 0, lddworkvt, queue);
	
				//Apply H' to A(i:m,i+nb:i+2*nb) from the left;
				//This is update for the next panel; part of the look-ahead  
				for(j = (la_mic + 1) % num_mics; nmic < num_mics; nmic++, j = (j + 1) % num_mics){
					magma_setdevice(j);
					magma_event_sync( event[j][0] );
					magma_int_t ncols, j_mic, myiloc;

					if (j == la_mic) {
						j_mic = i_loc + pnb; //it should be + nextpnb but next pnb is equal to pnb
						magma_kernel_sync( );
					}else if (j == panel_micnum){
                                                j_mic =  i_local + pnb;
					}else if (j < panel_micnum){
                                                j_mic =  i_local - blockoffset + nb;
					}else {
						j_mic  = i_local - blockoffset;
					}

					ncols  = n_local[j] - j_mic;
					if(ncols > 0) {
						magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
								rows, ncols, pnb,
								panel[j], 0, ldda, dT[j],  0, pnb,
								dlA(j, i, j_mic), 0, ldda, dwork[j], 0, lddwork, 
								dworkvt[j], 0, lddworkvt, queue);
					} 
				}     
			}
			else {
				//do the entire update as we exit and there would be no lookahead
				la_mic = (panel_micnum + 1) % num_mics;
				int nexti = i+pnb;
			        int i_loc = ((nexti / nb) / num_mics)*nb + nexti%nb;

				magma_setdevice(la_mic);
				magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
						rows, n_local[la_mic] - i_loc, pnb,
						panel[la_mic], 0, ldda, dT[la_mic],    0, pnb,
						dlA(la_mic, i, i_loc), 0, ldda, dwork[la_mic], 0, lddwork, 
						dworkvt[la_mic], 0, lddworkvt, queue);

				magma_setdevice(panel_micnum);
				magma_zsetmatrix( pnb, pnb,
						htmpR(blkid), 0, pnb,
						dlA(panel_micnum, i, i_local), 0, ldda, queue );
			}

			old_i  = i;
			old_rows = rows;
			old_pnb = pnb;
		}
	} else {
		i = 0;
	}

	for(j = 0; j < num_mics; j++){
		magma_setdevice(j);
		magma_free( dwork[j] );
	}

	// Use unblocked code to factor the last or only block 
	if (i < k) {
		ib   = n - i;
		rows = m - i;
		lhwork = lwork - rows * ib;

		panel_micnum = (panel_micnum + 1) % num_mics;
		int i_loc = (i) / (nb * num_mics) * nb;

		magma_setdevice(panel_micnum);
		magma_zgetmatrix( rows, ib,
				dlA(panel_micnum, i, i_loc), 0, ldda,
				tmppanel0, 0, rows, queue);

		lhqrwork = lwork - rows * ib;
		lapackf77_zgeqrf(&rows, &ib, tmppanel0, &rows, tau + i, tmppanel0 + ib * rows, &lhqrwork, info);

		magma_zsetmatrix( rows, ib,
				tmppanel0, 0,                      rows,
				dlA(panel_micnum, i, i_loc), 0, ldda, queue);
	}

	magma_setdevice(0);
	magma_free_pinned(work);

	return *info;
} /* magma_zgeqrf2_mmic */
