/*
   -- micMAGMA (version 0.1) --
   Univ. of Tennessee, Knoxville
   Univ. of California, Berkeley
   Univ. of Colorado, Denver
   April 2012

 @author Khairul Kabir
 @author Azzam Haidar

 @precisions normal z -> s d c

*/

#include <stdio.h>
#include "common_magma.h"

//#define ENABLE_TIMER
magma_err_t
magma_zgeqrf2_mic( magma_int_t m, magma_int_t n,
                magmaDoubleComplex_ptr dA, size_t dA_offset, magma_int_t ldda,
                magmaDoubleComplex *tau, magma_err_t *info,
                magma_queue_t queue)
{
        /*  -- micMAGMA (version 0.1) --
                Univ. of Tennessee, Knoxville
                Univ. of California, Berkeley
                Univ. of Colorado, Denver
                April 2012

                Purpose
                =======
                ZGEQRF computes a QR factorization of a complex M-by-N matrix A:
                A = Q * R.

                Arguments
                =========
                M       (input) INTEGER
                The number of rows of the matrix A.  M >= 0.

                N       (input) INTEGER
                The number of columns of the matrix A.  N >= 0.

                dA      (input/output) COMPLEX_16 array on the GPU, dimension (LDDA,N)
                On entry, the M-by-N matrix dA.
                On exit, the elements on and above the diagonal of the array
                contain the min(M,N)-by-N upper trapezoidal matrix R (R is
                upper triangular if m >= n); the elements below the diagonal,
                with the array TAU, represent the orthogonal matrix Q as a
                product of min(m,n) elementary reflectors (see Further
                Details).

                LDDA    (input) INTEGER
                The leading dimension of the array dA.  LDDA >= max(1,M).
                To benefit from coalescent memory accesses LDDA must be
                dividable by 16.

                TAU     (output) COMPLEX_16 array, dimension (min(M,N))
                The scalar factors of the elementary reflectors (see Further
                Details).

                INFO    (output) INTEGER
                = 0:  successful exit
                < 0:  if INFO = -i, the i-th argument had an illegal value
                if INFO = -9, internal GPU memory allocation failed.

                Further Details
                ===============

                The matrix Q is represented as a product of elementary reflectors

                Q = H(1) H(2) . . . H(k), where k = min(m,n).

                Each H(i) has the form

                H(i) = I - tau * v * v'

                where tau is a complex scalar, and v is a complex vector with
                v(1:i-1) = 0 and v(i) = 1; v(i+1:m) is stored on exit in A(i+1:m,i),
                and tau in TAU(i).
                =====================================================================    */

        #define dA(a_1, a_2)    dA, (dA_offset + (a_1) + (a_2)*(ldda))
        #define work_ref(a_1)  ( work + (a_1))
        #define hwork          ( work + (nb) * (m))

        magmaDoubleComplex_ptr dwork;
        magmaDoubleComplex_ptr dworkvt;
        magmaDoubleComplex  *work, *dT0,*dT1, *dT;

        magma_int_t i, k, ldwork, lddwork, old_i, old_ib, rows, blkid;
        magma_int_t lddworkvt;
        magma_int_t nbmin, nx, ib, nb;
        magma_int_t lhwork, lwork;

        *info = 0;
        if (m < 0) {
                *info = -1;
        } else if (n < 0) {
                *info = -2;
        } else if (ldda < max(1, m)) {
                *info = -4;
        }
        if (*info != 0) {
                magma_xerbla( __func__, -(*info) );
                return *info;
        }

        k = min(m, n);
        if (k == 0)
                return MAGMA_SUCCESS;

        //if(m >= 17000)
        //        nb = 1024;
        //else
                nb = 512;

        lwork  = (m + n) * nb;
        lhwork = lwork - (m) * nb;


        if ( MAGMA_SUCCESS != magma_zmalloc( &dwork, (n*nb + 2*nb*nb + min(m, n)*nb) )) {
                *info = MAGMA_ERR_DEVICE_ALLOC;
                return *info;
        }

        if ( MAGMA_SUCCESS != magma_zmalloc_pinned( &work, lwork ) ) {
                *info = MAGMA_ERR_HOST_ALLOC;
                magma_free( dwork );
                return *info;
        }

        magma_event_t event[2] = {NULL, NULL};                                                            

        nbmin = 2;
        nx    = 2 * nb;
        ldwork = m;
        lddwork= n;
        lddworkvt= min(m, n);
        dT0 = dwork + n*nb;
        dT1 = dT0 + nb*nb;
	dworkvt = dwork + n*nb + 2*nb*nb; 

	double trfb, tmtc, tqr, trft, tctmr, tctmv, tctmt, tnp;

        if (nb >= nbmin && nb < k && nx < k) {
                #if defined (ENABLE_TIMER)
                printf("\n\n %10s %10s %10s  %10s %10s %10s %10s  %10s  %10s %10s  %10s  %10s %10s\n", 
                                "i", "row", "panel", "dlarft", "QR+T", "CPU", "dlarfb", "nxtpU", "ctm R", "ctm V", "ctm T", "ctm","mtc");
                printf("     ====================================================================================================\n");
                #endif 

                /* Use blocked code initially */
                old_i = 0; old_ib = min(k, nb);;
                for (i = 0; i < k - nx; i += nb) {
			blkid = i/nb; 
                        ib    = min(k - i, nb);
                        rows  = m - i;
                        magma_kernel_sync();

                        #if defined (ENABLE_TIMER)
                        trfb = tmtc = tqr = trft = tctmr = tctmv = tctmt = tnp = 0.0;
                        #endif

                        if (i>0){
                                /* Apply H' to A(i:m,i+2*ib:n) from the left */
                                #if defined (ENABLE_TIMER)
                                trfb = get_time();
                                #endif
				dT = (blkid-1)%2 == 0 ? dT0 : dT1;
                                magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
                                                   m - old_i, n - old_i - 2 * old_ib, old_ib,
                                                   dA(old_i, old_i), ldda, 
						   dT, 0, nb,
                                                   dA(old_i, old_i + 2 * old_ib), ldda, dwork, 0, 
						   m-old_i, dworkvt, 0, lddworkvt, queue);
                                #if defined (ENABLE_TIMER)
                                magma_kernel_sync();
                                trfb = get_time() - trfb;
                                #endif
                        }
                        #if defined (ENABLE_TIMER)
                        tmtc = get_time();
                        #endif

                        magma_zgetmatrix_async(rows, ib, dA(i, i), ldda, work_ref(i), 0, ldwork, queue, &event[0]);
                        magma_event_sync(event[0]);

                            #if defined (ENABLE_TIMER)
                        tmtc = get_time() - tmtc;
                        tqr = get_time();
                            #endif

                        lapackf77_zgeqrf(&rows, &ib, work_ref(i), &ldwork, tau+i, hwork, &lhwork, info);
                        
                        #if defined (ENABLE_TIMER)
                        tqr = get_time() - tqr;
                        #endif

                        /* Form the triangular factor of the block reflector
                           H = H(i) H(i+1) . . . H(i+ib-1) */

                        #if defined (ENABLE_TIMER)
                        trft = get_time();
                        #endif 

                        memset(hwork,0,ib*ib*sizeof(magmaDoubleComplex));
                        lapackf77_zlarft( MagmaForwardStr, MagmaColumnwiseStr, 
                                        &rows, &ib, 
                                        work_ref(i), &ldwork, tau+i, hwork, &ib);

                        #if defined (ENABLE_TIMER)
                        trft = get_time() - trft;
                        tctmv = get_time();
                        #endif

			// ======================================================
			//            SEND V and T to MIC
			// ======================================================
			// store the upper part of V (the R) then put zero 
			// on the upper part of V send it and then put back the R
                        zpanel_to_q( MagmaUpper, ib, work_ref(i), ldwork, hwork+ib*ib );
                        magma_zsetmatrix_async(rows, ib, work_ref(i), 0, ldwork, dA(i,i), ldda, queue, &event[0]);
                        magma_event_sync(event[0]);
                        zq_to_panel( MagmaUpper, ib, work_ref(i), ldwork, hwork+ib*ib ); // could be moved to down where it is commented
                        // sending T 
			dT = blkid%2 == 0 ? dT0 : dT1;
                        magma_zsetmatrix_async(ib, ib, hwork, 0, ib, dT, 0, nb, queue, &event[0]);
                        magma_event_sync(event[0]);
                        //zq_to_panel( MagmaUpper, ib, work_ref(i), ldwork, hwork+ib*ib );

                        #if defined (ENABLE_TIMER)
                        tctmv = get_time() - tctmv;
                        #endif

                        /* Wait the dlarfb of step old_i has been finished, then 
                         * send the upper portion of V (which is R) of step old_i 
                         * to be stored in its place in dA */
                        if (i > 0) {
                                magma_kernel_sync();

                                #if defined (ENABLE_TIMER)
                                tctmr = get_time();
                                #endif
                                // sending R of previous step i_old
                                magma_zsetmatrix_async( old_ib, old_ib, work_ref(old_i), 0, ldwork,
                                                dA(old_i, old_i), ldda, queue, &event[1]); 
                                magma_event_sync(event[1]);

                                #if defined (ENABLE_TIMER)
                                tctmr = get_time() - tctmr;
                                #endif
                        }

                        if (i + ib < n) 
                        {
                                #if defined (ENABLE_TIMER)
                                tnp = get_time();
                                #endif

                                if (i+nb < k-nx) {

                                        /* Apply H' to A(i:m,i+ib:i+2*ib) from the left: next panel */
                                        magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
                                                        rows, ib, ib, 
                                                        dA(i, i   ), ldda, dT, 0,  nb, 
                                                        dA(i, i+ib), ldda, dwork, 0, rows, dworkvt, 0, lddworkvt, queue);
                                }
                                else {
                                        /* Apply H' form left: last step in the loop, apply H' to the last corner (last two block) of the
                                         * matrix because this loop is finished and then do not go through the main dlafrb
                                         * defined on the top of the loop. Note that the last two block is done on CPU -A */  
                                        magma_zlarfb_gemm( MagmaLeft, MagmaConjTrans, MagmaForward, MagmaColumnwise,
                                                        rows, n-i-ib, ib, 
                                                        dA(i, i   ), ldda, dT, 0,  nb, 
                                                        dA(i, i+ib), ldda, dwork, 0, rows, dworkvt, 0, lddworkvt, queue);

                                        magma_zsetmatrix(ib, ib, work_ref(i), 0, ldwork, dA(i,i), ldda, queue);
                                
                                }
                                old_i  = i;
                                old_ib = ib;

                                #if defined (ENABLE_TIMER)
                                magma_kernel_sync();
                                tnp = get_time() - tnp;
                                #endif
                        }
                            #if defined (ENABLE_TIMER)
                        printf("%10d %10d %10.3lf  %10.3lf  %10.3lf %10.3lf %10.3lf  %10.3lf  %10.3lf %10.3lf %10.3lf %10.3lf %10.3lf\n",
                                               i, rows, tqr * 1000, trft*1000, (trft+tqr)*1000, (trft+tqr+tmtc)*1000,  
                                        trfb * 1000, tnp * 1000,  tctmr * 1000, tctmv * 1000, tctmt * 1000,
                                        (tctmv+tctmt+tctmr)*1000, tmtc * 1000);
                            #endif
                }
        } else {
                i = 0;
        }

        magma_free(dwork);
        
        //printf("\n\n");


        /* Use unblocked code to factor the last or only block. */
        if (i < k) {
                ib   = n-i;
                rows = m-i;
                magma_zgetmatrix(rows, ib, dA(i, i), ldda, work, 0, rows, queue);

                lhwork = lwork - rows*ib;
                lapackf77_zgeqrf(&rows, &ib, work, &rows, tau+i, work+ib*rows, &lhwork, info);

                magma_zsetmatrix(rows, ib, work, 0, rows, dA(i, i), ldda, queue);
        }

        magma_free_pinned(work);

        return *info;
} /* magma_zgeqrf2_mic */
