@inproceedings {,
	title = {Evaluating PaRSEC Through Matrix Computations in Scientific Applications},
	volume = {14626},
	year = {2024},
	pages = {22{\textendash}33},
	publisher = {Springer},
	doi = {10.1007/978-3-031-61763-8_3},
	url = {https://doi.org/10.1007/978-3-031-61763-8_3},
	author = {Qinglei Cao and Thomas Herault and Aurelien Bouteiller and Joseph Schuchart and George Bosilca},
	editor = {Patrick Diehl and Joseph Schuchart and Pedro Valero-Lara and George Bosilca}
}
@booklet {,
	title = {Interface for Sparse Linear Algebra Operations},
	year = {2024},
	month = {2024-11},
	abstract = {The standardization of an interface for dense linear algebra operations in the BLAS standard has enabled interoperability between different linear algebra libraries, thereby boosting the success of scientific computing, in particular in scientific HPC. Despite numerous efforts in the past, the community has not yet agreed on a standardization for sparse linear algebra operations due to numerous reasons. One is the fact that sparse linear algebra objects allow for many different storage formats, and different hardware may favor different storage formats. This makes the definition of a FORTRAN-style all-circumventing interface extremely challenging. Another reason is that opposed to dense linear algebra functionality, in sparse linear algebra, the size of the sparse data structure for the operation result is not always known prior to the information. Furthermore, as opposed to the standardization effort for dense linear algebra, we are late in the technology readiness cycle, and many production-ready software libraries using sparse linear algebra routines have implemented and committed to their own sparse BLAS interface. At the same time, there exists a demand for standardization that would improve interoperability, and sustainability, and allow for easier integration of building blocks. In an inclusive, cross-institutional effort involving numerous academic institutions, US National Labs, and industry, we spent two years designing a hardware-portable interface for basic sparse linear algebra functionality that serves the user needs and is compatible with the different interfaces currently used by different vendors. In this paper, we present a C++ API for sparse linear algebra functionality, discuss the design choices, and detail how software developers preserve a lot of freedom in terms of how to implement functionality behind this API.},
	doi = {10.48550/arXiv.2411.13259},
	url = {https://arxiv.org/abs/2411.13259},
	author = {Ahmad Abdelfattah and Willow Ahrens and Hartwig Anzt and Chris Armstrong and Ben Brock and Aydin Buluc and Federico Busato and Terry Cojean and Tim Davis and Jim Demmel and Grace Dinh and David Gardener and Jan Fiala and Mark Gates and Azzam Haider and Toshiyuki Imamura and Pedro Valero Lara and Jose Moreira and Sherry Li and Piotr Luszczek and Max Melichenko and Jose Moeira and Yvan Mokwinski and Riley Murray and Spencer Patty and Slaven Peles and Tobias Ribizel and Jason Riedy and Siva Rajamanickam and Piyush Sao and Manu Shantharam and Keita Teranishi and Stan Tomov and Yu-Hsiang Tsai and Heiko Weichelt}
}
@article {,
	title = {Multi-GPU work sharing in a task-based dataflow programming model},
	journal = {Future Generation Computer Systems},
	volume = {156},
	year = {2024},
	month = {2024-07},
	pages = {313 - 324},
	issn = {0167739X},
	doi = {10.1016/j.future.2024.03.017},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S0167739X24000888},
	author = {John, Joseph and Milthorpe, Josh and Herault, Thomas and Bosilca, George}
}
@article {,
	title = {PaRSEC: Scalability, flexibility, and hybrid architecture support for task-based applications in ECP},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2024},
	month = {2024-10},
	abstract = {This paper highlights the most significant enhancements made to PaRSEC, a scalable task-based runtime system designed for hybrid machines, during the Exascale Computing Project (ECP). The enhancements focus on expanding the capabilities of PaRSEC to address the evolving landscape of parallel computing. Notable achievements include the integration of support for three major types of accelerators (NVIDIA, AMD, and Intel GPUs), the refinement and increased flexibility of the communication subsystem, and the introduction of new programming interfaces tailored for irregular applications. Additionally, the project resulted in the development of powerful debugging and performance analysis tools aimed at assisting users in understanding and optimizing their applications. We present a comprehensive demonstration of these advancements through a series of benchmarks and applications within ECP and beyond, thereby showcasing the enhanced capabilities of PaRSEC across the diverse architectures within the ECP, providing valuable insights into the runtime system{\textquoteright}s adaptability and performance across varied computing environments.},
	issn = {1094-3420},
	doi = {10.1177/10943420241290520},
	url = {https://journals.sagepub.com/doi/10.1177/10943420241290520},
	author = {Bouteiller, Aur{\'e}lien and Herault, Thomas and Cao, Qinglei and Schuchart, Joseph and Bosilca, George}
}
@article {,
	title = {A survey on checkpointing strategies: Should we always checkpoint {\`a} la Young/Daly?},
	journal = {Future Generation Computer Systems},
	year = {2024},
	month = {2024-07},
	abstract = {The Young/Daly formula provides an approximation of the optimal checkpointing period for a parallel application executing on a supercomputing platform. It was originally designed to handle fail-stop errors for preemptible tightly-coupled applications, but has been extended to other application and resilience frameworks. We provide some background and survey various scenarios to assess the usefulness and limitations of the formula, both for preemptible applications and workflow applications represented as a graph of tasks. We also discuss scenarios with uncertainties, and extend the study to silent errors. We exhibit cases where the optimal period is of a different order than that dictated by the Young/Daly formula, and finally we explain how checkpointing can be further combined with replication.},
	issn = {0167739X},
	doi = {10.1016/j.future.2024.07.022},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S0167739X24003777},
	author = {Bautista-Gomez, Leonardo and Benoit, Anne and Di, Sheng and Herault, Thomas and Robert, Yves and Sun, Hongyang}
}
@article {,
	title = {Then and Now: Improving Software Portability, Productivity, and 100{\texttimes} Performance},
	journal = {Computing in Science \& Engineering},
	year = {2024},
	month = {2024-04},
	pages = {1 - 10},
	abstract = {The US Exascale Computing Project (ECP) has succeeded in preparing applications to run efficiently on the first reported Exascale supercomputers in the world. To achieve this, it modernized the whole leadership software stack, from libraries to simulation codes. In this article, we contrast selected leadership software before and after ECP. We discuss how sustainable research software development for leadership computing can embrace the conversation with the hardware vendors, the leadership computing facilities, the software community, and the domain scientists who are the application developers and integrators of software products. We elaborate on how software needs to take portability as a central design principle and to benefit from interdependent teams; we also demonstrate how moving to programming languages with high momentum, like modern C++, can help improve the sustainability, interoperability, and performance of research software. Finally, we showcase how cross-institutional efforts can enable algorithm advances that are beyond incremental performance optimization.},
	issn = {1521-9615},
	doi = {10.1109/MCSE.2024.3387302},
	url = {https://ieeexplore.ieee.org/document/10495889/},
	author = {Anzt, Hartwig and Huebl, Axel and Li, Xiaoye S}
}
@article {,
	title = {XaaS: Acceleration as a Service to Enable Productive High-Performance Cloud Computing},
	journal = {Computing in Science \& Engineering},
	volume = {26},
	year = {2024},
	month = {2024-07},
	pages = {40 - 51},
	abstract = {High-performance computing (HPC) and the cloud have evolved independently, specializing their innovations into performance or productivity. Acceleration as a Service (XaaS) is a recipe to empower both fields with a shared execution platform that provides transparent access to computing resources, regardless of the underlying cloud or HPC service provider. Bridging HPC and cloud advancements, XaaS presents a unified architecture built on performance-portable containers. Our converged model concentrates on low-overhead, high-performance communication and computing, targeting resource-intensive workloads from climate simulations to machine learning. XaaS lifts the restricted allocation model of Function as a Service (FaaS), allowing users to benefit from the flexibility and efficient resource utilization of serverless computing while supporting long-running and performance-sensitive workloads from HPC.},
	issn = {1521-9615},
	doi = {10.1109/MCSE.2024.3382154},
	url = {https://ieeexplore.ieee.org/document/10489911/http://xplorestaging.ieee.org/ielx7/5992/10770761/10489911.pdf?arnumber=10489911},
	author = {Hoefler, Torsten and Copik, Marcin and Beckman, Pete and Jones, Andrew and Foster, Ian and Parashar, Manish and Reed, Daniel and Troyer, Matthias and Schulthess, Thomas and Ernst, Daniel and Dongarra, Jack}
}
@conference {,
	title = {AI Benchmarking for Science: Efforts from the MLCommons Science Working Group},
	booktitle = {Lecture Notes in Computer Science},
	volume = {13387},
	year = {2023},
	month = {2023-01},
	pages = {47 - 64},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {With machine learning (ML) becoming a transformative tool for science, the scientific community needs a clear catalogue of ML techniques, and their relative benefits on various scientific problems, if they were to make significant advances in science using AI. Although this comes under the purview of benchmarking, conventional benchmarking initiatives are focused on performance, and as such, science, often becomes a secondary criteria.

In this paper, we describe a community effort from a working group, namely, MLCommons Science Working Group, in developing science-specific AI benchmarking for the international scientific community. Since the inception of the working group in 2020, the group has worked very collaboratively with a number of national laboratories, academic institutions and industries, across the world, and has developed four science-specific AI benchmarks. We will describe the overall process, the resulting benchmarks along with some initial results. We foresee that this initiative is likely to be very transformative for the AI for Science, and for performance-focused communities.},
	isbn = {978-3-031-23219-0},
	doi = {10.1007/978-3-031-23220-610.1007/978-3-031-23220-6_4},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-23220-6_4},
	author = {Thiyagalingam, Jeyan and von Laszewski, Gregor and Yin, Junqi and Emani, Murali and Papay, Juri and Barrett, Gregg and Luszczek, Piotr and Tsaris, Aristeidis and Kirkpatrick, Christine and Wang, Feiyi and Gibbs, Tom and Vishwanath, Venkatram and Shankar, Mallikarjun and Fox, Geoffrey and Hey, Tony},
	editor = {Anzt, Hartwig and Bienz, Amanda and Luszczek, Piotr and Baboulin, Marc}
}
@article {,
	title = {Direct Determination of Optimal Real-Space Orbitals for Correlated Electronic Structure of Molecules},
	journal = {Journal of Chemical Theory and Computation},
	volume = {19},
	year = {2023},
	month = {2023-10},
	pages = {7230 - 7241},
	abstract = {We demonstrate how to determine numerically nearly exact orthonormal orbitals that are optimal for the evaluation of the energy of arbitrary (correlated) states of atoms and molecules by minimization of the energy Lagrangian. Orbitals are expressed in real space using a multiresolution spectral element basis that is refined adaptively to achieve the user-specified target precision while avoiding the ill-conditioning issues that plague AO basis set expansions traditionally used for correlated models of molecular electronic structure. For light atoms, the orbital solver, in conjunction with a variational electronic structure model [selected Configuration Interaction (CI)] provides energies of comparable precision to a state-of-the-art atomic CI solver. The computed electronic energies of atoms and molecules are significantly more accurate than the counterparts obtained with the orbital sets of the same rank expanded in Gaussian AO bases, and can be determined even when linear dependence issues preclude the use of the AO bases. It is feasible to optimize more than 100 fully correlated numerical orbitals on a single computer node, and significant room exists for additional improvement. These findings suggest that real-space orbital representations might be the preferred alternative to AO representations for high-end models of correlated electronic states of molecules and materials.},
	issn = {1549-9618},
	doi = {10.1021/acs.jctc.3c00732},
	url = {https://pubs.acs.org/doi/10.1021/acs.jctc.3c00732},
	author = {Valeev, Edward F. and Harrison, Robert J. and Holmes, Adam A. and Peterson, Charles C. and Penchoff, Deborah A.}
}
@booklet {,
	title = {Earth Virtualization Engines - A Technical Perspective},
	year = {2023},
	month = {2023-09},
	abstract = {Participants of the Berlin Summit on Earth Virtualization Engines (EVEs) discussed ideas and concepts to improve our ability to cope with climate change. EVEs aim to provide interactive and accessible climate simulations and data for a wide range of users. They combine high-resolution physics-based models with machine learning techniques to improve the fidelity, efficiency, and interpretability of climate projections. At their core, EVEs offer a federated data layer that enables simple and fast access to exabyte-sized climate data through simple interfaces. In this article, we summarize the technical challenges and opportunities for developing EVEs, and argue that they are essential for addressing the consequences of climate change.},
	url = {https://arxiv.org/abs/2309.09002},
	author = {Torsten Hoefler and Bjorn Stevens and Andreas F. Prein and Johanna Baehr and Thomas Schulthess and Thomas F. Stocker and John Taylor and Daniel Klocke and Pekka Manninen and Piers M. Forster and Tobias K{\"o}lling and Nicolas Gruber and Hartwig Anzt and Claudia Frauen and Florian Ziemen and Milan Kl{\"o}wer and Karthik Kashinath and Christoph Sch{\"a}r and Oliver Fuhrer and Bryan N. Lawrence}
}
@article {,
	title = {Revisiting I/O bandwidth-sharing strategies for HPC applications},
	number = {RR-9502},
	year = {2023},
	month = {2023-03},
	institution = {INRIA},
	abstract = {This work revisits I/O bandwidth-sharing strategies for HPC applications. When several applications post concurrent I/O operations, well-known approaches include serializing these operations (First-Come First-Served) or fair-sharing the bandwidth across them (FairShare). Another recent approach, I/O-Sets, assigns priorities to the applications, which are classified into different sets based upon the average length of their iterations. We introduce several new bandwidth-sharing strategies, some of them simple greedy algorithms, and some of them more complicated to implement, and we compare them with existing ones. Our new strategies do not rely on any a-priori knowledge of the behavior of the applications, such as the length of work phases, the volume of I/O operations, or some expected periodicity. We introduce a rigorous framework, namely steady-state windows, which enables to derive bounds on the competitive ratio of all bandwidth-sharing strategies for three different objectives: minimum yield, platform utilization, and global efficiency. To the best of our knowledge, this work is the first to provide a quantitative assessment of the online competitiveness of any bandwidth-sharing strategy. This theory-oriented assessment is complemented by a comprehensive set of simulations, based upon both synthetic and realistic traces. The main conclusion is that our simple and low-complexity greedy strategies significantly outperform First-Come First-Served, FairShare and I/O-Sets, and we recommend that the I/O community implements them for further assessment.},
	keywords = {bandwidth sharing, HPC applications, I/O, scheduling strategy},
	url = {https://hal.inria.fr/hal-04038011},
	author = {Anne Benoit and Thomas Herault and Lucas Perotin and Yves Robert and Frederic Vivien}
}
@inproceedings {,
	title = {Synchronizing MPI Processes in Space and Time},
	year = {2023},
	month = {2023-09},
	publisher = {ACM},
	address = {Bristol, United Kingdom},
	abstract = {Performance benchmarks are an integral part of the development and evaluation of parallel algorithms, both in distributed applications as well as MPI implementations themselves. The initial step of the benchmark process is to obtain a common timestamp to mark the start of an operation across all involved processes, and the state-of-the-art in many applications and widely used MPI benchmark suites is the use of MPI barriers. In this paper, we show that the synchronization in space provided by an MPI_Barrier is insufficient for proper benchmark results of parallel distributed algorithms, using MPI collective operations as examples. The resulting lack of a global start timestamp for an operation leads to skewed results, with a significant impact of the used barrier algorithm. In order to mitigate these issues, we propose and discuss the implementation of MPIX_Harmonize, which extends the synchronization in space provided by MPI_Barrier with a time synchronization to guarantee a common starting timestamp across all involved processes. By replacing the use of MPI_Barrier with MPIX_Harmonize, benchmark implementors can eliminate skews resulting from barrier algorithms and achieve stable performance benchmark results. We will show that the proper time synchronization can have significant impact on the benchmark results for various implementations of MPI_Allreduce, MPI_Reduce, and MPI_Bcast.},
	isbn = {9798400709135},
	doi = {10.1145/3615318.3615325},
	url = {https://dl.acm.org/doi/proceedings/10.1145/3615318},
	author = {Schuchart, Joseph and Hunold, Sascha and Bosilca, George}
}
@conference {,
	title = {When to checkpoint at the end of a fixed-length reservation?},
	booktitle = {Fault Tolerance for HPC at eXtreme Scales (FTXS) Workshop},
	year = {2023},
	month = {2023-08},
	address = {Denver, United States},
	abstract = {This work considers an application executing for a fixed duration, namely the length of the reservation that it has been granted. The checkpoint duration is a stochastic random variable that obeys some well-known probability distribution law. The question is when to take a checkpoint towards the end of the execution, so that the expectation of the work done is maximized. We address two scenarios. In the first scenario, a checkpoint can be taken at any time; despite its simplicity, this natural problem has not been considered yet (to the best of our knowledge). We provide the optimal solution for a variety of probability distribution laws modeling checkpoint duration. The second scenario is more involved: the application is a linear workflow consisting of a chain of tasks with IID stochastic execution times, and a checkpoint can be taken only at the end of a task. First, we introduce a static strategy where we compute the optimal number of tasks before the application checkpoints at the beginning of the execution. Then, we design a dynamic strategy that decides whether to checkpoint or to continue executing at the end of each task. We instantiate this second scenario with several examples of probability distribution laws for task durations.},
	url = {https://inria.hal.science/hal-04215554},
	author = {Quentin Barbut and Anne Benoit and Thomas Herault and Yves Robert and Frederic Vivien}
}
@inproceedings {,
	title = {Checkpointing {\`a} la Young/Daly: An Overview},
	year = {2022},
	month = {2022-08},
	pages = {701-710},
	publisher = {ACM Press},
	address = {Noida, India},
	abstract = {The Young/Daly formula provides an approximation of the optimal checkpoint period for a parallel application executing on a supercomputing platform. The Young/Daly formula was originally designed for preemptible tightly-coupled applications. We provide some background and survey various application scenarios to assess the usefulness and limitations of the formula.},
	isbn = {9781450396752},
	doi = {10.1145/3549206},
	url = {https://dl.acm.org/doi/fullHtml/10.1145/3549206.3549328},
	author = {Anne Benoit and Yishu Du and Thomas Herault and Loris Marchal and Guillaume Pallez and Lucas Perotin and Yves Robert and Hongyang Sun and Frederic Vivien}
}
@article {,
	title = {Comparing Distributed Termination Detection Algorithms for Modern HPC Platforms},
	journal = {International Journal of Networking and Computing},
	volume = {12},
	year = {2022},
	month = {2022-01},
	pages = {26 - 46},
	abstract = {This paper revisits distributed termination detection algorithms in the context of High-Performance Computing (HPC) applications. We introduce an efficient variant of the Credit Distribution Algorithm (CDA) and compare it to the original algorithm (HCDA) as well as to its two primary competitors: the Four Counters algorithm (4C) and the Efficient Delay-Optimal Distributed algorithm (EDOD). We analyze the behavior of each algorithm for some simplified task-based kernels and show the superiority of CDA in terms of the number of control messages. We then compare the implementation of these algorithms over a task-based runtime system, PaRSEC and show the advantages and limitations of each approach in a real implementation.},
	issn = {2185-2839},
	doi = {10.15803/ijnc.12.1_26},
	url = {https://www.jstage.jst.go.jp/article/ijnc/12/1/12_26/_article},
	author = {George Bosilca and Bouteiller, Aur{\'e}lien and Herault, Thomas and Le F{\`e}vre, Valentin and Robert, Yves and Dongarra, Jack}
}
@conference {,
	title = {Composition of Algorithmic Building Blocks in Template Task Graphs},
	booktitle = {2022 IEEE/ACM Parallel Applications Workshop: Alternatives To MPI+X (PAW-ATM)},
	year = {2022},
	month = {2023-01},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Dallas, TX, USA},
	doi = {10.1109/PAW-ATM56565.2022.00008},
	url = {https://ieeexplore.ieee.org/document/10024647/},
	author = {Herault, Thomas and Schuchart, Joseph and Valeev, Edward F. and George Bosilca}
}
@article {,
	title = {Evaluations of molecular modeling and machine learning for predictive capabilities in binding of lanthanum and actinium with carboxylic acids},
	journal = {Journal of Radioanalytical and Nuclear Chemistry},
	year = {2022},
	month = {2022-12},
	issn = {0236-5731},
	doi = {10.1007/s10967-022-08620-7},
	url = {https://rdcu.be/c2lGj},
	author = {Penchoff, Deborah A. and Peterson, Charles C. and Wrancher, Eleigha M. and George Bosilca and Harrison, Robert J. and Valeev, Edward F. and Benny, Paul D.}
}
@conference {,
	title = {Generalized Flow-Graph Programming Using Template Task-Graphs: Initial Implementation and Assessment},
	booktitle = {2022 IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2022},
	month = {2022-07},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Lyon, France},
	abstract = {We present and evaluate TTG, a novel programming model and its C++ implementation that by marrying the ideas of control and data flowgraph programming supports compact specification and efficient distributed execution of dynamic and irregular applications. Programming interfaces that support task-based execution often only support shared memory parallel environments; a few support distributed memory environments, either by discovering the entire DAG of tasks on all processes, or by introducing explicit communications. The first approach limits scalability, while the second increases the complexity of programming. We demonstrate how TTG can address these issues without sacrificing scalability or programmability by providing higher-level abstractions than conventionally provided by task-centric programming systems, without impeding the ability of these runtimes to manage task creation and execution as well as data and resource management efficiently. TTG supports distributed memory execution over 2 different task runtimes, PaRSEC and MADNESS. Performance of four paradigmatic applications (in graph analytics, dense and block-sparse linear algebra, and numerical integrodifferential calculus) with various degrees of irregularity implemented in TTG is illustrated on large distributed-memory platforms and compared to the state-of-the-art implementations.},
	doi = {10.1109/IPDPS53621.2022.00086},
	url = {https://ieeexplore.ieee.org/abstract/document/9820613},
	author = {Schuchart, Joseph and Nookala, Poornima and Javanmard, Mohammad Mahdi and Herault, Thomas and Valeev, Edward F. and George Bosilca and Harrison, Robert J.}
}
@article {,
	title = {OpenMP application experiences: Porting to accelerated nodes},
	journal = {Parallel Computing},
	volume = {109},
	year = {2022},
	month = {2022-03},
	abstract = {As recent enhancements to the OpenMP specification become available in its implementations, there is a need to share the results of experimentation in order to better understand the OpenMP implementation{\textquoteright}s behavior in practice, to identify pitfalls, and to learn how the implementations can be effectively deployed in scientific codes. We report on experiences gained and practices adopted when using OpenMP to port a variety of ECP applications, mini-apps and libraries based on different computational motifs to accelerator-based leadership-class high-performance supercomputer systems at the United States Department of Energy. Additionally, we identify important challenges and open problems related to the deployment of OpenMP. Through our report of experiences, we find that OpenMP implementations are successful on current supercomputing platforms and that OpenMP is a promising programming model to use for applications to be run on emerging and future platforms with accelerated nodes.},
	issn = {01678191},
	doi = {10.1016/j.parco.2021.102856},
	url = {https://www.sciencedirect.com/science/article/pii/S0167819121001009},
	author = {Bak, Seonmyeong and Bertoni, Colleen and Boehm, Swen and Budiardja, Reuben and Chapman, Barbara M. and Doerfert, Johannes and Eisenbach, Markus and Finkel, Hal and Hernandez, Oscar and Huber, Joseph and Iwasaki, Shintaro and Kale, Vivek and Kent, Paul R.C. and Kwack, JaeHyuk and Lin, Meifeng and Luszczek, Piotr and Luo, Ye and Pham, Buu and Pophale, Swaroop and Ravikumar, Kiran and Sarkar, Vivek and Scogland, Thomas and Tian, Shilei and Yeung, P.K.}
}
@conference {,
	title = {Performance Analysis of Parallel FFT on Large Multi-GPU Systems},
	booktitle = {2022 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2022},
	month = {2022-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Lyon, France},
	doi = {10.1109/IPDPSW55747.2022.00072},
	url = {https://ieeexplore.ieee.org/document/9835388/},
	author = {Ayala, Alan and Stanimire Tomov and Stoyanov, Miroslav and Haidar, Azzam and Dongarra, Jack}
}
@inproceedings {,
	title = {Porting Sparse Linear Algebra to~Intel GPUs},
	volume = {13098},
	year = {2022},
	month = {2022-06},
	pages = {57 - 68},
	publisher = {Springer International Publishing},
	address = {Lisbon, Portugal},
	abstract = {With discrete Intel GPUs entering the high performance computing landscape, there is an urgent need for production-ready software stacks for these platforms. In this paper, we report how we prepare the Ginkgo math library for Intel GPUs by developing a kernel backed based on the DPC++ programming environment. We discuss conceptual differences to the CUDA and HIP programming models and describe workflows for simplified code conversion. We benchmark advanced sparse linear algebra routines utilizing the converted kernels to assess the efficiency of the DPC++ backend in the hardware-specific performance bounds. We compare the performance of basic building blocks against routines providing the same functionality that ship with Intel{\textquoteright}s oneMKL vendor library.},
	keywords = {Ginkgo, Intel GPUs, math library, oneAPI, SpMV},
	isbn = {978-3-031-06155-4},
	doi = {10.1007/978-3-031-06156-1_5},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-06156-1_5},
	author = {Tsai, Yuhsiang M. and Cojean, Terry and Anzt, Hartwig},
	editor = {Chaves, Ricardo and B. Heras, Dora and Ilic, Aleksandar and Unat, Didem and Badia, Rosa M. and Bracciali, Andrea and Diehl, Patrick and Dubey, Anshu and Sangyoon, Oh and L. Scott, Stephen and Ricci, Laura}
}
@conference {,
	title = {Pushing the Boundaries of Small Tasks: Scalable Low-Overhead Data-Flow Programming in TTG},
	booktitle = {2022 IEEE International Conference on Cluster Computing (CLUSTER)},
	year = {2022},
	month = {2022-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Heidelberg, Germany},
	abstract = {Shared memory parallel programming models strive to provide low-overhead execution environments. Task-based programming models, in particular, are well-suited to cope with the ubiquitous multi- and many-core systems since they allow applications to express all available concurrency to a scheduler, which is tasked with exploiting the available hardware resources. It is general consensus that atomic operations should be preferred over locks and mutexes to avoid inter-thread serialization and the resulting loss in efficiency. However, even atomic operations may serialize threads if not used judiciously. In this work, we will discuss several optimizations applied to TTG and the underlying PaRSEC runtime system aiming at removing contentious atomic operations to reduce the overhead of task management to a few hundred clock cycles. The result is an optimized data-flow programming system that seamlessly scales from a single node to distributed execution and which is able to compete with OpenMP in shared memory.},
	keywords = {Dataflow graph, Hardware, Instruction sets, Memory management, PaR-SEC, parallel programming, runtime, scalability, Task analysis, task-based programming, Template Task Graph, TTG},
	doi = {10.1109/CLUSTER51413.2022.00026},
	url = {https://ieeexplore.ieee.org/document/9912704/},
	author = {Schuchart, Joseph and Nookala, Poornima and Herault, Thomas and Valeev, Edward F. and George Bosilca}
}
@article {,
	title = {Resiliency in numerical algorithm design for extreme scale simulations},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {36371337212766180823},
	year = {2022},
	month = {2022-03},
	pages = {251 - 285},
	keywords = {Fault tolerance, Numerical algorithms, parallel computer architecture, resilience},
	issn = {1094-3420},
	doi = {10.1177/10943420211055188},
	url = {http://journals.sagepub.com/doi/10.1177/10943420211055188http://journals.sagepub.com/doi/pdf/10.1177/10943420211055188http://journals.sagepub.com/doi/pdf/10.1177/10943420211055188http://journals.sagepub.com/doi/full-xml/10.1177/10943420211055188},
	author = {Agullo, Emmanuel and Altenbernd, Mirco and Anzt, Hartwig and Bautista-Gomez, Leonardo and Benacchio, Tommaso and Bonaventura, Luca and Bungartz, Hans-Joachim and Chatterjee, Sanjay and Ciorba, Florina M and DeBardeleben, Nathan and Drzisga, Daniel and Eibl, Sebastian and Engelmann, Christian and Gansterer, Wilfried N and Giraud, Luc and G{\"o}ddeke, Dominik and Heisig, Marco and J{\'e}z{\'e}quel, Fabienne and Kohl, Nils and Li, Xiaoye Sherry and Lion, Romain and Mehl, Miriam and Mycek, Paul and Obersteiner, Michael and Quintana-Ort{\'\i}, Enrique S and Rizzi, Francesco and R{\"u}de, Ulrich and Schulz, Martin and Fung, Fred and Speck, Robert and Stals, Linda and Teranishi, Keita and Thibault, Samuel and Th{\"o}nnes, Dominik and Wagner, Andreas and Wohlmuth, Barbara}
}
@article {,
	title = {Accelerating FFT towards Exascale Computing},
	year = {2021},
	publisher = {NVIDIA GPU Technology Conference (GTC2021)},
	author = {Alan Ayala and Stanimire Tomov and Haidar, Azzam and Stoyanov, M. and Cayrols, Sebastien and Li, Jiali and George Bosilca and Jack Dongarra}
}
@conference {,
	title = {Accelerating Multi - Process Communication for Parallel 3-D FFT},
	booktitle = {2021 Workshop on Exascale MPI (ExaMPI)},
	year = {2021},
	month = {2021-12},
	publisher = {IEEE},
	organization = {IEEE},
	address = {St. Louis, MO, USA},
	abstract = {Today largest and most powerful supercomputers in the world are built on heterogeneous platforms; and using the combined power of multi-core CPUs and GPUs, has had a great impact accelerating large-scale applications. However, on these architectures, parallel algorithms, such as the Fast Fourier Transform (FFT), encounter that inter-processor communication become a bottleneck and limits their scalability. In this paper, we present techniques for speeding up multi-process communication cost during the computation of FFTs, considering hybrid network connections as those expected on upcoming exascale machines. Among our techniques, we present algorithmic tuning, making use of phase diagrams; parametric tuning, using different FFT settings; and MPI distribution tuning based on FFT size and computational resources available. We present several experiments obtained on Summit supercomputer at Oak Ridge National Laboratory, using up to 40,960 IBM Power9 cores and 6,144 NVIDIA V-100 GPUs.},
	doi = {10.1109/ExaMPI54564.2021.00011},
	url = {https://ieeexplore.ieee.org/document/9652837/},
	author = {Ayala, Alan and Tomov, Stan and Stoyanov, Miroslav and Haidar, Azzam and Dongarra, Jack}
}
@conference {,
	title = {Distributed-Memory Multi-GPU Block-Sparse Tensor Contraction for Electronic Structure},
	booktitle = {35th IEEE International Parallel \&  Distributed Processing Symposium (IPDPS 2021)},
	year = {2021},
	month = {2021-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Portland, OR},
	abstract = {Many domains of scientific simulation (chemistry, condensed matter physics, data science) increasingly eschew dense tensors for block-sparse tensors, sometimes with additional structure (recursive hierarchy, rank sparsity, etc.). Distributed-memory parallel computation with block-sparse tensorial data is paramount to minimize the time-tosolution (e.g., to study dynamical problems or for real-time analysis) and to accommodate problems of realistic size that are too large to fit into the host/device memory of a single node equipped with accelerators. Unfortunately, computation with such irregular data structures is a poor match to the dominant imperative, bulk-synchronous parallel programming model. In this paper, we focus on the critical element of block-sparse tensor algebra, namely binary tensor contraction, and report on an efficient and scalable implementation using the task-focused PaRSEC runtime. High performance of the block-sparse tensor contraction on the Summit supercomputer is demonstrated for synthetic data as well as for real data involved in electronic structure simulations of unprecedented size.},
	keywords = {block-sparse matrix multiplication, distributed-memory, Electronic structure, multi-GPU node, parsec, tensor contraction},
	url = {https://hal.inria.fr/hal-02970659/document},
	author = {Thomas Herault and Yves Robert and George Bosilca and Robert Harrison and Cannada Lewis and Edward Valeev and Jack Dongarra}
}
@article {,
	title = {DTE: PaRSEC Enabled Libraries and Applications},
	year = {2021},
	month = {2021-04},
	publisher = {2021 Exascale Computing Project Annual Meeting},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@article {,
	title = {An international survey on MPI users},
	journal = {Parallel Computing},
	volume = {108},
	year = {2021},
	month = {2021-12},
	abstract = {The Message Passing Interface (MPI) plays a crucial part in the parallel computing ecosystem, a driving force behind many of thehigh-performance computing (HPC) successes. To maintain its relevance to the user community{\textemdash}and in particular to the growingHPC community at large{\textemdash}the MPI standard needs to identify and understand the MPI users{\textquoteright} concerns and expectations, and adaptaccordingly to continue to efficiently bridge the gap between users and hardware.  This questionnaire survey was conducted usingtwo online questionnaire frameworks and has gathered more than 850 answers from 42 countries since February 2019.  Some ofpreceding surveys of MPI uses are questionnaire surveys like ours, while others are conducted either by analyzing MPI programsto reveal static behavior or by using profiling tools to analyze the dynamic runtime behavior of MPI jobs.  Our survey is differentfrom other questionnaire surveys in terms of its larger number of participants and wide geographic spread. As a result, it is possibleto illustrate the current status of MPI users more accurately and with a wider geographical distribution. In this report, we will showsome interesting findings, compare the results with preceding studies when possible, and provide some recommendations for MPIForum based on the findings.},
	keywords = {message passing interface, MPI, survey},
	doi = {10.1016/j.parco.2021.102853},
	url = {https://www.sciencedirect.com/science/article/abs/pii/S0167819121000983},
	author = {Atsushi Hori and Emmanuel Jeannot and George Bosilca and Takahiro Ogura and Balazs Gerofi and Jie Yin and Yutaka Ishikawa}
}
@inbook {,
	title = {An Introduction to High Performance Computing and Its Intersection with Advances in Modeling Rare Earth Elements and Actinides},
	booktitle = {Rare Earth Elements and Actinides: Progress in Computational Science Applications},
	volume = {1388},
	year = {2021},
	month = {2021-10},
	pages = {3-53},
	publisher = {American Chemical Society},
	organization = {American Chemical Society},
	chapter = {1},
	address = {Washington, DC},
	abstract = {Computationally driven solutions in nuclear and radiochemistry heavily depend on efficient modeling of Rare Earth Elements (REEs) and actinides. Accurate modeling of REEs and actinides faces challenges stemming from limitations from an imbalanced hardware-software ecosystem and its implications on inefficient use of High Performance Computing (HPC). This chapter provides a historical perspective on the evolution of HPC hardware, its intersectionality with domain sciences, the importance of benchmarks for performance, and an overview of challenges and advances in modeling REEs and actinides. This chapter intends to provide an introduction for researchers at the intersection of scientific computing, software development for HPC, and applied computational modeling of REEs and actinides. The chapter is structured in five sections. First, the Introduction includes subsections focusing on the Importance of REEs and Actinides (1.1), Hardware, Software, and the HPC Ecosystem (1.2), and Electronic Structure Modeling of REEs and Actinides (1.3). Second, a section in High Performance Computing focuses on the TOP500 (2.1), HPC Performance (2.2), HPC Benchmarks: Processing, Bandwidth, and Latency (2.3), and HPC Benchmarks and their Relationship to Chemical Modeling (2.4). Third, the Software Challenges and Advances focus on NWChem/NWChemEx (3.1), MADNESS (3.2), and MPQC (3.3). The fourth section provides a short overview of Artificial Intelligence in HPC applications relevant to nuclear and radiochemistry. The fifth section illustrates A Protocol to Evaluate Complexation Preferences in Separations of REEs and Actinides through Computational Modeling.},
	keywords = {actinide, Computational modeling, HPC, REE},
	isbn = {ISBN13: 9780841298255 eISBN: 9780841298248},
	doi = {10.1021/bk-2021-1388.ch001},
	url = {https://pubs.acs.org/doi/10.1021/bk-2021-1388.ch001},
	author = {Deborah A. Penchoff and Edward Valeev and Heike Jagode and Piotr Luszczek and Anthony Danalis and George Bosilca and Robert J. Harrison and Jack Dongarra and Theresa L. Windus}
}
@article {,
	title = {Linear Algebra Prepara.on for Emergent Neural Network Architectures: MAGMA, BLAS, and Batched GPU Computing},
	year = {2021},
	month = {2021-11},
	publisher = {LAPENNA Workshop},
	address = {Virtual},
	author = {Stanimire Tomov and Kwai Wong and Rocco Febbo and Julian Halloy}
}
@techreport {,
	title = {P1673R3: A Free Function Linear algebra Interface Based on the BLAS},
	number = {P1673R3},
	year = {2021},
	month = {2021-04},
	publisher = {ISO},
	type = {standard},
	abstract = {We believe this proposal is complementary to P1385, a proposal for a C++ Standard linear algebra library that introduces matrix and vector classes and overloaded arithmetic operators. In fact, we think that our proposal would make a natural foundation for a library like what P1385 proposes. However, a free function interface -- which clearly separates algorithms from data structures -- more naturally allows for a richer set of operations such as what the BLAS provides. A natural extension of the present proposal would include accepting P1385{\textquoteright}s matrix and vector objects as input for the algorithms proposed here. A straightforward way to do that would be for P1385{\textquoteright}s matrix and vector objects to make views of their data available as basic_mdspan.},
	keywords = {C++, linear algebra},
	url = {http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p1673r3.pdf},
	author = {Mark Hoemmen and Daisy Hollman and Christian Trott and Daniel Sunderland and Nevin Liber and Li-Ta Lo and Damien Lebrun-Grandie and Graham Lopez and Peter Caday and Sarah Knepper and Piotr Luszczek and Timothy Costa}
}
@inproceedings {,
	title = {Revisiting Credit Distribution Algorithms for Distributed Termination Detection},
	year = {2021},
	pages = {611{\textendash}620},
	publisher = {IEEE},
	abstract = {This paper revisits distributed termination detection
algorithms in the context of High-Performance Computing (HPC)
applications. We introduce an efficient variant of the Credit
Distribution Algorithm (CDA) and compare it to the original
algorithm (HCDA) as well as to its two primary competitors: the
Four Counters algorithm (4C) and the Efficient Delay-Optimal
Distributed algorithm (EDOD). We analyze the behavior of each
algorithm for some simplified task-based kernels and show the
superiority of CDA in terms of the number of control messages.},
	keywords = {control messages, credit distribution algorithms, task-based HPC application, Termination detection},
	doi = {10.1109/IPDPSW52791.2021.00095},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Le F{\`e}vre, Valentin and Robert, Yves and Jack Dongarra}
}
@article {,
	title = {A Set of Batched Basic Linear Algebra Subprograms and LAPACK Routines},
	journal = {ACM Transactions on Mathematical Software (TOMS)},
	volume = {47},
	number = {3},
	year = {2021},
	pages = {1{\textendash}23},
	abstract = {This article describes a standard API for a set of Batched Basic Linear Algebra Subprograms (Batched BLAS or BBLAS). The focus is on many independent BLAS operations on small matrices that are grouped together and processed by a single routine, called a Batched BLAS routine. The matrices are grouped together in uniformly sized groups, with just one group if all the matrices are of equal size. The aim is to provide more efficient, but portable, implementations of algorithms on high-performance many-core platforms. These include multicore and many-core CPU processors, GPUs and coprocessors, and other hardware accelerators with floating-point compute facility. As well as the standard types of single and double precision, we also include half and quadruple precision in the standard. In particular, half precision is used in many very large scale applications, such as those associated with machine learning.},
	keywords = {Computations on matrices, Mathematical analysis, Mathematics of computing, Numerical analysis},
	doi = {10.1145/3431921},
	author = {Abdelfattah, Ahmad and Costa, Timothy and Jack Dongarra and Mark Gates and Haidar, Azzam and Hammarling, Sven and Higham, Nicholas J and Kurzak, Jakub and Piotr Luszczek and Stanimire Tomov and others}
}
@article {,
	title = {A survey of numerical linear algebra methods utilizing mixed-precision arithmetic},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {35},
	number = {4},
	year = {2021},
	pages = {344{\textendash}369},
	abstract = {The efficient utilization of mixed-precision numerical linear algebra algorithms can offer attractive acceleration to scientific computing applications. Especially with the hardware integration of low-precision special-function units designed for machine learning applications, the traditional numerical algorithms community urgently needs to reconsider the floating point formats used in the distinct operations to efficiently leverage the available compute power. In this work, we provide a comprehensive survey of mixed-precision numerical linear algebra routines, including the underlying concepts, theoretical background, and experimental results for both dense and sparse linear algebra problems.},
	keywords = {GPUs, High-performance computing, linear algebra, Mixed-precision arithmetic, numerical mathematics},
	doi = {10.1177/10943420211003313},
	author = {Abdelfattah, Ahmad and Anzt, Hartwig and Boman, Erik G and Carson, Erin and Cojean, Terry and Jack Dongarra and Fox, Alyson and Mark Gates and Higham, Nicholas J and Li, Xiaoye S and others}
}
@inproceedings {,
	title = {Task-graph scheduling extensions for efficient synchronization and communication},
	year = {2021},
	pages = {88{\textendash}101},
	abstract = {Task graphs have been studied for decades as a foundation for scheduling irregular parallel applications and incorporated in many programming models including OpenMP. While many high-performance parallel libraries are based on task graphs, they also have additional scheduling requirements, such as synchronization within inner levels of data parallelism and internal blocking communications. In this paper, we extend task-graph scheduling to support efficient synchronization and communication within tasks. Compared to past work, our scheduler avoids deadlock and oversubscription of worker threads, and refines victim selection to increase the overlap of sibling tasks. To the best of our knowledge, our approach is the first to combine gang-scheduling and work-stealing in a single runtime. Our approach has been evaluated on the SLATE high-performance linear algebra library. Relative to the LLVM OMP runtime, our runtime demonstrates performance improvements of up to 13.82\%, 15.2\%, and 36.94\% for LU, QR, and Cholesky, respectively, evaluated across different configurations related to matrix size, number of nodes, and use of CPUs vs GPUs.},
	keywords = {Compilers, Computing methodologies, Parallel computing methodologies, Parallel programming languages, Runtime environments, Software and its engineering, Software notations and tools},
	doi = {10.1145/3447818.3461616},
	author = {Bak, Seonmyeong and Hernandez, Oscar and Mark Gates and Piotr Luszczek and Sarkar, Vivek}
}
@techreport {,
	title = {ASCR@40: Four Decades of Department of Energy Leadership in Advanced Scientific Computing Research},
	year = {2020},
	month = {2020-08},
	publisher = {Advanced Scientific Computing Advisory Committee (ASCAC), US Department of Energy},
	url = {https://computing.llnl.gov/misc/ASCR@40-Highlights.pdf},
	author = {Bruce Hendrickson and Paul Messina and Buddy Bland and Jackie Chen and Phil Colella and Eli Dart and Jack Dongarra and Thom Dunning and Ian Foster and Richard Gerber and Rachel Harken and Wendy Huntoon and Bill Johnston and John Sarrao and Jeff Vetter}
}
@techreport {,
	title = {ASCR@40: Highlights and Impacts of ASCR{\textquoteright}s Programs},
	year = {2020},
	month = {2020-06},
	publisher = {US Department of Energy{\textquoteright}s Office of Advanced Scientific Computing Research},
	abstract = {The Office of Advanced Scientific Computing Research (ASCR) sits within the Office of Science in the Department of Energy (DOE). Per their web pages, {\textquotedblleft}the mission of the ASCR program is to discover, develop, and deploy computational and networking capabilities to analyze, model, simulate, and predict complex phenomena important to the DOE.{\textquotedblright} This succinct statement encompasses a wide range of responsibilities for computing and networking facilities; for procuring, deploying, and operating high performance computing, networking, and storage resources; for basic research in mathematics and computer science; for developing and sustaining a large body of software; and for partnering with organizations across the Office of Science and beyond. While its mission statement may seem very contemporary, the roots of ASCR are quite deep{\textemdash}long predating the creation of DOE. Applied mathematics and advanced computing were both elements of the Theoretical Division of the Manhattan Project. In the early 1950s, the Manhattan Project scientist and mathematician John von Neumann, then a commissioner for the AEC (Atomic Energy Commission), advocated for the creation of a Mathematics program to support the continued development and applications of digital computing. Los Alamos National Laboratory (LANL) scientist John Pasta created such a program to fund researchers at universities and AEC laboratories. Under several organizational name changes, this program has persisted ever since, and would eventually grow to become ASCR.},
	doi = {https://doi.org/10.2172/1631812},
	url = {https://www.osti.gov/servlets/purl/1631812},
	author = {Bruce Hendrickson and Paul Messina and Buddy Bland and Jackie Chen and Phil Colella and Eli Dart and Jack Dongarra and Thom Dunning and Ian Foster and Richard Gerber and Rachel Harken and Wendy Huntoon and Bill Johnston and John Sarrao and Jeff Vetter}
}
@article {,
	title = {DTE: PaRSEC Enabled Libraries and Applications (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@article {,
	title = {DTE: PaRSEC Systems and Interfaces (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@conference {,
	title = {Energy-Aware Strategies for Reliability-Oriented Real-Time Task Allocation on Heterogeneous Platforms},
	booktitle = {49th International Conference on Parallel Processing (ICPP 2020)},
	year = {2020},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Edmonton, AB, Canada},
	author = {Li Han and Yiqin Gao and Jing Liu and Yves Robert and Frederic Vivien}
}
@conference {,
	title = {FFT-Based Gradient Sparsification for the Distributed Training of Deep Neural Networks},
	booktitle = {9th International Symposium on High-Performance Parallel and Distributed Computing (HPDC 20)},
	year = {2020},
	month = {2020-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Stockholm, Sweden},
	abstract = {The performance and efficiency of distributed training of Deep Neural Networks (DNN) highly depend on the performance of gradient averaging among participating processes, a step bound by communication costs. There are two major approaches to reduce communication overhead: overlap communications with computations (lossless), or reduce communications (lossy). The lossless solution works well for linear neural architectures, e.g. VGG, AlexNet, but more recent networks such as ResNet and Inception limit the opportunity for such overlapping. Therefore, approaches that reduce the amount of data (lossy) become more suitable. In this paper, we present a novel, explainable lossy method that sparsifies gradients in the frequency domain, in addition to a new range-based float point representation to quantize and further compress gradients. These dynamic techniques strike a balance between compression ratio, accuracy, and computational overhead, and are optimized to maximize performance in heterogeneous environments.

Unlike existing works that strive for a higher compression ratio, we stress the robustness of our methods, and provide guidance to recover accuracy from failures. To achieve this, we prove how the FFT sparsification affects the convergence and accuracy, and show that our method is guaranteed to converge using a diminishing θ in training. Reducing θ can also be used to recover accuracy from the failure. Compared to STOA lossy methods, e.g., QSGD, TernGrad, and Top-k sparsification, our approach incurs less approximation error, thereby better in both the wall-time and accuracy. On an 8 GPUs, InfiniBand interconnected cluster, our techniques effectively accelerate AlexNet training up to 2.26x to the baseline of no compression, and 1.31x to QSGD, 1.25x to Terngrad and 1.47x to Top-K sparsification.},
	keywords = {FFT, Gradient Compression, Loosy Gradients, Machine Learning, Neural Networks},
	doi = {https://doi.org/10.1145/3369583.3392681},
	author = {Linnan Wang and Wei Wu and Junyu Zhang and Hang Liu and George Bosilca and Maurice Herlihy and Rodrigo Fonseca}
}
@techreport {1461,
	title = {FFT-ECP API and High-Performance Library Prototype for 2-D and 3-D FFTs on Large-Scale Heterogeneous Systems with GPUs},
	number = {FFT-ECP STML13-27},
	year = {2020},
	note = {revision 01-2020},
	month = {2020-01},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	type = {ECP WBS 2.3.3.13 Milestone Report},
	author = {Stanimire Tomov and Alan Ayala and Azzam Haidar and Jack Dongarra}
}
@conference {1481,
	title = {heFFTe: Highly Efficient FFT for Exascale},
	booktitle = {International Conference on Computational Science (ICCS 2020)},
	year = {2020},
	month = {2020-06},
	address = {Amsterdam, Netherlands},
	abstract = {Exascale computing aspires to meet the increasing demands from large scientific applications. Software targeting exascale is typically designed for heterogeneous architectures; henceforth, it is not only important to develop well-designed software, but also make it aware of the hardware architecture and efficiently exploit its power. Currently, several and diverse applications, such as those part of the Exascale Computing Project (ECP) in the United States, rely on efficient computation of the Fast Fourier Transform (FFT). In this context, we present the design and implementation of heFFTe (Highly Efficient FFT for Exascale) library, which targets the upcoming exascale supercomputers. We provide highly (linearly) scalable GPU kernels that achieve more than 40{\texttimes} speedup with respect to local kernels from CPU state-of-the-art libraries, and over 2{\texttimes} speedup for the whole FFT computation. A communication model for parallel FFTs is also provided to analyze the bottleneck for large-scale problems. We show experiments obtained on Summit supercomputer at Oak Ridge National Laboratory, using up to 24,576 IBM Power9 cores and 6,144 NVIDIA V-100 GPUs.},
	keywords = {exascale, FFT, gpu, scalable algorithm},
	doi = {https://doi.org/10.1007/978-3-030-50371-0_19},
	author = {Alan Ayala and Stanimire Tomov and Azzam Haidar and Jack Dongarra}
}
@article {,
	title = {heFFTe: Highly Efficient FFT for Exascale (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {Alan Ayala and Stanimire Tomov and Jack Dongarra and Azzam Haidar}
}
@article {,
	title = {heFFTe: Highly Efficient FFT for Exascale (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {SIAM Conference on Parallel Processing for Scientific Computing (SIAM PP20)},
	address = {Seattle, WA},
	abstract = {Considered one of the top 10 algorithms of the 20th century, the Fast Fourier Transform (FFT) is widely used by applications in science and engineering. Large scale parallel applications targeting exascale, such as those part of the DOE Exascale Computing Project (ECP), are designed for heterogeneous architectures and, currently, more than a dozen ECP applications use FFTs in their codes. To address the applications needs, we developed the highly efficient FFTs for exascale (heFFTe) library. The heFFTe library release features very good weak and strong scalability and performance that is close to 90\% of the roofline peak performance. We present these performance results on the Summit supercomputer. heFFTe is also integrated in a number of applications and we present how the overall performance gets improved by using hFFTe. Performance model, limitations, and challenges are discussed for current and upcoming computer architectures.},
	author = {Alan Ayala and Stanimire Tomov and Azzam Haidar and Jack Dongarra}
}
@article {,
	title = {heFFTe: Highly Efficient FFT for Exascale (Poster)},
	year = {2020},
	month = {2020-10},
	publisher = {NVIDIA GPU Technology Conference (GTC2020)},
	author = {Alan Ayala and Stanimire Tomov and Azzam Haidar and Jack Dongarra}
}
@article {,
	title = {How to Build Your Own Deep Neural Network},
	year = {2020},
	month = {2020-07},
	publisher = {PEARC20},
	keywords = {AI, Deep Neural Networks, dense linear algebra, HPC, ML},
	author = {Kwai Wong and Stanimire Tomov and Daniel Nichols and Rocco Febbo and Florent Lopez and Julian Halloy and Xianfeng Ma}
}
@conference {1372,
	title = {Improved Energy-Aware Strategies for Periodic Real-Time Tasks under Reliability Constraints},
	booktitle = {40th IEEE Real-Time Systems Symposium (RTSS 2019)},
	year = {2020},
	month = {2020-02},
	publisher = {IEEE Press},
	organization = {IEEE Press},
	address = {York, UK},
	author = {Li Han and Louis-Claude Canon and Jing Liu and Yves Robert and Frederic Vivien}
}
@article {,
	title = {MAGMA Templates for Scalable Linear Algebra on Emerging Architectures},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {34},
	year = {2020},
	month = {2020-11},
	pages = {645-658},
	abstract = {With the acquisition and widespread use of more resources that rely on accelerator/wide vector{\textendash}based computing, there has been a strong demand for science and engineering applications to take advantage of these latest assets. This, however, has been extremely challenging due to the diversity of systems to support their extreme concurrency, complex memory hierarchies, costly data movement, and heterogeneous node architectures. To address these challenges, we design a programming model and describe its ease of use in the development of a new MAGMA Templates library that delivers high-performance scalable linear algebra portable on current and emerging architectures. MAGMA Templates derives its performance and portability by (1) building on existing state-of-the-art linear algebra libraries, like MAGMA, SLATE, Trilinos, and vendor-optimized math libraries, and (2) providing access (seamlessly to the users) to the latest algorithms and architecture-specific optimizations through a single, easy-to-use C++-based API.},
	issn = {1094-3420},
	doi = {https://doi.org/10.1177/1094342020938421},
	author = {Mohammed Al Farhan and Ahmad Abdelfattah and Stanimire Tomov and Mark Gates and Dalal Sukkari and Azzam Haidar and Robert Rosenberg and Jack Dongarra}
}
@article {,
	title = {Mixed-Precision Iterative Refinement using Tensor Cores on GPUs to Accelerate Solution of Linear Systems},
	journal = {Proceedings of the Royal Society A},
	volume = {476},
	year = {2020},
	month = {2020-11},
	abstract = {Double-precision floating-point arithmetic (FP64) has been the de facto standard for engineering and scientific simulations for several decades. Problem complexity and the sheer volume of data coming from various instruments and sensors motivate researchers to mix and match various approaches to optimize compute resources, including different levels of floating-point precision. In recent years, machine learning has motivated hardware support for half-precision floating-point arithmetic. A primary challenge in high-performance computing is to leverage reduced-precision and mixed-precision hardware. We show how the FP16/FP32 Tensor Cores on NVIDIA GPUs can be exploited to accelerate the solution of linear systems of equations Ax = b without sacrificing numerical stability. The techniques we employ include multiprecision LU factorization, the preconditioned generalized minimal residual algorithm (GMRES), and scaling and auto-adaptive rounding to avoid overflow. We also show how to efficiently handle systems with multiple right-hand sides. On the NVIDIA Quadro GV100 (Volta) GPU, we achieve a 4{\texttimes}-5{\texttimes} performance increase and 5{\texttimes} better energy efficiency versus the standard FP64 implementation while maintaining an FP64 level of numerical stability.},
	keywords = {GMRESLU factorization, GPU computing, half precision arithmetic, iterative refinement, mixed precision solvers},
	issn = {1471-2946},
	doi = {https://doi.org/10.1098/rspa.2020.0110},
	author = {Azzam Haidar and Harun Bayraktar and Stanimire Tomov and Jack Dongarra and Nicholas J. Higham}
}
@techreport {1471,
	title = {Mixed-Precision Solution of Linear Systems Using Accelerator-Based Computing},
	number = {ICL-UT-20-05},
	year = {2020},
	month = {2020-05},
	publisher = {University of Tennessee},
	abstract = {Double-precision floating-point arithmetic (FP64) has been the de facto standard for engineering and scientific simulations for several decades. Problem complexity and the sheer volume of data coming from various instruments and sensors motivate researchers to mix and match various approaches to optimize compute resources, including different levels of floating-point precision. In recent years, machine learning has motivated hardware support for half-precision floating-point arithmetic. A primary challenge in high-performance computing is to leverage reduced- and mixed-precision hardware. We show how the FP16/FP32 Tensor Cores on NVIDIA GPUs can be exploited to accelerate the solution of linear systems of equations Ax = b without sacrificing numerical stability. We achieve a 4{\texttimes}{\textendash}5{\texttimes} performance increase and 5{\texttimes} better energy efficiency versus the standard FP64 implementation while maintaining an FP64 level of numerical stability.},
	author = {Azzam Haidar and Harun Bayraktar and Stanimire Tomov and Jack Dongarra and Nicholas J. Higham}
}
@article {1476,
	title = {Numerical Algorithms for High-Performance Computational Science},
	journal = {Philosophical Transactions of the Royal Society A},
	volume = {378},
	year = {2020},
	abstract = {A number of features of today{\textquoteright}s high-performance computers make it challenging to exploit these machines fully for computational science. These include increasing core counts but stagnant clock frequencies; the high cost of data movement; use of accelerators (GPUs, FPGAs, coprocessors), making architectures increasingly heterogeneous; and multi- ple precisions of floating-point arithmetic, including half-precision. Moreover, as well as maximizing speed and accuracy, minimizing energy consumption is an important criterion. New generations of algorithms are needed to tackle these challenges. We discuss some approaches that we can take to develop numerical algorithms for high-performance computational science, with a view to exploiting the next generation of supercomputers.},
	issn = {1471-2962},
	doi = {https://doi.org/10.1098/rsta.2019.0066},
	author = {Jack Dongarra and Laura Grigori and Nicholas J. Higham}
}
@article {1460,
	title = {Overhead of Using Spare Nodes},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2020},
	month = {2020-02},
	abstract = {With the increasing fault rate on high-end supercomputers, the topic of fault tolerance has been gathering attention. To cope with this situation, various fault-tolerance techniques are under investigation; these include user-level, algorithm-based fault-tolerance techniques and parallel execution environments that enable jobs to continue following node failure. Even with these techniques, some programs with static load balancing, such as stencil computation, may underperform after a failure recovery. Even when spare nodes are present, they are not always substituted for failed nodes in an effective way. This article considers the questions of how spare nodes should be allocated, how to substitute them for faulty nodes, and how much the communication performance is affected by such a substitution. The third question stems from the modification of the rank mapping by node substitutions, which can incur additional message collisions. In a stencil computation, rank mapping is done in a straightforward way on a Cartesian network without incurring any message collisions. However, once a substitution has occurred, the optimal node-rank mapping may be destroyed. Therefore, these questions must be answered in a way that minimizes the degradation of communication performance. In this article, several spare node allocation and failed node substitution methods will be proposed, analyzed, and compared in terms of communication performance following the substitution. The proposed substitution methods are named sliding methods. The sliding methods are analyzed by using our developed simulation program and evaluated by using the K computer, Blue Gene/Q (BG/Q), and TSUBAME 2.5. It will be shown that when failures occur, the stencil communication performance on the K and BG/Q can be slowed around 10 times depending on the number of node failures. The barrier performance on the K can be cut in half. On BG/Q, barrier performance can be slowed by a factor of 10. Further, it will also be shown that almost no such communication performance degradation can be seen on TSUBAME 2.5. This is because TSUBAME 2.5 has an Infiniband network connected with a FatTree topology, while the K computer and BG/Q have dedicated Cartesian networks. Thus, the communication performance degradation depends on network characteristics.},
	keywords = {communication performance, fault mitigation, Fault tolerance, sliding method, spare node},
	issn = {1094-3420},
	doi = {https://doi.org/10.1177\%2F1094342020901885},
	url = {https://journals.sagepub.com/doi/10.1177/1094342020901885},
	author = {Atsushi Hori and Kazumi Yoshinaga and Thomas Herault and Aurelien Bouteiller and George Bosilca and Yutaka Ishikawa}
}
@conference {,
	title = {Predicting MPI Collective Communication Performance Using Machine Learning},
	booktitle = {2020 IEEE International Conference on Cluster Computing (CLUSTER)},
	year = {2020},
	month = {2020-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Kobe, Japan},
	abstract = {The Message Passing Interface (MPI) defines the semantics of data communication operations, while the implementing libraries provide several parameterized algorithms for each operation. Each algorithm of an MPI collective operation may work best on a particular system and may be dependent on the specific communication problem. Internally, MPI libraries employ heuristics to select the best algorithm for a given communication problem when being called by an MPI application. The majority of MPI libraries allow users to override the default algorithm selection, enabling the tuning of this selection process. The problem then becomes how to select the best possible algorithm for a specific case automatically. In this paper, we address the algorithm selection problem for MPI collective communication operations. To solve this problem, we propose an auto-tuning framework for collective MPI operations based on machine-learning techniques. First, we execute a set of benchmarks of an MPI library and its entire set of collective algorithms. Second, for each algorithm, we fit a performance model by applying regression learners. Last, we use the regression models to predict the best possible (fastest) algorithm for an unseen communication problem. We evaluate our approach for different MPI libraries and several parallel machines. The experimental results show that our approach outperforms the standard algorithm selection heuristics, which are hard-coded into the MPI libraries, by a significant margin.},
	keywords = {Auto-tuning, GAM, KNN, Machine Learning, message passing interface, Performance Prediction, XGBoost},
	doi = {https://doi.org/10.1109/CLUSTER49012.2020.00036},
	author = {Sascha Hunold and Abhinav Bhatele and George Bosilca and Peter Knees}
}
@article {,
	title = {A Report of the MPI International Survey (Poster)},
	year = {2020},
	month = {2020-09},
	publisher = {EuroMPI/USA {\textquoteright}20: 27th European MPI Users{\textquoteright} Group Meeting},
	address = {Austin, TX},
	author = {Atsushi Hori and Takahiro Ogura and Balazs Gerofi and Jie Yin and Yutaka Ishikawa and Emmanuel Jeannot and George Bosilca}
}
@conference {,
	title = {Reservation and Checkpointing Strategies for Stochastic Jobs},
	booktitle = {34th IEEE International Parallel and Distributed Processing Symposium (IPDPS 2020)},
	year = {2020},
	month = {2020-05},
	publisher = {IEEE Computer Society Press},
	organization = {IEEE Computer Society Press},
	address = {New Orleans, LA},
	author = {Ana Gainaru and Brice Goglin and Valentin Honor{\'e} and Padma Raghavan and Guillaume Pallez and Padma Raghavan and Yves Robert and Hongyang Sun}
}
@article {,
	title = {A Set of Batched Basic Linear Algebra Subprograms},
	journal = {ACM Transactions on Mathematical Software},
	year = {2020},
	month = {2020-10},
	abstract = {This paper describes a standard API for a set of Batched Basic Linear Algebra Subprograms (Batched BLAS or BBLAS). The focus is on many independent BLAS operations on small matrices that are grouped together and processed by a single routine, called a Batched BLAS routine. The matrices are grouped together in uniformly sized groups, with just one group if all the matrices are of equal size. The aim is to provide more efficient, but portable, implementations of algorithms on high-performance many-core platforms. These include multicore and many-core CPU processors, GPUs and coprocessors, and other hardware accelerators with floating-point compute facility. As well as the standard types of single and double precision, we also include half and quadruple precision in the standard. In particular half precision is used in many very large scale applications, such as those associated with machine learning.},
	author = {Ahmad Abdelfattah and Timothy Costa and Jack Dongarra and Mark Gates and Azzam Haidar and Sven Hammarling and Nicholas J. Higham and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Mawussi Zounon}
}
@techreport {,
	title = {A Survey of Numerical Methods Utilizing Mixed Precision Arithmetic},
	number = {15, ICL-UT-20-08},
	year = {2020},
	month = {2020-07},
	publisher = {University of Tennessee},
	type = {SLATE Working Notes},
	author = {Ahmad Abdelfattah and Hartwig Anzt and Erik Boman and Erin Carson and Terry Cojean and Jack Dongarra and Mark Gates and Thomas Gruetzmacher and Nicholas J. Higham and Sherry Li and Neil Lindquist and Yang Liu and Jennifer Loe and Piotr Luszczek and Pratik Nayak and Sri Pranesh and Siva Rajamanickam and Tobias Ribizel and Barry Smith and Kasia Swirydowicz and Stephen Thomas and Stanimire Tomov and Yaohung Tsai and Ichitaro Yamazaki and Urike Meier Yang}
}
@conference {,
	title = {The Template Task Graph (TTG) - An Emerging Practical Dataflow Programming Paradigm for Scientific Simulation at Extreme Scale},
	booktitle = { 2020 IEEE/ACM 5th International Workshop on Extreme Scale Programming Models and Middleware (ESPM2)},
	year = {2020},
	month = {2020-11},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We describe TESSE, an emerging general-purpose, open-source software ecosystem that attacks the twin challenges of programmer productivity and portable performance for advanced scientific applications on modern high-performance computers. TESSE builds upon and extends the ParsecDAG/-dataflow runtime with a new Domain Specific Languages (DSL) and new integration capabilities. Motivating this work is our belief that such a dataflow model, perhaps with applications composed in domain specific languages, can overcome many of the challenges faced by a wide variety of irregular applications that are poorly served by current programming and execution models. Two such applications from many-body physics and applied mathematics are briefly explored. This paper focuses upon the Template Task Graph (TTG), which is TESSE{\textquoteright}s main C++ Api that provides a powerful work/data-flow programming model. Algorithms on spatial trees, block-sparse tensors, and wave fronts are used to illustrate the API and associated concepts, as well as to compare with related approaches.},
	keywords = {dag, dataflow, exascale, graph, High-performance computing, workflow},
	doi = {https://doi.org/10.1109/ESPM251964.2020.00011},
	author = {George Bosilca and Robert Harrison and Thomas Herault and Mohammad Mahdi Javanmard and Poornima Nookala and Edward Valeev}
}
@article {1319,
	title = {Adaptive Precision in Block-Jacobi Preconditioning for Iterative Sparse Linear System Solvers},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {31},
	number = {6},
	year = {2019},
	month = {2019-03},
	pages = {e4460},
	abstract = {Summary We propose an adaptive scheme to reduce communication overhead caused by data movement by selectively storing the diagonal blocks of a block-Jacobi preconditioner in different precision formats (half, single, or double). This specialized preconditioner can then be combined with any Krylov subspace method for the solution of sparse linear systems to perform all arithmetic in double precision. We assess the effects of the adaptive precision preconditioner on the iteration count and data transfer cost of a preconditioned conjugate gradient solver. A preconditioned conjugate gradient method is, in general, a memory bandwidth-bound algorithm, and therefore its execution time and energy consumption are largely dominated by the costs of accessing the problem{\textquoteright}s data in memory. Given this observation, we propose a model that quantifies the time and energy savings of our approach based on the assumption that these two costs depend linearly on the bit length of a floating point number. Furthermore, we use a number of test problems from the SuiteSparse matrix collection to estimate the potential benefits of the adaptive block-Jacobi preconditioning scheme.},
	keywords = {adaptive precision, block-Jacobi preconditioning, communication reduction, energy efficiency, Krylov subspace methods, sparse linear systems},
	doi = {https://doi.org/10.1002/cpe.4460},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/cpe.4460},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Nicholas J. Higham and Enrique S. Quintana-Orti}
}
@article {1262,
	title = {Algorithms and Optimization Techniques for High-Performance Matrix-Matrix Multiplications of Very Small Matrices},
	journal = {Parallel Computing},
	volume = {81},
	year = {2019},
	month = {2019-01},
	pages = {1{\textendash}21},
	abstract = {Expressing scientific computations in terms of BLAS, and in particular the general dense matrix-matrix multiplication (GEMM), is of fundamental importance for obtaining high performance portability across architectures. However, GEMMs for small matrices of sizes smaller than 32 are not sufficiently optimized in existing libraries. We consider the computation of many small GEMMs and its performance portability for a wide range of computer architectures, including Intel CPUs, ARM, IBM, Intel Xeon Phi, and GPUs. These computations often occur in applications like big data analytics, machine learning, high-order finite element methods (FEM), and others. The GEMMs are grouped together in a single batched routine. For these cases, we present algorithms and their optimization techniques that are specialized for the matrix sizes and architectures of interest. We derive a performance model and show that the new developments can be tuned to obtain performance that is within 90\% of the optimal for any of the architectures of interest. For example, on a V100 GPU for square matrices of size 32, we achieve an execution rate of about 1600 gigaFLOP/s in double-precision arithmetic, which is 95\% of the theoretically derived peak for this computation on a V100 GPU. We also show that these results outperform currently available state-of-the-art implementations such as vendor-tuned math libraries, including Intel MKL and NVIDIA CUBLAS, as well as open-source libraries like OpenBLAS and Eigen.},
	keywords = {Autotuning, Batched GEMM, HPC, Matrix-matrix product, optimization, Small matrices},
	doi = {https://doi.org/10.1016/j.parco.2018.10.003},
	author = {Ian Masliah and Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Marc Baboulin and Jo{\"e}l Falcou and Jack Dongarra}
}
@article {1305,
	title = {Checkpointing Strategies for Shared High-Performance Computing Platforms},
	journal = {International Journal of Networking and Computing},
	volume = {9},
	number = {1},
	year = {2019},
	pages = {28{\textendash}52},
	abstract = {Input/output (I/O) from various sources often contend for scarcely available bandwidth. For example, checkpoint/restart (CR) protocols can help to ensure application progress in failure-prone environments. However, CR I/O alongside an application{\textquoteright}s normal, requisite I/O can increase I/O contention and might negatively impact performance. In this work, we consider different aspects (system-level scheduling policies and hardware) that optimize the overall performance of concurrently executing CR-based applications that share I/O resources. We provide a theoretical model and derive a set of necessary constraints to minimize the global waste on a given platform. Our results demonstrate that Young/Daly{\textquoteright}s optimal checkpoint interval, despite providing a sensible metric for a single, undisturbed application, is not sufficient to optimally address resource contention at scale. We show that by combining optimal checkpointing periods with contention-aware system-level I/O scheduling strategies, we can significantly improve overall application performance and maximize the platform throughput. Finally, we evaluate how specialized hardware, namely burst buffers, may help to mitigate the I/O contention problem. Overall, these results provide critical analysis and direct guidance on how to design efficient, CR ready, large -scale platforms without a large investment in the I/O subsystem.},
	issn = {2185-2847},
	url = {http://www.ijnc.org/index.php/ijnc/article/view/195},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@techreport {1399,
	title = {A Collection of White Papers from the BDEC2 Workshop in Poznan, Poland},
	number = {ICL-UT-19-10},
	year = {2019},
	month = {2019-05},
	publisher = {University of Tennessee, Knoxville},
	author = {Gabriel Antoniu and Alexandru Costan and Ovidiu Marcu and Maria S. P{\'e}rez and Nenad Stojanovic and Rosa M. Badia and Miguel V{\'a}zquez and Sergi Girona and Micah Beck and Terry Moore and Piotr Luszczek and Ezra Kissel and Martin Swany and Geoffrey Fox and Vibhatha Abeykoon and Selahattin Akkas and Kannan Govindarajan and Gurhan Gunduz and Supun Kamburugamuve and Niranda Perera and Ahmet Uyar and Pulasthi Wickramasinghe and Chathura Widanage and Maria Girone and Toshihiro Hanawa and Richard Moreno and Ariel Oleksiak and Martin Swany and Ryousei Takano and M.P. van Haarlem and J. van Leeuwen and J.B.R. Oonk and T. Shimwell and L.V.E. Koopmans}
}
@techreport {1408,
	title = {A Collection of White Papers from the BDEC2 Workshop in San Diego, CA},
	number = {ICL-UT-19-13},
	year = {2019},
	month = {2019-10},
	publisher = {University of Tennessee},
	author = {Ilkay Altintas and Kyle Marcus and Volkan Vural and Shweta Purawat and Daniel Crawl and Gabriel Antoniu and Alexandru Costan and Ovidiu Marcu and Prasanna Balaprakash and Rongqiang Cao and Yangang Wang and Franck Cappello and Robert Underwood and Sheng Di and Justin M. Wozniak and Jon C. Calhoun and Cong Xu and Antonio Lain and Paolo Faraboschi and Nic Dube and Dejan Milojicic and Balazs Gerofi and Maria Girone and Viktor Khristenko and Tony Hey and Erza Kissel and Yu Liu and Richard Loft and Pekka Manninen and Sebastian von Alfthan and Takemasa Miyoshi and Bruno Raffin and Olivier Richard and Denis Trystram and Maryam Rahnemoonfar and Robin Murphy and Joel Saltz and Kentaro Sano and Rupak Roy and Kento Sato and Jian Guo and Jen s Domke and Weikuan Yu and Takaki Hatsui and Yasumasa Joti and Alex Szalay and William M. Tang and Michael R. Wyatt II and Michela Taufer and Todd Gamblin and Stephen Herbein and Adam Moody and Dong H. Ahn and Rich Wolski and Chandra Krintz and Fatih Bakir and Wei-tsung Lin and Gareth George}
}
@article {1301,
	title = {Comparing the Performance of Rigid, Moldable, and Grid-Shaped Applications on Failure-Prone HPC Platforms},
	journal = {Parallel Computing},
	volume = {85},
	year = {2019},
	month = {2019-07},
	pages = {1{\textendash}12},
	doi = {https://doi.org/10.1016/j.parco.2019.02.002},
	author = {Valentin Le F{\`e}vre and Thomas Herault and Yves Robert and Aurelien Bouteiller and Atsushi Hori and George Bosilca and Jack Dongarra}
}
@conference {1379,
	title = {Counter Inspection Toolkit: Making Sense out of Hardware Performance Events},
	booktitle = {11th International Workshop on Parallel Tools for High Performance Computing},
	year = {2019},
	month = {2019-02},
	publisher = {Cham, Switzerland: Springer},
	organization = {Cham, Switzerland: Springer},
	address = {Dresden, Germany},
	abstract = {Hardware counters play an essential role in understanding the behavior of performance-critical applications, and inform any effort to identify opportunities for performance optimization. However, because modern hardware is becoming increasingly complex, the number of counters that are offered by the vendors increases and, in some cases, so does their complexity. In this paper we present a toolkit that aims to assist application developers invested in performance analysis by automatically categorizing and disambiguating performance counters. We present and discuss the set of microbenchmarks and analyses that we developed as part of our toolkit. We explain why they work and discuss the non-obvious reasons why some of our early benchmarks and analyses did not work in an effort to share with the rest of the community the wisdom we acquired from negative results.},
	doi = {https://doi.org/10.1007/978-3-030-11987-4_2},
	author = {Anthony Danalis and Heike Jagode and H Hanumantharayappa and Sangamesh Ragate and Jack Dongarra}
}
@techreport {1322,
	title = {Design and Implementation for FFT-ECP on Distributed Accelerated Systems},
	number = {ICL-UT-19-05},
	year = {2019},
	month = {2019-04},
	publisher = {University of Tennessee},
	type = {ECP WBS 2.3.3.09 Milestone Report},
	author = {Stanimire Tomov and Azzam Haidar and Alan Ayala and Daniel Schultz and Jack Dongarra}
}
@article {1269,
	title = {Evaluation of Directive-Based Performance Portable Programming Models},
	journal = {International Journal of High Performance Computing and Networking},
	volume = {14},
	year = {2019},
	month = {2019{\textendash}07},
	pages = {165-182},
	abstract = {We present an extended exploration of the performance portability of directives provided by OpenMP 4 and OpenACC to program various types of node architecture with attached accelerators, both self-hosted multicore and offload multicore/GPU. Our goal is to examine how successful OpenACC and the newer offload features of OpenMP 4.5 are for moving codes between architectures, and we document how much tuning might be required and what lessons we can learn from these experiences. To do this, we use examples of algorithms with varying computational intensities for our evaluation, as both compute and data access efficiency are important considerations for overall application performance. To better understand fundamental compute vs. bandwidth bound characteristics, we add the compute-bound Level 3 BLAS GEMM kernel to our linear algebra evaluation. We implement the kernels of interest using various methods provided by newer OpenACC and OpenMP implementations, and we evaluate their performance on various platforms including both x86_64 and Power8 with attached NVIDIA GPUs, x86_64 multicores, self-hosted Intel Xeon Phi KNL, as well as an x86_64 host system with Intel Xeon Phi coprocessors. We update these evaluations with the newest version of the NVIDIA Pascal architecture (P100), Intel KNL 7230, Power8+, and the newest supporting compiler implementations. Furthermore, we present in detail what factors affected the performance portability, including how to pick the right programming model, its programming style, its availability on different platforms, and how well compilers can optimise and target multiple platforms.},
	keywords = {OpenACC, OpenMP 4, performance portability, Programming models},
	doi = {http://dx.doi.org/10.1504/IJHPCN.2017.10009064 },
	author = {M. Graham Lopez and Wayne Joubert and Ver{\'o}nica Larrea and Oscar Hernandez and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1329,
	title = {FFT-ECP Fast Fourier Transform},
	year = {2019},
	month = {2019-01},
	publisher = {2019 ECP Annual Meeting (Research Poster)},
	address = {Houston, TX},
	author = {Stanimire Tomov and Azzam Haidar and Alan Ayala and Daniel Schultz and Jack Dongarra}
}
@techreport {1401,
	title = {FFT-ECP Implementation Optimizations and Features Phase},
	number = {ICL-UT-19-12},
	year = {2019},
	month = {2019-10},
	publisher = {University of Tennessee},
	author = {Stanimire Tomov and Azzam Haidar and Alan Ayala and Hejer Shaiek and Jack Dongarra}
}
@article {1314,
	title = {A Generic Approach to Scheduling and Checkpointing Workflows},
	journal = {International Journal of High Performance Computing Applications},
	volume = {33},
	year = {2019},
	month = {2019-11},
	pages = {1255-1274},
	keywords = {checkpoint, fail-stop error, resilience, workflow},
	doi = {https://doi.org/10.1177/1094342019866891},
	author = {Li Han and Valentin Le F{\`e}vre and Louis-Claude Canon and Yves Robert and Frederic Vivien}
}
@article {,
	title = {{A Generic Approach to Scheduling and Checkpointing Workflows}},
	journal = {Int. Journal of High Performance Computing Applications},
	volume = {33},
	number = {6},
	year = {2019},
	pages = {1255-1274},
	author = {Han, Li and Le F{\`e}vre, Valentin and Canon, Louis-Claude and Robert, Yves and Vivien, Fr{\'e}d{\'e}ric}
}
@conference {1410,
	title = {Generic Matrix Multiplication for Multi-GPU Accelerated Distributed-Memory Platforms over PaRSEC},
	booktitle = {ScalA{\textquoteright}19: 10th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2019},
	month = {2019-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Denver, CO},
	author = {Thomas Herault and Yves Robert and George Bosilca and Jack Dongarra}
}
@conference {1395,
	title = {Give MPI Threading a Fair Chance: A Study of Multithreaded MPI Designs},
	booktitle = {IEEE Cluster},
	year = {2019},
	month = {2019-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Albuquerque, NM},
	abstract = {The Message Passing Interface (MPI) has been one of the most prominent programming paradigms in highperformance computing (HPC) for the past decade. Lately, with changes in modern hardware leading to a drastic increase in the number of processor cores, developers of parallel applications are moving toward more integrated parallel programming paradigms, where MPI is used along with other, possibly node-level, programming paradigms, or MPI+X. MPI+threads emerged as one of the favorite choices in HPC community, according to a survey of the HPC community. However, threading support in MPI comes with many compromises to the overall performance delivered, and, therefore, its adoption is compromised. This paper studies in depth the MPI multi-threaded implementation design in one of the leading MPI implementations, Open MPI, and expose some of the shortcomings of the current design. We propose, implement, and evaluate a new design of the internal handling of communication progress which allows for a significant boost in multi-threading performance, increasing the viability of MPI in the MPI+X programming paradigm.},
	keywords = {communication contention, MPI, thread},
	author = {Thananon Patinyasakdikul and David Eberius and George Bosilca and Nathan Hjelm}
}
@article {1385,
	title = {GPUDirect MPI Communications and Optimizations to Accelerate FFTs on Exascale Systems},
	number = {icl-ut-19-06},
	year = {2019},
	month = {2019-09},
	publisher = {ICL},
	type = {Extended Abstract},
	abstract = {Fast Fourier transforms (FFTs) are used in applications ranging from molecular dynamics and spectrum estimation to machine learn- ing, fast convolution and correlation, signal modulation, wireless multimedia applications, and others. However, FFTs are memory bound, and therefore, to accelerate them, it is crucial to avoid and optimize the FFTs{\textquoteright} communications. To this end, we present a 3-D FFT design for distributed graphics processing unit (GPU) systems that: (1) efficiently uses GPUs{\textquoteright} high bandwidth, (2) reduces global communications algorithmically, when possible, and (3) employs GPUDirect technologies as well as MPI optimizations in the development of high-performance FFTs for large-scale GPU-accelerated systems. We show that these developments and optimizations lead to very good strong scalability and a performance that is close to 90\% of the theoretical peak.},
	keywords = {CUDA-Aware MPI, ECP, FFT, FFT-ECP, gpu, GPUDirect},
	author = {Hejer Shaiek and Stanimire Tomov and Alan Ayala and Azzam Haidar and Jack Dongarra}
}
@conference {1403,
	title = {Impacts of Multi-GPU MPI Collective Communications on Large FFT Computation},
	booktitle = {Workshop on Exascale MPI (ExaMPI) at SC19},
	year = {2019},
	month = {2019-11},
	address = {Denver, CO},
	keywords = {Collective MPI, Exascale applications, FFT, Heterogeneous systems, scalable},
	author = {Alan Ayala and Stanimire Tomov and Xi Luo and Hejer Shaiek and Azzam Haidar and George Bosilca and Jack Dongarra}
}
@article {1366,
	title = {MagmaDNN 0.2 High-Performance Data Analytics for Manycore GPUs and CPUs},
	year = {2019},
	month = {2019-01},
	publisher = {University of Tennessee},
	doi = {10.13140/RG.2.2.14906.64961},
	author = {Lucien Ng and Sihan Chen and Alex Gessinger and Daniel Nichols and Sophia Cheng and Anu Meenasorna and Kwai Wong and Stanimire Tomov and Azzam Haidar and Eduardo D{\textquoteright}Azevedo and Jack Dongarra}
}
@conference {1452,
	title = {Performance Analysis of Tile Low-Rank Cholesky Factorization Using PaRSEC Instrumentation Tools},
	booktitle = {Workshop on Programming and Performance Visualization Tools (ProTools 19) at SC19},
	year = {2019},
	month = {2019-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	author = {Qinglei Cao and Yu Pei and Thomas Herault and Kadir Akbudak and Aleksandr Mikhalev and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@article {1270,
	title = {PLASMA: Parallel Linear Algebra Software for Multicore Using OpenMP},
	journal = {ACM Transactions on Mathematical Software},
	volume = {45},
	year = {2019},
	month = {2019-06},
	doi = {https://doi.org/10.1145/3264491},
	author = {Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Maksims Abalenkovs and Negin Bagherpour and Sven Hammarling and Jakub Sistek}
}
@conference {1371,
	title = {Replication is More Efficient Than You Think},
	booktitle = {The IEEE/ACM Conference on High Performance Computing Networking, Storage and Analysis (SC19)},
	year = {2019},
	month = {2019-11},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Denver, CO},
	author = {Anne Benoit and Thomas Herault and Valentin Le F{\`e}vre and Yves Robert}
}
@conference {1316,
	title = {Reservation Strategies for Stochastic Jobs},
	booktitle = {33rd IEEE International Parallel and Distributed Processing Symposium (IPDPS 2019)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE Computer Society Press},
	organization = {IEEE Computer Society Press},
	address = {Rio de Janeiro, Brazil},
	author = {Guillaume Aupy and Ana Gainaru and Valentin Honor{\'e} and Padma Raghavan and Yves Robert and Hongyang Sun}
}
@conference {1378,
	title = {Software-Defined Events through PAPI},
	booktitle = {2019 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	abstract = {PAPI has been used for almost two decades as an abstraction and standardization layer for profiling hardware-specific performance metrics. However, application developers-and profiling software packages-are quite often interested in information beyond hardware counters, such as the behavior of libraries used by the software that is being profiled. So far, accessing this information has required interfacing directly with the libraries on a case-by-case basis, or low-level binary instrumentation. In this paper, we introduce the new Software-Defined Event (SDE) component of PAPI which aims to enable PAPI to serve as an abstraction and standardization layer for events that originate in software layers as well. Extending PAPI to include SDEs enables monitoring of both types of performance events-hardware-and software-related events-in a uniform way, through the same consistent PAPI interface. Furthermore, implementing SDE as a PAPI component means that the new API is aimed only at the library developers who wish to export events from within their libraries. The API for reading PAPI events-both hardware and software-remains the same, so all legacy codes and tools that use PAPI will not only continue to work, but they will automatically be able to read SDEs wherever those are available. The goal of this paper is threefold. First, we outline our design decisions regarding the functionality we offer through the new SDE interface, and offer simple examples of usage. Second, we illustrate how those events can be utilized by different software packages, specifically, by showcasing their use in the task-based runtime PaRSEC, and the HPCG supercomputing benchmark. Third, we provide a thorough performance analysis of the overhead that results from monitoring different types of SDEs, and showcase the negligible overhead of using PAPI SDE even in cases of extremely heavy use.},
	doi = {https://doi.org/10.1109/IPDPSW.2019.00069},
	author = {Anthony Danalis and Heike Jagode and Thomas Herault and Piotr Luszczek and Jack Dongarra}
}
@inbook {1302,
	title = {System Software for Many-Core and Multi-Core Architectures},
	booktitle = {Advanced Software Technologies for Post-Peta Scale Computing: The Japanese Post-Peta CREST Research Project},
	year = {2019},
	pages = {59{\textendash}75},
	publisher = {Springer Singapore},
	organization = {Springer Singapore},
	address = {Singapore},
	abstract = {In this project, the software technologies for the post-peta scale computing were explored. More specifically, OS technologies for heterogeneous architectures, lightweight thread, scalable I/O, and fault mitigation were investigated. As for the OS technologies, a new parallel execution model, Partitioned Virtual Address Space (PVAS), for the many-core CPU was proposed. For the heterogeneous architectures, where multi-core CPU and many-core CPU are connected with an I/O bus, an extension of PVAS, Multiple-PVAS, to have a unified virtual address space of multi-core and many-core CPUs was proposed. The proposed PVAS was also enhanced to have multiple processes where process context switch can take place at the user level (named User-Level Process: ULP). As for the scalable I/O, EARTH, optimization techniques for MPI collective I/O, was proposed. Lastly, for the fault mitigation, User Level Fault Mitigation, ULFM was improved to have faster agreement process, and sliding methods to substitute failed nodes with spare nodes was proposed. The funding of this project was ended in 2016; however, many proposed technologies are still being propelled.},
	isbn = {978-981-13-1924-2},
	doi = {10.1007/978-981-13-1924-2_4},
	url = {https://doi.org/10.1007/978-981-13-1924-2_4},
	author = {Atsushi Hori and Tsujita, Yuichi and Shimada, Akio and Yoshinaga, Kazumi and Mitaro, Namiki and Fukazawa, Go and Sato, Mikiko and George Bosilca and Aurelien Bouteiller and Thomas Herault},
	editor = {Sato, Mitsuhisa}
}
@article {1336,
	title = {Accelerating Linear Algebra with MAGMA},
	year = {2018},
	month = {2018-02},
	publisher = {ECP Annual Meeting 2018, Tutorial},
	address = {Knoxville, TN},
	author = {Stanimire Tomov and Mark Gates and Azzam Haidar}
}
@article {1266,
	title = {Accelerating the SVD Bi-Diagonalization of a Batch of Small Matrices using GPUs},
	journal = {Journal of Computational Science},
	volume = {26},
	year = {2018},
	month = {2018-05},
	pages = {237{\textendash}245},
	abstract = {The acceleration of many small-sized linear algebra problems has become extremely challenging for current many-core architectures, and in particular GPUs. Standard interfaces have been proposed for some of these problems, called batched problems, so that they get targeted for optimization and used in a standard way in applications, calling them directly from highly optimized, standard numerical libraries, like (batched) BLAS and LAPACK. While most of the developments have been for one-sided factorizations and solvers, many important applications {\textendash} from big data analytics to information retrieval, low-rank approximations for solvers and preconditioners {\textendash} require two-sided factorizations, and most notably the SVD factorization. To address these needs and the parallelization challenges related to them, we developed a number of new batched computing techniques and designed batched Basic Linear Algebra Subroutines (BLAS) routines, and in particular the Level-2 BLAS GEMV and the Level-3 BLAS GEMM routines, to solve them. We propose a device functions-based methodology and big-tile setting techniques in our batched BLAS design. The different optimization techniques result in many software versions that must be tuned, for which we adopt an auto-tuning strategy to automatically derive the optimized instances of the routines. We illustrate our batched BLAS approach to optimize batched SVD bi-diagonalization progressively on GPUs. The progression is illustrated on an NVIDIA K40c GPU, and also, ported and presented on AMD Fiji Nano GPU, using AMD{\textquoteright}s Heterogeneous{\textendash}Compute Interface for Portability (HIP) C++ runtime API. We demonstrate achieving 80\% of the theoretically achievable peak performance for the overall algorithm, and significant acceleration of the Level-2 BLAS GEMV and Level-3 BLAS GEMM needed compared to vendor-optimized libraries on GPUs and multicore CPUs. The optimization techniques in this paper are applicable to the other two-sided factorizations as well.},
	keywords = {Batched, Eigenvalue and singular value problems, hardware accelerators, numerical linear algebra, Two-sided factorization algorithms},
	doi = {https://doi.org/10.1016/j.jocs.2018.01.007},
	author = {Tingxing Dong and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@techreport {1229,
	title = {Algorithms and Optimization Techniques for High-Performance Matrix-Matrix Multiplications of Very Small Matrices},
	number = {ICL-UT-18-09},
	year = {2018},
	month = {2018-09},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	abstract = {Expressing scientific computations in terms of BLAS, and in particular the general dense matrix-matrix multiplication (GEMM), is of fundamental importance for obtaining high performance portability across architectures. However, GEMMs for small matrices of sizes smaller than 32 are not sufficiently optimized in existing libraries. We consider the computation of many small GEMMs and its performance portability for a wide range of computer architectures, including Intel CPUs, ARM, IBM, Intel Xeon Phi, and GPUs. These computations often occur in applications like big data analytics, machine learning, high-order finite element methods (FEM), and others. The GEMMs are grouped together in a single batched routine. For these cases, we present algorithms and their optimization techniques that are specialized for the matrix sizes and architectures of interest. We derive a performance model and show that the new developments can be tuned to obtain performance that is within 90\% of the optimal for any of the architectures of interest. For example, on a V100 GPU for square matrices of size 32, we achieve an execution rate of about 1; 600 gigaFLOP/s in double-precision arithmetic, which is 95\% of the theoretically derived peak for this computation on a V100 GPU. We also show that these results outperform currently available state-of-the-art implementations such as vendor-tuned math libraries, including Intel MKL and NVIDIA CUBLAS, as well as open-source libraries like OpenBLAS and Eigen.},
	author = {Ian Masliah and Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Marc Baboulin and Jo{\"e}l Falcou and Jack Dongarra}
}
@article {1260,
	title = {Analysis and Design Techniques towards High-Performance and Energy-Efficient Dense Linear Solvers on GPUs},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {29},
	year = {2018},
	month = {2018-12},
	pages = {2700{\textendash}2712},
	abstract = {Graphics Processing Units (GPUs) are widely used in accelerating dense linear solvers. The matrix factorizations, which dominate the runtime for these solvers, are often designed using a hybrid scheme, where GPUs perform trailing matrix updates, while the CPUs perform the panel factorizations. Consequently, hybrid solutions require high-end CPUs and optimized CPU software in order to deliver high performance. Furthermore, they lack the energy efficiency inherent for GPUs due to the use of less energy-efficient CPUs, as well as CPU-GPU communications. This paper presents analysis and design techniques that overcome the shortcomings of the hybrid algorithms, and allow the design of high-performance and energy-efficient dense LU and Cholesky factorizations that use GPUs only. The full GPU solution eliminates the need for a high-end CPU and optimized CPU software, which leads to a better energy efficiency. We discuss different design choices, and introduce optimized GPU kernels for panel factorizations. The developed solutions achieve 90+ percent of the performance of optimized hybrid solutions, while improving the energy efficiency by 50 percent. They outperform the vendor library by 30-50 percent in single precision, and 15-50 percent in double precision. We also show that hybrid designs trail the proposed solutions in performance when optimized CPU software is not available.},
	keywords = {Dense linear solvers, energy efficiency, GPU computing},
	doi = {10.1109/TPDS.2018.2842785},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1268,
	title = {Autotuning in High-Performance Computing Applications},
	journal = {Proceedings of the IEEE},
	volume = {106},
	year = {2018},
	month = {2018-11},
	pages = {2068{\textendash}2083},
	abstract = {Autotuning refers to the automatic generation of a search space of possible implementations of a computation that are evaluated through models and/or empirical measurement to identify the most desirable implementation. Autotuning has the potential to dramatically improve the performance portability of petascale and exascale applications. To date, autotuning has been used primarily in high-performance applications through tunable libraries or previously tuned application code that is integrated directly into the application. This paper draws on the authors{\textquoteright} extensive experience applying autotuning to high-performance applications, describing both successes and future challenges. If autotuning is to be widely used in the HPC community, researchers must address the software engineering challenges, manage configuration overheads, and continue to demonstrate significant performance gains and portability across architectures. In particular, tools that configure the application must be integrated into the application build process so that tuning can be reapplied as the application and target architectures evolve.},
	keywords = {High-performance computing, performance tuning programming systems},
	doi = {10.1109/JPROC.2018.2841200},
	author = {Prasanna Balaprakash and Jack Dongarra and Todd Gamblin and Mary Hall and Jeffrey Hollingsworth and Boyana Norris and Richard Vuduc}
}
@article {1300,
	title = {Batched BLAS (Basic Linear Algebra Subprograms) 2018 Specification},
	year = {2018},
	month = {2018-07},
	abstract = {This document describes an API for Batch Basic Linear Algebra Subprograms (Batched BLAS or BBLAS). We focus on many independent BLAS operations on small matrices that are grouped together and processed by a single routine, called a Batched BLAS routine. The extensions beyond the original BLAS standard are considered that specify a programming interface not only for routines with uniformly-sized matrices and/or vectors but also for the situation where the sizes vary. The aim is to provide more efficient, but portable, implementations of algorithms on high-performance manycore platforms. These include multicore and many-core CPU processors; GPUs and coprocessors; as well as other hardware accelerators with floating-point compute facility.},
	author = {Jack Dongarra and Iain Duff and Mark Gates and Azzam Haidar and Sven Hammarling and Nicholas J. Higham and Jonathan Hogg and Pedro Valero Lara and Piotr Luszczek and Mawussi Zounon and Samuel D. Relton and Stanimire Tomov and Timothy Costa and Sarah Knepper}
}
@article {1209,
	title = {Batched One-Sided Factorizations of Tiny Matrices Using GPUs: Challenges and Countermeasures},
	journal = {Journal of Computational Science},
	volume = {26},
	year = {2018},
	month = {2018-05},
	pages = {226{\textendash}236},
	abstract = {The use of batched matrix computations recently gained a lot of interest for applications, where the same operation is applied to many small independent matrices. The batched computational pattern is frequently encountered in applications of data analytics, direct/iterative solvers and preconditioners, computer vision, astrophysics, and more, and often requires specific designs for vectorization and extreme parallelism to map well on today{\textquoteright}s high-end many-core architectures. This has led to the development of optimized software for batch computations, and to an ongoing community effort to develop standard interfaces for batched linear algebra software. Furthering these developments, we present GPU design and optimization techniques for high-performance batched one-sided factorizations of millions of tiny matrices (of size 32 and less). We quantify the effects and relevance of different techniques in order to select the best-performing LU, QR, and Cholesky factorization designs. While we adapt common optimization techniques, such as optimal memory traffic, register blocking, and concurrency control, we also show that a different mindset and techniques are needed when matrices are tiny, and in particular, sub-vector/warp in size. The proposed routines are part of the MAGMA library and deliver significant speedups compared to their counterparts in currently available vendor-optimized libraries. Notably, we tune the developments for the newest V100 GPU from NVIDIA to show speedups of up to 11.8{\texttimes}.},
	keywords = {batch computation, GPU computing, matrix factorization},
	doi = {https://doi.org/10.1016/j.jocs.2018.01.005},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1211,
	title = {Big Data and Extreme-Scale Computing: Pathways to Convergence - Toward a Shaping Strategy for a Future Software and Data Ecosystem for Scientific Inquiry},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	year = {2018},
	month = {2018-07},
	pages = {435{\textendash}479},
	abstract = {Over the past four years, the Big Data and Exascale Computing (BDEC) project organized a series of five international workshops that aimed to explore the ways in which the new forms of data-centric discovery introduced by the ongoing revolution in high-end data analysis (HDA) might be integrated with the established, simulation-centric paradigm of the high-performance computing (HPC) community. Based on those meetings, we argue that the rapid proliferation of digital data generators, the unprecedented growth in the volume and diversity of the data they generate, and the intense evolution of the methods for analyzing and using that data are radically reshaping the landscape of scientific computing. The most critical problems involve the logistics of wide-area, multistage workflows that will move back and forth across the computing continuum, between the multitude of distributed sensors, instruments and other devices at the networks edge, and the centralized resources of commercial clouds and HPC centers. We suggest that the prospects for the future integration of technological infrastructures and research ecosystems need to be considered at three different levels. First, we discuss the convergence of research applications and workflows that establish a research paradigm that combines both HPC and HDA, where ongoing progress is already motivating efforts at the other two levels. Second, we offer an account of some of the problems involved with creating a converged infrastructure for peripheral environments, that is, a shared infrastructure that can be deployed throughout the network in a scalable manner to meet the highly diverse requirements for processing, communication, and buffering/storage of massive data workflows of many different scientific domains. Third, we focus on some opportunities for software ecosystem convergence in big, logically centralized facilities that execute large-scale simulations and models and/or perform large-scale data analytics. We close by offering some conclusions and recommendations for future investment and policy review.},
	doi = {https://doi.org/10.1177/1094342018778123},
	author = {Mark Asch and Terry Moore and Rosa M. Badia and Micah Beck and Pete Beckman and Thierry Bidot and Fran{\c c}ois Bodin and Franck Cappello and Alok Choudhary and Bronis R. de Supinski and Ewa Deelman and Jack Dongarra and Anshu Dubey and Geoffrey Fox and Haohuan Fu and Sergi Girona and Michael Heroux and Yutaka Ishikawa and Kate Keahey and David Keyes and William T. Kramer and Jean-Fran{\c c}ois Lavignon and Yutong Lu and Satoshi Matsuoka and Bernd Mohr and St{\'e}phane Requena and Joel Saltz and Thomas Schulthess and Rick Stevens and Martin Swany and Alexander Szalay and William Tang and Ga{\"e}l Varoquaux and Jean-Pierre Vilotte and Robert W. Wisniewski and Zhiwei Xu and Igor Zacharov}
}
@article {1187,
	title = {Checkpointing Workflows for Fail-Stop Errors},
	journal = {IEEE Transactions on Computers},
	volume = {67},
	year = {2018},
	month = {2018-08},
	pages = {1105{\textendash}1120},
	abstract = {We consider the problem of orchestrating the execution of workflow applications structured as Directed Acyclic Graphs (DAGs) on parallel computing platforms that are subject to fail-stop failures. The objective is to minimize expected overall execution time, or makespan. A solution to this problem consists of a schedule of the workflow tasks on the available processors and of a decision of which application data to checkpoint to stable storage, so as to mitigate the impact of processor failures. To address this challenge, we consider a restricted class of graphs, Minimal Series-Parallel Graphs (M-SPGS), which is relevant to many real-world workflow applications. For this class of graphs, we propose a recursive list-scheduling algorithm that exploits the M-SPG structure to assign sub-graphs to individual processors, and uses dynamic programming to decide how to checkpoint these sub-graphs. We assess the performance of our algorithm for production workflow configurations, comparing it to an approach in which all application data is checkpointed and an approach in which no application data is checkpointed. Results demonstrate that our algorithm outperforms both the former approach, because of lower checkpointing overhead, and the latter approach, because of better resilience to failures.},
	keywords = {checkpoint, fail-stop error, resilience, workflow},
	url = {http://ieeexplore.ieee.org/document/8279499/},
	author = {Li Han and Louis-Claude Canon and Henri Casanova and Yves Robert and Frederic Vivien}
}
@techreport {1397,
	title = {A Collection of White Papers from the BDEC2 Workshop in Bloomington, IN},
	number = {ICL-UT-18-15},
	year = {2018},
	month = {2018-11},
	publisher = {University of Tennessee, Knoxville},
	author = {James Ahrens and Christopher M. Biwer and Alexandru Costan and Gabriel Antoniu and Maria S. P{\'e}rez and Nenad Stojanovic and Rosa Badia and Oliver Beckstein and Geoffrey Fox and Shantenu Jha and Micah Beck and Terry Moore and Sunita Chandrasekaran and Carlos Costa and Thierry Deutsch and Luigi Genovese and Tarek El-Ghazawi and Ian Foster and Dennis Gannon and Toshihiro Hanawa and Tevfik Kosar and William Kramer and Madhav V. Marathe and Christopher L. Barrett and Takemasa Miyoshi and Alex Pothen and Ariful Azad and Judy Qiu and Bo Peng and Ravi Teja and Sahil Tyagi and Chathura Widanage and Jon Koskey and Maryam Rahnemoonfar and Umakishore Ramachandran and Miles Deegan and William Tang and Osamu Tatebe and Michela Taufer and Michel Cuende and Ewa Deelman and Trilce Estrada and Rafael Ferreira Da Silva and Harrel Weinstein and Rodrigo Vargas and Miwako Tsuji and Kevin G. Yager and Wanling Gao and Jianfeng Zhan and Lei Wang and Chunjie Luo and Daoyi Zheng and Xu Wen and Rui Ren and Chen Zheng and Xiwen He and Hainan Ye and Haoning Tang and Zheng Cao and Shujie Zhang and Jiahui Dai}
}
@article {1263,
	title = {Computational Benefit of GPU Optimization for Atmospheric Chemistry Modeling},
	journal = {Journal of Advances in Modeling Earth Systems},
	volume = {10},
	year = {2018},
	month = {2018-08},
	pages = {1952{\textendash}1969},
	abstract = {Global chemistry-climate models are computationally burdened as the chemical mechanisms become more complex and realistic. Optimization for graphics processing units (GPU) may make longer global simulation with regional detail possible, but limited study has been done to explore the potential benefit for the atmospheric chemistry modeling. Hence, in this study, the second-order Rosenbrock solver of the chemistry module of CAM4-Chem is ported to the GPU to gauge potential speed-up. We find that on the CPU, the fastest performance is achieved using the Intel compiler with a block interleaved memory layout. Different combinations of compiler and memory layout lead to ~11.02{\texttimes} difference in the computational time. In contrast, the GPU version performs the best when using a combination of fully interleaved memory layout with block size equal to the warp size, CUDA streams for independent kernels, and constant memory. Moreover, the most efficient data transfer between CPU and GPU is gained by allocating the memory contiguously during the data initialization on the GPU. Compared to one CPU core, the speed-up of using one GPU alone reaches a factor of ~11.7{\texttimes} for the computation alone and ~3.82{\texttimes} when the data transfer between CPU and GPU is considered. Using one GPU alone is also generally faster than the multithreaded implementation for 16 CPU cores in a compute node and the single-source solution (OpenACC). The best performance is achieved by the implementation of the hybrid CPU/GPU version, but rescheduling the workload among the CPU cores is required before the practical CAM4-Chem simulation.},
	keywords = {compiler, CUDA, data transfer, gpu, hybrid, memory layout},
	doi = {https://doi.org/10.1029/2018MS001276},
	author = {Jian Sun and Joshua Fu and John Drake and Qingzhao Zhu and Azzam Haidar and Mark Gates and Stanimire Tomov and Jack Dongarra}
}
@article {1193,
	title = {Computing the Expected Makespan of Task Graphs in the Presence of Silent Errors},
	journal = {Parallel Computing},
	volume = {75},
	year = {2018},
	month = {2018-07},
	pages = {41{\textendash}60},
	abstract = {Applications structured as Directed Acyclic Graphs (DAGs) of tasks occur in many domains, including popular scientific workflows. DAG scheduling has thus received an enormous amount of attention. Many of the popular DAG scheduling heuristics make scheduling deci- sions based on path lengths. At large scale compute platforms are subject to various types of failures with non-negligible probabilities of occurrence. Failures that have recently re- ceived increased attention are {\textquotedblleft}silent errors,{\textquotedblright} which cause data corruption. Tolerating silent errors is done by checking the validity of computed results and possibly re-executing a task from scratch. The execution time of a task then becomes a random variable, and so do path lengths in a DAG. Unfortunately, computing the expected makespan of a DAG (and equivalently computing expected path lengths in a DAG) is a computationally dif- ficult problem. Consequently, designing effective scheduling heuristics in this context is challenging. In this work, we propose an algorithm that computes a first order approxi- mation of the expected makespan of a DAG when tasks are subject to silent errors. We find that our proposed approximation outperforms previously proposed approaches both in terms of approximation error and of speed.},
	keywords = {Expected makespan, fault-tolerance, scheduling, Scientific workflows, silent errors, Task graphs},
	doi = {https://doi.org/10.1016/j.parco.2018.03.004},
	author = {Henri Casanova and Julien Herrmann and Yves Robert}
}
@techreport {1200,
	title = {Data Movement Interfaces to Support Dataflow Runtimes},
	number = {ICL-UT-18-03},
	year = {2018},
	month = {2018-05},
	publisher = {University of Tennessee},
	abstract = {This document presents the design study and reports on the implementation of a portable hosted accelerator device support in the PaRSEC Dataflow Tasking at Exascale runtime, undertaken as part of the ECP contract 17-SC-20-SC. The document discusses different technological approaches to transfer data to/from hosted accelerators, issues recommendations for technology providers, and presents the design of an OpenMP-based accelerator support in PaRSEC.},
	author = {Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra}
}
@inproceedings {1259,
	title = {The Design of Fast and Energy-Efficient Linear Solvers: On the Potential of Half-Precision Arithmetic and Iterative Refinement Techniques},
	volume = {10860},
	year = {2018},
	month = {2018-06},
	pages = {586{\textendash}600},
	publisher = {Springer},
	address = {Wuxi, China},
	abstract = {As parallel computers approach exascale, power efficiency in high-performance computing (HPC) systems is of increasing concern. Exploiting both the hardware features and algorithms is an effective solution to achieve power efficiency, and to address the energy constraints in modern and future HPC systems. In this work, we present a novel design and implementation of an energy-efficient solution for dense linear systems of equations, which are at the heart of large-scale HPC applications. The proposed energy-efficient linear system solvers are based on two main components: (1) iterative refinement techniques, and (2) reduced-precision computing features in modern accelerators and coprocessors. While most of the energy efficiency approaches aim to reduce the consumption with a minimal performance penalty, our method improves both the performance and the energy efficiency. Compared to highly-optimized linear system solvers, our kernels deliver the same accuracy solution up to   2{\texttimes}  faster and reduce the energy consumption up to half on Intel Knights Landing (KNL) architectures. By efficiently using the Tensor Cores available in the NVIDIA V100 PCIe GPUs, the speedups can be up to   4{\texttimes} , with more than 80\% reduction in the energy consumption.},
	doi = {https://doi.org/10.1007/978-3-319-93698-7_45},
	url = {https://rdcu.be/bcKSC},
	author = {Azzam Haidar and Ahmad Abdelfattah and Mawussi Zounon and Panruo Wu and Srikara Pranesh and Stanimire Tomov and Jack Dongarra}
}
@techreport {1320,
	title = {Distributed Termination Detection for HPC Task-Based Environments},
	number = {ICL-UT-18-14},
	year = {2018},
	month = {2018-06},
	publisher = {University of Tennessee},
	abstract = {This paper revisits distributed termination detection algorithms in the context of high-performance computing applications in task systems. We first outline the need to efficiently detect termination in workflows for which the total number of tasks is data dependent and therefore not known statically but only revealed dynamically during execution. We introduce an efficient variant of the Credit Distribution Algorithm (CDA) and compare it to the original algorithm (HCDA) as well as to its two primary competitors: the Four Counters algorithm (4C) and the Efficient Delay-Optimal Distributed algorithm (EDOD). On the theoretical side, we analyze the behavior of each algorithm for some simplified task-based kernels and show the superiority of CDA in terms of the number of control messages. On the practical side, we provide a highly tuned implementation of each termination detection algorithm within PaRSEC and compare their performance for a variety of benchmarks, extracted from scientific applications that exhibit dynamic behaviors.},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Valentin Le F{\`e}vre and Yves Robert and Jack Dongarra}
}
@conference {1214,
	title = {Do moldable applications perform better on failure-prone HPC platforms?},
	booktitle = {11th Workshop on Resiliency in High Performance Computing in Clusters, Clouds, and Grids},
	series = {LNCS},
	year = {2018},
	month = {2018-08},
	publisher = {Springer Verlag},
	organization = {Springer Verlag},
	address = {Turin, Italy},
	abstract = {This paper compares the performance of different approaches to tolerate failures using checkpoint/restart when executed on large-scale failure-prone platforms. We study (i) Rigid applications, which use a constant number of processors throughout execution; (ii) Moldable applications, which can use a different number of processors after each restart following a fail-stop error; and (iii) GridShaped applications, which are moldable applications restricted to use rectangular processor grids (such as many dense linear algebra kernels). For each application type, we compute the optimal number of failures to tolerate before relinquishing the current allocation and waiting until a new resource can be allocated, and we determine the optimal yield that can be achieved. We instantiate our performance model with a realistic applicative scenario and make it publicly available for further usage.},
	author = {Valentin Le F{\`e}vre and George Bosilca and Aurelien Bouteiller and Thomas Herault and Atsushi Hori and Yves Robert and Jack Dongarra}
}
@techreport {1232,
	title = {Evaluation and Design of FFT for Distributed Accelerated Systems},
	number = {FFT-ECP ST-MS-10-1216},
	year = {2018},
	month = {2018-10},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	author = {Stanimire Tomov and Azzam Haidar and Daniel Schultz and Jack Dongarra}
}
@article {1201,
	title = {Evaluation of Dataflow Programming Models for Electronic Structure Theory},
	journal = {Concurrency and Computation: Practice and Experience: Special Issue on Parallel and Distributed Algorithms},
	volume = {2018},
	year = {2018},
	month = {2018-05},
	pages = {1{\textendash}20},
	abstract = {Dataflow programming models have been growing in popularity as a means to deliver a good balance between performance and portability in the post-petascale era. In this paper, we evaluate different dataflow programming models for electronic structure methods and compare them in terms of programmability, resource utilization, and scalability. In particular, we evaluate two programming paradigms for expressing scientific applications in a dataflow form: (1) explicit dataflow, where the dataflow is specified explicitly by the developer, and (2) implicit dataflow, where a task scheduling runtime derives the dataflow using per-task data-access information embedded in a serial program. We discuss our findings and present a thorough experimental analysis using methods from the NWChem quantum chemistry application as our case study, and OpenMP, StarPU, and PaRSEC as the task-based runtimes that enable the different forms of dataflow execution. Furthermore, we derive an abstract model to explore the limits of the different dataflow programming paradigms.},
	keywords = {CCSD, coupled cluster methods, dataflow, NWChem, OpenMP, parsec, StarPU, task-based runtime},
	doi = {https://doi.org/10.1002/cpe.4490},
	author = {Heike Jagode and Anthony Danalis and Reazul Hoque and Mathieu Faverge and Jack Dongarra}
}
@article {1089,
	title = {A Failure Detector for HPC Platforms},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	year = {2018},
	month = {2018-01},
	pages = {139{\textendash}158},
	abstract = {Building an infrastructure for exascale applications requires, in addition to many other key components, a stable and efficient failure detector. This article describes the design and evaluation of a robust failure detector that can maintain and distribute the correct list of alive resources within proven and scalable bounds. The detection and distribution of the fault information follow different overlay topologies that together guarantee minimal disturbance to the applications. A virtual observation ring minimizes the overhead by allowing each node to be observed by another single node, providing an unobtrusive behavior. The propagation stage uses a nonuniform variant of a reliable broadcast over a circulant graph overlay network and guarantees a logarithmic fault propagation. Extensive simulations, together with experiments on the Titan Oak Ridge National Laboratory supercomputer, show that the algorithm performs extremely well and exhibits all the desired properties of an exascale-ready algorithm.},
	keywords = {failure detection, Fault tolerance, MPI},
	doi = {https://doi.org/10.1177/1094342017711505},
	author = {George Bosilca and Aurelien Bouteiller and Amina Guermouche and Thomas Herault and Yves Robert and Pierre Sens and Jack Dongarra}
}
@conference {1215,
	title = {A Generic Approach to Scheduling and Checkpointing Workflows},
	booktitle = { The 47th International Conference on Parallel Processing (ICPP 2018)},
	year = {2018},
	month = {2018-08},
	publisher = {IEEE Computer Society Press},
	organization = {IEEE Computer Society Press},
	address = {Eugene, OR},
	abstract = {This work deals with scheduling and checkpointing strategies to execute scientific workflows on failure-prone large-scale platforms. To the best of our knowledge, this work is the first to target failstop errors for arbitrary workflows. Most previous work addresses soft errors, which corrupt the task being executed by a processor but do not cause the entire memory of that processor to be lost, contrarily to fail-stop errors. We revisit classical mapping heuristics such as HEFT and MinMin and complement them with several checkpointing strategies. The objective is to derive an efficient trade-off between checkpointing every task (CkptAll), which is an overkill when failures are rare events, and checkpointing no task (CkptNone), which induces dramatic re-execution overhead even when only a few failures strike during execution. Contrarily to previous work, our approach applies to arbitrary workflows, not just special classes of dependence graphs such as M-SPGs (Minimal Series-Parallel Graphs). Extensive experiments report significant gain over both CkptAll and CkptNone, for a wide variety of workflows.},
	author = {Li Han and Valentin Le F{\`e}vre and Louis-Claude Canon and Yves Robert and Frederic Vivien}
}
@article {1208,
	title = {A Guide for Achieving High Performance with Very Small Matrices on GPUs: A Case Study of Batched LU and Cholesky Factorizations},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {29},
	year = {2018},
	month = {2018-05},
	pages = {973{\textendash}984},
	abstract = {We present a high-performance GPU kernel with a substantial speedup over vendor libraries for very small matrix computations. In addition, we discuss most of the challenges that hinder the design of efficient GPU kernels for small matrix algorithms. We propose relevant algorithm analysis to harness the full power of a GPU, and strategies for predicting the performance, before introducing a proper implementation. We develop a theoretical analysis and a methodology for high-performance linear solvers for very small matrices. As test cases, we take the Cholesky and LU factorizations and show how the proposed methodology enables us to achieve a performance close to the theoretical upper bound of the hardware. This work investigates and proposes novel algorithms for designing highly optimized GPU kernels for solving batches of hundreds of thousands of small-size Cholesky and LU factorizations. Our focus on efficient batched Cholesky and batched LU kernels is motivated by the increasing need for these kernels in scientific simulations (e.g., astrophysics applications). Techniques for optimal memory traffic, register blocking, and tunable concurrency are incorporated in our proposed design. The proposed GPU kernels achieve performance speedups versus CUBLAS of up to 6x for the factorizations, using double precision arithmetic on an NVIDIA Pascal P100 GPU.},
	doi = {10.1109/TPDS.2017.2783929},
	author = {Azzam Haidar and Ahmad Abdelfattah and Mawussi Zounon and Stanimire Tomov and Jack Dongarra}
}
@conference {1264,
	title = {Harnessing GPU Tensor Cores for Fast FP16 Arithmetic to Speed up Mixed-Precision Iterative Refinement Solvers},
	booktitle = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC18)},
	year = {2018},
	month = {2018-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Dallas, TX},
	abstract = {Low-precision floating-point arithmetic is a powerful tool for accelerating scientific computing applications, especially those in artificial intelligence. Here, we present an investigation showing that other high-performance computing (HPC) applications can also harness this power. Specifically, we use the general HPC problem, Ax = b, where A is a large dense matrix, and a double precision (FP64) solution is needed for accuracy. Our approach is based on mixed-precision (FP16-FP64) iterative refinement, and we generalize and extend prior advances into a framework, for which we develop architecture-specific algorithms and highly tuned implementations. These new methods show how using half-precision Tensor Cores (FP16-TC) for the arithmetic can provide up to 4{\texttimes} speedup. This is due to the performance boost that the FP16-TC provide as well as to the improved accuracy over the classical FP16 arithmetic that is obtained because the GEMM accumulation occurs in FP32 arithmetic.},
	doi = {https://doi.org/10.1109/SC.2018.00050},
	author = {Azzam Haidar and Stanimire Tomov and Jack Dongarra and Nicholas J. Higham}
}
@article {1335,
	title = {Harnessing GPU{\textquoteright}s Tensor Cores Fast FP16 Arithmetic to Speedup Mixed-Precision Iterative Refinement Solvers and Achieve 74 Gflops/Watt on Nvidia V100},
	year = {2018},
	month = {2018-03},
	publisher = {GPU Technology Conference (GTC), Poster},
	address = {San Jose, CA},
	author = {Azzam Haidar and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@article {1158,
	title = {Incomplete Sparse Approximate Inverses for Parallel Preconditioning},
	journal = {Parallel Computing},
	volume = {71},
	year = {2018},
	month = {2018-01},
	pages = {1{\textendash}22},
	abstract = {In this paper, we propose a new preconditioning method that can be seen as a generalization of block-Jacobi methods, or as a simplification of the sparse approximate inverse (SAI) preconditioners. The {\textquotedblleft}Incomplete Sparse Approximate Inverses{\textquotedblright} (ISAI) is in particular efficient in the solution of sparse triangular linear systems of equations. Those arise, for example, in the context of incomplete factorization preconditioning. ISAI preconditioners can be generated via an algorithm providing fine-grained parallelism, which makes them attractive for hardware with a high concurrency level. In a study covering a large number of matrices, we identify the ISAI preconditioner as an attractive alternative to exact triangular solves in the context of incomplete factorization preconditioning.},
	issn = {01678191},
	doi = {10.1016/j.parco.2017.10.003},
	url = {http://www.sciencedirect.com/science/article/pii/S016781911730176X},
	author = {Hartwig Anzt and Thomas Huckle and J{\"u}rgen Br{\"a}ckle and Jack Dongarra}
}
@article {1199,
	title = {Investigating Power Capping toward Energy-Efficient Scientific Applications},
	journal = {Concurrency Computation: Practice and Experience},
	volume = {2018},
	year = {2018},
	month = {2018-04},
	pages = {1-14},
	abstract = {The emergence of power efficiency as a primary constraint in processor and system design poses new challenges concerning power and energy awareness for numerical libraries and scientific applications. Power consumption also plays a major role in the design of data centers, which may house petascale or exascale-level computing systems. At these extreme scales, understanding and improving the energy efficiency of numerical libraries and their related applications becomes a crucial part of the successful implementation and operation of the computing system. In this paper, we study and investigate the practice of controlling a compute system{\textquoteright}s power usage, and we explore how different power caps affect the performance of numerical algorithms with different computational intensities. Further, we determine the impact, in terms of performance and energy usage, that these caps have on a system running scientific applications. This analysis will enable us to characterize the types of algorithms that benefit most from these power management schemes. Our experiments are performed using a set of representative kernels and several popular scientific benchmarks. We quantify a number of power and performance measurements and draw observations and conclusions that can be viewed as a roadmap to achieving energy efficiency in the design and execution of scientific algorithms.},
	keywords = {energy efficiency, High Performance Computing, Intel Xeon Phi, Knights landing, papi, performance analysis, Performance Counters, power efficiency},
	doi = {https://doi.org/10.1002/cpe.4485},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/cpe.4485},
	author = {Azzam Haidar and Heike Jagode and Phil Vaccaro and Asim YarKhan and Stanimire Tomov and Jack Dongarra}
}
@article {1330,
	title = {MATEDOR: MAtrix, TEnsor, and Deep-learning Optimized Routines},
	year = {2018},
	month = {2018-11},
	publisher = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC18), Research Poster},
	address = {Dallas, TX},
	author = {Ahmad Abdelfattah and Jack Dongarra and Azzam Haidar and Stanimire Tomov and Ichitaro Yamazaki}
}
@article {1333,
	title = {MAtrix, TEnsor, and Deep-learning Optimized Routines (MATEDOR)},
	year = {2018},
	month = {2018-04},
	publisher = {NSF PI Meeting, Poster},
	address = {Washington, DC},
	doi = {https://doi.org/10.6084/m9.figshare.6174143.v3},
	author = {Azzam Haidar and Stanimire Tomov and Ahmad Abdelfattah and Ichitaro Yamazaki and Jack Dongarra}
}
@conference {1196,
	title = {Optimal Cooperative Checkpointing for Shared High-Performance Computing Platforms},
	booktitle = {2018 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW), Best Paper Award},
	year = {2018},
	month = {2018-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Vancouver, BC, Canada},
	abstract = {In high-performance computing environments, input/output (I/O) from various sources often contend for scarce available bandwidth. Adding to the I/O operations inherent to the failure-free execution of an application, I/O from checkpoint/restart (CR) operations (used to ensure progress in the presence of failures) place an additional burden as it increase I/O contention, leading to degraded performance. In this work, we consider a cooperative scheduling policy that optimizes the overall performance of concurrently executing CR-based applications which share valuable I/O resources. First, we provide a theoretical model and then derive a set of necessary constraints needed to minimize the global waste on the platform. Our results demonstrate that the optimal checkpoint interval, as defined by Young/Daly, despite providing a sensible metric for a single application, is not sufficient to optimally address resource contention at the platform scale. We therefore show that combining optimal checkpointing periods with I/O scheduling strategies can provide a significant improvement on the overall application performance, thereby maximizing platform throughput. Overall, these results provide critical analysis and direct guidance on checkpointing large-scale workloads in the presence of competing I/O while minimizing the impact on application performance.},
	doi = {10.1109/IPDPSW.2018.00127},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@conference {1210,
	title = {Optimizing GPU Kernels for Irregular Batch Workloads: A Case Study for Cholesky Factorization},
	booktitle = {IEEE High Performance Extreme Computing Conference (HPEC{\textquoteright}18)},
	year = {2018},
	month = {2018-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {This paper introduces several frameworks for the design and implementation of high performance GPU kernels that target batch workloads with irregular sizes. Such workloads are ubiquitous in many scientific applications, including sparse direct solvers, astrophysics, and quantum chemistry. The paper addresses two main categories of frameworks, taking the Cholesky factorization as a case study. The first uses hostside kernel launches, and the second uses device-side launches. Within each category, different design options are introduced, with an emphasis on the advantages and the disadvantages of each approach. Our best performing design outperforms the state-of-the-art CPU implementation, scoring up to 4.7{\texttimes} speedup in double precision on a Pascal P100 GPU.},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1227,
	title = {PMIx: Process Management for Exascale Environments},
	journal = {Parallel Computing},
	volume = {79},
	year = {2018},
	month = {2018-01},
	pages = {9{\textendash}29},
	issn = {01678191},
	doi = {10.1016/j.parco.2018.08.002},
	url = {https://linkinghub.elsevier.com/retrieve/pii/S0167819118302424https://api.elsevier.com/content/article/PII:S0167819118302424?httpAccept=text/xmlhttps://api.elsevier.com/content/article/PII:S0167819118302424?httpAccept=text/plain},
	author = {Ralph Castain and Joshua Hursey and Aurelien Bouteiller and David Solt}
}
@article {1192,
	title = {Production Implementations of Pipelined \& Communication-Avoiding Iterative Linear Solvers},
	year = {2018},
	month = {2018-03},
	publisher = {SIAM Conference on Parallel Processing for Scientific Computing},
	address = {Tokyo, Japan},
	author = {Mark Hoemmen and Ichitaro Yamazaki}
}
@article {1258,
	title = {The Singular Value Decomposition: Anatomy of Optimizing an Algorithm for Extreme Scale},
	journal = {SIAM Review},
	volume = {60},
	year = {2018},
	month = {2018-11},
	pages = {808{\textendash}865},
	abstract = {The computation of the singular value decomposition, or SVD, has a long history with many improvements over the years, both in its implementations and algorithmically. Here, we survey the evolution of SVD algorithms for dense matrices, discussing the motivation and performance impacts of changes. There are two main branches of dense SVD methods: bidiagonalization and Jacobi. Bidiagonalization methods started with the implementation by Golub and Reinsch in Algol60, which was subsequently ported to Fortran in the EISPACK library, and was later more efficiently implemented in the LINPACK library, targeting contemporary vector machines. To address cache-based memory hierarchies, the SVD algorithm was reformulated to use Level 3 BLAS in the LAPACK library. To address new architectures, ScaLAPACK was introduced to take advantage of distributed computing, and MAGMA was developed for accelerators such as GPUs. Algorithmically, the divide and conquer and MRRR algorithms were developed to reduce the number of operations. Still, these methods remained memory bound, so two-stage algorithms were developed to reduce memory operations and increase the computational intensity, with efficient implementations in PLASMA, DPLASMA, and MAGMA. Jacobi methods started with the two-sided method of Kogbetliantz and the one-sided method of Hestenes. They have likewise had many developments, including parallel and block versions and preconditioning to improve convergence. In this paper, we investigate the impact of these changes by testing various historical and current implementations on a common, modern multicore machine and a distributed computing platform. We show that algorithmic and implementation improvements have increased the speed of the SVD by several orders of magnitude, while using up to 40 times less energy.},
	keywords = {bidiagonal matrix, bisection, Divide and conquer, Hestenes method, Jacobi method, Kogbetliantz method, MRRR, QR iteration, Singular value decomposition, SVD},
	issn = {0036-1445},
	doi = {10.1137/17M1117732},
	url = {https://epubs.siam.org/doi/10.1137/17M1117732},
	author = {Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Ichitaro Yamazaki}
}
@techreport {1275,
	title = {Software-Defined Events (SDEs) in MAGMA-Sparse},
	number = {ICL-UT-18-12},
	year = {2018},
	month = {2018-12},
	publisher = {University of Tennessee},
	author = {Heike Jagode and Anthony Danalis and Hartwig Anzt and Ichitaro Yamazaki and Mark Hoemmen and Erik Boman and Stanimire Tomov and Jack Dongarra}
}
@techreport {1204,
	title = {Solver Interface \& Performance on Cori},
	number = {ICL-UT-18-05},
	year = {2018},
	month = {2018-06},
	publisher = {University of Tennessee},
	author = {Hartwig Anzt and Ichitaro Yamazaki and Mark Hoemmen and Erik Boman and Jack Dongarra}
}
@techreport {1280,
	title = {Tensor Contraction on Distributed Hybrid Architectures using a Task-Based Runtime System},
	number = {ICL-UT-18-13},
	year = {2018},
	month = {2018-12},
	publisher = {University of Tennessee},
	abstract = {The needs for predictive simulation of electronic structure in chemistry and materials science calls for fast/reduced-scaling formulations of quantum n-body methods that replace the traditional dense tensors with element-, block-, rank-, and block-rank-sparse (data-sparse) tensors. The resulting, highly irregular data structures are a poor match to imperative, bulk-synchronous parallel programming style due to the dynamic nature of the problem and to the lack of clear domain decomposition to guarantee a fair load-balance. TESSE runtime and the associated programming model aim to support performance-portable composition of applications involving irregular and dynamically changing data. In this paper we report an implementation of irregular dense tensor contraction in a paradigmatic electronic structure application based on the TESSE extension of PaRSEC, a distributed hybrid task runtime system, and analyze the resulting performance on a distributed memory cluster of multi-GPU nodes. Unprecedented strong scaling and promising efficiency indicate a viable future for task-based programming of complete production-quality reduced scaling models of electronic structure.},
	author = {George Bosilca and Damien Genet and Robert Harrison and Thomas Herault and Mohammad Mahdi Javanmard and Chong Peng and Edward Valeev}
}
@article {1334,
	title = {Tensor Contractions using Optimized Batch GEMM Routines},
	year = {2018},
	month = {2018-03},
	publisher = {GPU Technology Conference (GTC), Poster},
	address = {San Jose, CA},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {1265,
	title = {Using GPU FP16 Tensor Cores Arithmetic to Accelerate Mixed-Precision Iterative Refinement Solvers and Reduce Energy Consumption},
	booktitle = {ISC High Performance (ISC{\textquoteright}18), Best Poster},
	year = {2018},
	month = {2018-06},
	address = {Frankfurt, Germany},
	author = {Azzam Haidar and Stanimire Tomov and Ahmad Abdelfattah and Mawussi Zounon and Jack Dongarra}
}
@article {1332,
	title = {Using GPU FP16 Tensor Cores Arithmetic to Accelerate Mixed-Precision Iterative Refinement Solvers and Reduce Energy Consumption},
	year = {2018},
	month = {2018-06},
	publisher = {ISC High Performance (ISC18), Best Poster Award},
	address = {Frankfurt, Germany},
	author = {Azzam Haidar and Stanimire Tomov and Ahmad Abdelfattah and Mawussi Zounon and Jack Dongarra}
}
@article {1341,
	title = {Accelerating Tensor Contractions in High-Order FEM with MAGMA Batched},
	year = {2017},
	month = {2017-03},
	publisher = {SIAM Conference on Computer Science and Engineering (SIAM CSE17), Presentation},
	address = {Atlanta, GA},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Christopher Earl and Jo{\"e}l Falcou and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@article {1176,
	title = {Argobots: A Lightweight Low-Level Threading and Tasking Framework},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	year = {2017},
	month = {2017-10},
	abstract = {In the past few decades, a number of user-level threading and tasking models have been proposed in the literature to address the shortcomings of OS-level threads, primarily with respect to cost and flexibility. Current state-of-the-art user-level threading and tasking models, however, are either too specific to applications or architectures or are not as powerful or flexible. In this paper, we present Argobots, a lightweight, low-level threading and tasking framework that is designed as a portable and performant substrate for high-level programming models or runtime systems. Argobots offers a carefully designed execution model that balances generality of functionality with providing a rich set of controls to allow specialization by the user or high-level programming model. We describe the design, implementation, and optimization of Argobots and present integrations with three example high-level models: OpenMP, MPI, and co-located I/O service. Evaluations show that (1) Argobots outperforms existing generic threading runtimes; (2) our OpenMP runtime offers more efficient interoperability capabilities than production OpenMP runtimes do; (3) when MPI interoperates with Argobots instead of Pthreads, it enjoys reduced synchronization costs and better latency hiding capabilities; and (4) I/O service with Argobots reduces interference with co-located applications, achieving performance competitive with that of the Pthreads version.},
	keywords = {Argobots, context switch, I/O, interoperability, lightweight, MPI, OpenMP, stackable scheduler, tasklet, user-level thread},
	doi = {10.1109/TPDS.2017.2766062},
	url = {http://ieeexplore.ieee.org/document/8082139/},
	author = {Sangmin Seo and Abdelhalim Amer and Pavan Balaji and Cyril Bordage and George Bosilca and Alex Brooks and Philip Carns and Adrian Castello and Damien Genet and Thomas Herault and Shintaro Iwasaki and Prateek Jindal and Sanjay Kale and Sriram Krishnamoorthy and Jonathan Lifflander and Huiwei Lu and Esteban Meneses and Mar Snir and Yanhua Sun and Kenjiro Taura and Pete Beckman}
}
@techreport {1175,
	title = {C++ API for Batch BLAS},
	number = {04, ICL-UT-17-12},
	year = {2017},
	month = {2017-12},
	publisher = {University of Tennessee},
	author = {Ahmad Abdelfattah and Konstantin Arturov and Cris Cecka and Jack Dongarra and Chip Freitag and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Panruo Wu}
}
@conference {1098,
	title = {Checkpointing Workflows for Fail-Stop Errors},
	booktitle = {IEEE Cluster},
	year = {2017},
	month = {2017-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Honolulu, Hawaii},
	abstract = {We consider the problem of orchestrating the execution of workflow applications structured as Directed Acyclic Graphs (DAGs) on parallel computing platforms that are subject to fail-stop failures. The objective is to minimize expected overall execution time, or makespan. A solution to this problem consists of a schedule of the workflow tasks on the available processors and of a decision of which application data to checkpoint to stable storage, so as to mitigate the impact of processor failures. For general DAGs this problem is hopelessly intractable. In fact, given a solution, computing its expected makespan is still a difficult problem. To address this challenge, we consider a restricted class
of graphs, Minimal Series-Parallel Graphs (M-SPGS). It turns out that many real-world workflow applications are naturally structured as M-SPGS. For this class of graphs, we propose a recursive list-scheduling algorithm that exploits the M-SPG structure to assign sub-graphs to individual processors, and uses dynamic programming to decide which tasks in these sub-graphs should be checkpointed. Furthermore, it is possible to efficiently compute the expected makespan for the solution produced by this algorithm, using a first-order approximation of task weights and existing evaluation algorithms for 2-state probabilistic DAGs. We assess the performance of our algorithm for production workflow configurations, comparing it to (i) an approach in which all application data is checkpointed, which corresponds to the standard way in which most production workflows are executed today; and (ii) an approach in which no application data is checkpointed. Our results demonstrate that our algorithm strikes a good compromise between these two approaches, leading to lower checkpointing overhead than the former and to better resilience to failure than the latter.},
	author = {Li Han and Louis-Claude Canon and Henri Casanova and Yves Robert and Frederic Vivien}
}
@article {1131,
	title = {Comparing performance of s-step and pipelined GMRES on distributed-memory multicore CPUs},
	year = {2017},
	month = {2017-07},
	publisher = {SIAM Annual Meeting},
	address = {Pittsburgh, Pennsylvania},
	author = {Ichitaro Yamazaki and Mark Hoemmen and Piotr Luszczek and Jack Dongarra}
}
@conference {1168,
	title = {The Design and Performance of Batched BLAS on Modern High-Performance Computing Systems},
	booktitle = {International Conference on Computational Science (ICCS 2017)},
	year = {2017},
	month = {2017-06},
	publisher = {Elsevier},
	organization = {Elsevier},
	address = {Z{\"u}rich, Switzerland},
	abstract = {A current trend in high-performance computing is to decompose a large linear algebra problem into batches containing thousands of smaller problems, that can be solved independently, before collating the results. To standardize the interface to these routines, the community is developing an extension to the BLAS standard (the batched BLAS), enabling users to perform thousands of small BLAS operations in parallel whilst making efficient use of their hardware. We discuss the benefits and drawbacks of the current batched BLAS proposals and perform a number of experiments, focusing on a general matrix-matrix multiplication (GEMM), to explore their affect on the performance. In particular we analyze the effect of novel data layouts which, for example, interleave the matrices in memory to aid vectorization and prefetching of data. Utilizing these modifications our code outperforms both MKL1 CuBLAS2 by up to 6 times on the self-hosted Intel KNL (codenamed Knights Landing) and Kepler GPU architectures, for large numbers of double precision GEMM operations using matrices of size 2 {\texttimes} 2 to 20 {\texttimes} 20.},
	keywords = {Batched BLAS, BLAS, High-performance computing, Memory management, Parallel processing, Scientific computing},
	doi = {DOI:10.1016/j.procs.2017.05.138},
	author = {Jack Dongarra and Sven Hammarling and Nicholas J. Higham and Samuel Relton and Pedro Valero-Lara and Mawussi Zounon}
}
@inproceedings {1183,
	title = {Dynamic Task Discovery in PaRSEC- A data-flow task-based Runtime},
	year = {2017},
	month = {2017-09},
	publisher = {ACM},
	address = {Denver},
	abstract = {Successfully exploiting distributed collections of heterogeneous many-cores architectures with complex memory hierarchy through a portable programming model is a challenge for application developers. The literature is not short of proposals addressing this problem, including many evolutionary solutions that seek to extend the capabilities of current message passing paradigms with intranode features (MPI+X). A different, more revolutionary, solution explores data-flow task-based runtime systems as a substitute to both local and distributed data dependencies management. The solution explored in this paper, PaRSEC, is based on such a programming paradigm, supported by a highly efficient task-based runtime. This paper compares two programming paradigms present in PaRSEC, Parameterized Task Graph (PTG) and Dynamic Task Discovery (DTD) in terms of capabilities, overhead and potential benefits.},
	keywords = {data-flow, dynamic task-graph, parsec, task-based runtime},
	isbn = {978-1-4503-5125-6},
	doi = {10.1145/3148226.3148233},
	url = {https://dl.acm.org/citation.cfm?doid=3148226.3148233},
	author = {Reazul Hoque and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {1103,
	title = {Factorization and Inversion of a Million Matrices using GPUs: Challenges and Countermeasures},
	journal = {Procedia Computer Science},
	volume = {108},
	year = {2017},
	month = {2017-06},
	pages = {606{\textendash}615},
	abstract = {This paper presents new algorithmic approaches and optimization techniques for LU factorization and matrix inversion of millions of very small matrices using GPUs. These problems appear in many scientific applications including astrophysics and generation of block-Jacobi preconditioners. We show that, for very small problem sizes, design and optimization of GPU kernels require a mindset different from the one usually used when designing LAPACK algorithms for GPUs. Techniques for optimal memory traffic, register blocking, and tunable concurrency are incorporated in our proposed design. We also take advantage of the small matrix sizes to eliminate the intermediate row interchanges in both the factorization and inversion kernels. The proposed GPU kernels achieve performance speedups vs. CUBLAS of up to 6{\texttimes} for the factorization, and 14{\texttimes} for the inversion, using double precision arithmetic on a Pascal P100 GPU.},
	doi = {https://doi.org/10.1016/j.procs.2017.05.250},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1102,
	title = {Fast Cholesky Factorization on GPUs for Batch and Native Modes in MAGMA},
	journal = {Journal of Computational Science},
	volume = {20},
	year = {2017},
	month = {2017-05},
	pages = {85{\textendash}93},
	abstract = {This paper presents a GPU-accelerated Cholesky factorization for two different modes of operation. The first one is the batch mode, where many independent factorizations on small matrices can be performed concurrently. This mode supports fixed size and variable size problems, and is found in many scientific applications. The second mode is the native mode, where one factorization is performed on a large matrix without any CPU involvement, which allows the CPU do other useful work. We show that, despite the different workloads, both modes of operation share a common code-base that uses the GPU only. We also show that the developed routines achieve significant speedups against a multicore CPU using the MKL library, and against a GPU implementation by cuSOLVER. This work is part of the MAGMA library.},
	keywords = {GPU computing; Cholesky factorization; Batched execution},
	doi = {https://doi.org/10.1016/j.jocs.2016.12.009},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1083,
	title = {A Framework for Out of Memory SVD Algorithms},
	journal = {ISC High Performance 2017},
	year = {2017},
	month = {2017-06},
	pages = {158{\textendash}178},
	abstract = {Many important applications {\textendash} from big data analytics to information retrieval, gene expression analysis, and numerical weather prediction {\textendash} require the solution of large dense singular value decompositions (SVD). In many cases the problems are too large to fit into the computer{\textquoteright}s main memory, and thus require specialized out-of-core algorithms that use disk storage. In this paper, we analyze the SVD communications, as related to hierarchical memories, and design a class of algorithms that minimizes them. This class includes out-of-core SVDs but can also be applied between other consecutive levels of the memory hierarchy, e.g., GPU SVD using the CPU memory for large problems. We call these out-of-memory (OOM) algorithms. To design OOM SVDs, we first study the communications for both classical one-stage blocked SVD and two-stage tiled SVD. We present the theoretical analysis and strategies to design, as well as implement, these communication avoiding OOM SVD algorithms. We show performance results for multicore architecture that illustrate our theoretical findings and match our performance models.},
	doi = {https://doi.org/10.1007/978-3-319-58667-0_9},
	author = {Khairul Kabir and Azzam Haidar and Stanimire Tomov and Aurelien Bouteiller and Jack Dongarra}
}
@conference {1142,
	title = {High-performance Cholesky Factorization for GPU-only Execution},
	booktitle = {Proceedings of the General Purpose GPUs (GPGPU-10)},
	year = {2017},
	month = {2017-02},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {We present our performance analysis, algorithm designs, and the optimizations needed for the development of high-performance GPU-only algorithms, and in particular, for the dense Cholesky factorization. In contrast to currently promoted designs that solve parallelism challenges on multicore architectures by representing algorithms as Directed Acyclic Graphs (DAGs), where nodes are tasks of fine granularity and edges are the dependencies between the tasks, our designs explicitly target manycore architectures like GPUs and feature coarse granularity tasks (that can be hierarchically split into fine grain data-parallel subtasks). Furthermore, in contrast to hybrid algorithms that schedule difficult to parallelize tasks on CPUs, we develop highly-efficient code for entirely GPU execution. GPU-only codes remove the expensive CPU-to-GPU communications and the tuning challenges related to slow CPU and/or low CPU-to-GPU bandwidth. We show that on latest GPUs, like the P100, this becomes so important that the GPU-only code even outperforms the hybrid MAGMA algorithms when the CPU tasks and communications can not be entirely overlapped with GPU computations. We achieve up to 4,300 GFlop/s in double precision on a P100 GPU, which is about 7-8{\texttimes} faster than high-end multicore CPUs, e.g., two 10-cores Intel Xeon E5-2650 v3 Haswell CPUs, where MKL runs up to about 500-600 Gflop/s. The new algorithm also outperforms significantly the GPU-only implementation currently available in the NVIDIA cuSOLVER library.},
	doi = {https://doi.org/10.1145/3038228.3038237},
	author = {Azzam Haidar and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {1011,
	title = {Improving Performance of GMRES by Reducing Communication and Pipelining Global Collectives},
	year = {2017},
	month = {2017-06},
	address = {Orlando, FL},
	abstract = {We compare the performance of pipelined and s-step GMRES, respectively referred to as l-GMRES and s-GMRES, on distributed multicore CPUs. Compared to standard GMRES, s-GMRES requires fewer all-reduces, while l-GMRES overlaps the all-reduces with computation. To combine the best features of two algorithms, we propose another variant, (l, t)-GMRES, that not only does fewer global all-reduces than standard GMRES, but also overlaps those all-reduces with other work. We implemented the thread-parallelism and communication-overlap in two different ways. The first uses nonblocking MPI collectives with thread-parallel computational kernels. The second relies on a shared-memory task scheduler. In our experiments, (l, t)-GMRES performed better than l-GMRES by factors of up to 1.67{\texttimes}. In addition, though we only used 50 nodes, when the latency cost became significant, our variant performed up to 1.22{\texttimes} better than s-GMRES by hiding all-reduces.},
	doi = {https://doi.org/10.1109/IPDPSW.2017.65},
	author = {Ichitaro Yamazaki and Mark Hoemmen and Piotr Luszczek and Jack Dongarra}
}
@conference {1140,
	title = {Investigating Half Precision Arithmetic to Accelerate Dense Linear System Solvers},
	booktitle = {ScalA17: 8th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2017},
	month = {11/2017},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	abstract = {The use of low-precision arithmetic in mixed-precision computing methods has been a powerful tool to accelerate numerous scientific computing applications. Artificial intelligence (AI) in particular has pushed this to current extremes, making use of half-precision floating-point arithmetic (FP16) in approaches based on neural networks. The appeal of FP16 is in the high performance that can be achieved using it on today{\textquoteright}s powerful manycore GPU accelerators, e.g., like the NVIDIA V100, that can provide 120 TeraFLOPS alone in FP16. We present an investigation showing that other HPC applications can harness this power too, and in particular, the general HPC problem of solving Ax = b, where A is a large dense matrix, and the solution is needed in FP32 or FP64 accuracy. Our approach is based on the mixed-precision iterative refinement technique {\textendash} we generalize and extend prior advances into a framework, for which we develop architecture-specific algorithms and highly-tuned implementations that resolve the main computational challenges of efficiently parallelizing, scaling, and using FP16 arithmetic in the approach on high-end GPUs. Subsequently, we show for a first time how the use of FP16 arithmetic can significantly accelerate, as well as make more energy efficient, FP32 or FP64-precision Ax = b solvers. Our results are reproducible and the developments will be made available through the MAGMA library. We quantify in practice the performance, and limitations of the approach.},
	author = {Azzam Haidar and Panruo Wu and Stanimire Tomov and Jack Dongarra}
}
@article {1340,
	title = {MAGMA Tensors and Batched Computing for Accelerating Applications on GPUs},
	year = {2017},
	month = {2017-05},
	publisher = {GPU Technology Conference (GTC17), Presentation in Session S7728},
	address = {San Jose, CA},
	author = {Stanimire Tomov and Azzam Haidar}
}
@article {1337,
	title = {MagmaDNN {\textendash} High-Performance Data Analytics for Manycore GPUs and CPUs},
	year = {2017},
	month = {2017-12},
	publisher = {2017 Summer Research Experiences for Undergraduate (REU), Presentation},
	address = {Knoxville, TN},
	author = {Lucien Ng and Kwai Wong and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@techreport {1130,
	title = {MAGMA-sparse Interface Design Whitepaper},
	number = {ICL-UT-17-05},
	year = {2017},
	month = {2017-09},
	type = {Technical Report},
	abstract = {In this report we describe the logic and interface we develop for the MAGMA-sparse library
to allow for easy integration as third-party library into a top-level software ecosystem. The
design choices are based on extensive consultation with other software library developers, in
particular the Trilinos software development team. The interface documentation is at this point
not exhaustive, but a first proposal for setting a standard. Although the interface description
targets the MAGMA-sparse software module, we hope that the design choices carry beyond this
specific library, and are attractive for adoption in other packages.
This report is not intended as static document, but will be updated over time to reflect the agile
software development in the ECP 1.3.3.11 STMS11-PEEKS project.},
	author = {Hartwig Anzt and Erik Boman and Jack Dongarra and Goran Flegar and Mark Gates and Mike Heroux and Mark Hoemmen and Jakub Kurzak and Piotr Luszczek and Sivasankaran Rajamanickam and Stanimire Tomov and Stephen Wood and Ichitaro Yamazaki}
}
@conference {1084,
	title = {Novel HPC Techniques to Batch Execution of Many Variable Size BLAS Computations on GPUs},
	booktitle = {International Conference on Supercomputing (ICS {\textquoteright}17)},
	year = {2017},
	month = {2017-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Chicago, Illinois},
	doi = {10.1145/3079079.3079103},
	url = {http://dl.acm.org/citation.cfm?id=3079103},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {1170,
	title = {Optimized Batched Linear Algebra for Modern Architectures},
	booktitle = {Euro-Par 2017},
	year = {2017},
	month = {2017-08},
	publisher = {Springer},
	organization = {Springer},
	address = {Santiago de Compostela, Spain},
	abstract = {Solving large numbers of small linear algebra problems simultaneously is becoming increasingly important in many application areas. Whilst many researchers have investigated the design of efficient batch linear algebra kernels for GPU architectures, the common approach for many/multi-core CPUs is to use one core per subproblem in the batch. When solving batches of very small matrices, 2 {\texttimes} 2 for example, this design exhibits two main issues: it fails to fully utilize the vector units and the cache of modern architectures, since the matrices are too small. Our approach to resolve this is as follows: given a batch of small matrices spread throughout the primary memory, we first reorganize the elements of the matrices into a contiguous array, using a block interleaved memory format, which allows us to process the small independent problems as a single large matrix problem and enables cross-matrix vectorization. The large problem is solved using blocking strategies that attempt to optimize the use of the cache. The solution is then converted back to the original storage format. To explain our approach we focus on two BLAS routines: general matrix-matrix multiplication (GEMM) and the triangular solve (TRSM). We extend this idea to LAPACK routines using the Cholesky factorization and solve (POSV). Our focus is primarily on very small matrices ranging in size from 2 {\texttimes} 2 to 32 {\texttimes} 32. Compared to both MKL and OpenMP implementations, our approach can be up to 4 times faster for GEMM, up to 14 times faster for TRSM, and up to 40 times faster for POSV on the new Intel Xeon Phi processor, code-named Knights Landing (KNL). Furthermore, we discuss strategies to avoid data movement between sockets when using our interleaved approach on a NUMA node.},
	doi = {https://doi.org/10.1007/978-3-319-64203-1_37},
	author = {Jack Dongarra and Sven Hammarling and Nicholas J. Higham and Samuel Relton and Mawussi Zounon}
}
@conference {1085,
	title = {Optimizing the SVD Bidiagonalization Process for a Batch of Small Matrices},
	booktitle = {International Conference on Computational Science (ICCS 2017)},
	year = {2017},
	month = {2017-06},
	publisher = {Procedia Computer Science},
	organization = {Procedia Computer Science},
	address = {Zurich, Switzerland},
	abstract = {A challenging class of problems arising in many GPU applications, called batched problems, involves linear algebra operations on many small-sized matrices. We designed batched BLAS (Basic Linear Algebra Subroutines) routines, and in particular the Level-2 BLAS GEMV and the Level-3 BLAS GEMM routines, to solve them. We proposed device functions and big-tile settings in our batched BLAS design. We adopted auto-tuning to optimize different instances of GEMV routines. We illustrated our batched BLAS approach to optimize batched bi-diagonalization progressively on a K40c GPU. The optimization techniques in this paper are applicable to the other two-sided factorizations as well.},
	doi = {https://doi.org/10.1016/j.procs.2017.05.237},
	url = {http://www.sciencedirect.com/science/article/pii/S1877050917308645},
	author = {Tingxing Dong and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {1141,
	title = {Out of Memory SVD Solver for Big Data},
	booktitle = {2017 IEEE High Performance Extreme Computing Conference (HPEC{\textquoteright}17)},
	year = {2017},
	month = {2017-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {Many applications {\textendash} from data compression to numerical weather prediction and information retrieval {\textendash} need to compute large dense singular value decompositions (SVD). When the problems are too large to fit into the computer{\textquoteright}s main memory, specialized out-of-core algorithms that use disk storage are required. A typical example is when trying to analyze a large data set through tools like MATLAB or Octave, but the data is just too large to be loaded. To overcome this, we designed a class of out-of-memory (OOM) algorithms to reduce, as well as overlap communication with computation. Of particular interest is OOM algorithms for matrices of size m{\texttimes}n, where m >> n or m << n, e.g., corresponding to cases of too many variables, or too many observations. To design OOM SVDs, we first study the communications cost for the SVD techniques as well as for the QR/LQ factorization followed by SVD. We present the theoretical analysis about the data movement cost and strategies to design OOM SVD algorithms. We show performance results for multicore architecture that illustrate our theoretical findings and match our performance models. Moreover, our experimental results show the feasibility and superiority of the OOM SVD.},
	author = {Azzam Haidar and Khairul Kabir and Diana Fayad and Stanimire Tomov and Jack Dongarra}
}
@inbook {1384,
	title = {Performance Analysis and Debugging Tools at Scale},
	booktitle = {Exascale Scientific Applications: Scalability and Performance Portability},
	year = {2017},
	month = {2017-11},
	pages = {17-50},
	publisher = {Chapman \& Hall / CRC Press},
	organization = {Chapman \& Hall / CRC Press},
	chapter = {2},
	abstract = {This chapter explores present-day challenges and those likely to arise as new hardware and software technologies are introduced on the path to exascale. It covers some of the underlying hardware, software, and techniques that enable tools and debuggers. Performance tools and debuggers are critical components that enable computational scientists to fully exploit the computing power of While high-performance computing systems. Instrumentation is the insertion of code to perform measurement in a program. It is vital step in performance analysis, especially for parallel programs. The essence of a debugging tool is enabling observation, exploration, and control of program state, such that a developer can, for example, verify that what is currently occurring correlates to what is intended. The increased complexity and volume of performance and debugging data likely to be seen on exascale systems risks overwhelming tool users. Tools and debuggers may need to develop advanced techniques such as automated filtering and analysis to reduce the complexity seen by the user.},
	isbn = {9781315277400},
	doi = {https://doi.org/10.1201/b21930},
	author = {Scott Parker and John Mellor-Crummey and Dong H. Ahn and Heike Jagode and Holger Brunst and Sameer Shende and Allen D. Malony and David DelSignore and Ronny Tschuter and Ralph Castain and Kevin Harms and Philip Carns and Ray Loy and Kalyan Kumaran}
}
@techreport {1173,
	title = {PLASMA 17 Performance Report},
	number = {ICL-UT-17-11},
	year = {2017},
	month = {2017-06},
	publisher = {University of Tennessee},
	abstract = {PLASMA (Parallel Linear Algebra for Multicore Architectures) is a dense linear algebra package at the forefront of multicore computing. PLASMA is designed to deliver the highest possible performance from a system with multiple sockets of multicore processors. PLASMA achieves this objective by combining state of the art solutions in parallel algorithms, scheduling, and software engineering. PLASMA currently offers a collection of routines for solving linear systems of equations and least square problems.},
	author = {Maksims Abalenkovs and Negin Bagherpour and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Samuel Relton and Jakub Sistek and David Stevens and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Mawussi Zounon}
}
@techreport {1172,
	title = {PLASMA 17.1 Functionality Report},
	number = {ICL-UT-17-10},
	year = {2017},
	month = {2017-06},
	publisher = {University of Tennessee},
	abstract = {PLASMA (Parallel Linear Algebra for Multicore Architectures) is a dense linear algebra package at the forefront of multicore computing. PLASMA is designed to deliver the highest possible performance from a system with multiple sockets of multicore processors. PLASMA achieves this objective by combining state of the art solutions in parallel algorithms, scheduling, and software engineering. PLASMA currently offers a collection of routines for solving linear systems of equations and least square problems.},
	author = {Maksims Abalenkovs and Negin Bagherpour and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Samuel Relton and Jakub Sistek and David Stevens and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Mawussi Zounon}
}
@inproceedings {1307,
	title = {PMIx: Process Management for Exascale Environments},
	year = {2017},
	pages = {14:1{\textendash}14:10},
	publisher = {ACM},
	address = {New York, NY, USA},
	abstract = {High-Performance Computing (HPC) applications have historically executed in static resource allocations, using programming models that ran independently from the resident system management stack (SMS). Achieving exascale performance that is both cost-effective and fits within site-level environmental constraints will, however, require that the application and SMS collaboratively orchestrate the flow of work to optimize resource utilization and compensate for on-the-fly faults. The Process Management Interface - Exascale (PMIx) community is committed to establishing scalable workflow orchestration by defining an abstract set of interfaces by which not only applications and tools can interact with the resident SMS, but also the various SMS components can interact with each other. This paper presents a high-level overview of the goals and current state of the PMIx standard, and lays out a roadmap for future directions.
},
	isbn = {978-1-4503-4849-2},
	doi = {10.1145/3127024.3127027},
	url = {http://doi.acm.org/10.1145/3127024.3127027},
	author = {Castain, Ralph H. and David Solt and Joshua Hursey and Aurelien Bouteiller}
}
@techreport {1171,
	title = {POMPEI: Programming with OpenMP4 for Exascale Investigations},
	number = {ICL-UT-17-09},
	year = {2017},
	month = {2017-12},
	publisher = {University of Tennessee},
	abstract = {The objective of the Programming with OpenMP4 for Exascale Investigations (POMPEI) project is to explore new task-based programming techniques together with data structure centric programming for scientific applications to harness the potential of extreme-scale systems. Tasking is a well established by now approach on such systems as it has been used successfully to handle their large-scale parallelism and heterogeneity, which are leading challenges on the way to exascale computing. The approach is to harness the latest features of OpenMP4.5 and OpenACC2.5 to design abstractions shared among tasks and mapped efficiently to data-structure driven programming paradigms. This technical report describes the approach, along with its reference implementation and results for dense linear algebra algorithms.},
	author = {Jack Dongarra and Azzam Haidar and Oscar Hernandez and Stanimire Tomov and Manjunath Gorentla Venkata}
}
@conference {1134,
	title = {Power-aware Computing: Measurement, Control, and Performance Analysis for Intel Xeon Phi},
	booktitle = {2017 IEEE High Performance Extreme Computing Conference (HPEC{\textquoteright}17), Best Paper Finalist},
	year = {2017},
	month = {2017-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {The emergence of power efficiency as a primary constraint in processor and system designs poses new challenges concerning power and energy awareness for numerical libraries and scientific applications. Power consumption also plays a major role in the design of data centers in particular for peta- and exa- scale systems. Understanding and improving the energy efficiency of numerical simulation becomes very crucial.

We present a detailed study and investigation toward control- ling power usage and exploring how different power caps affect the performance of numerical algorithms with different computa- tional intensities, and determine the impact and correlation with performance of scientific applications.

Our analyses is performed using a set of representatives kernels, as well as many highly used scientific benchmarks. We quantify a number of power and performance measurements, and draw observations and conclusions that can be viewed as a roadmap toward achieving energy efficiency computing algorithms.},
	doi = {https://doi.org/10.1109/HPEC.2017.8091085},
	author = {Azzam Haidar and Heike Jagode and Asim YarKhan and Phil Vaccaro and Stanimire Tomov and Jack Dongarra}
}
@article {1338,
	title = {Power-Aware HPC on Intel Xeon Phi KNL Processors},
	year = {2017},
	month = {2017-06},
	publisher = {ISC High Performance (ISC17), Intel Booth Presentation},
	address = {Frankfurt, Germany},
	author = {Azzam Haidar and Heike Jagode and Asim YarKhan and Phil Vaccaro and Stanimire Tomov and Jack Dongarra}
}
@techreport {1080,
	title = {Roadmap for the Development of a Linear Algebra Library for Exascale Computing: SLATE: Software for Linear Algebra Targeting Exascale},
	number = {01, ICL-UT-17-02},
	year = {2017},
	month = {2017-06},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	type = {SLATE Working Notes},
	author = {Ahmad Abdelfattah and Hartwig Anzt and Aurelien Bouteiller and Anthony Danalis and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Stephen Wood and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan}
}
@techreport {1082,
	title = {Small Tensor Operations on Advanced Architectures for High-Order Applications},
	number = {UT-EECS-17-749},
	year = {2017},
	month = {2017-04},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@article {,
	title = {With Extreme Computing, the Rules Have Changed},
	journal = {Computing in Science \& Engineering},
	volume = {19},
	year = {2017},
	month = {2017-05},
	pages = {52-62},
	abstract = {On the eve of exascale computing, traditional wisdom no longer applies. High-performance computing is gone as we know it. This article discusses a range of new algorithmic techniques emerging in the context of exascale computing, many of which defy the common wisdom of high-performance computing and are considered unorthodox, but could turn out to be a necessity in near future.},
	doi = {https://doi.org/10.1109/MCSE.2017.48},
	author = {Jack Dongarra and Stanimire Tomov and Piotr Luszczek and Jakub Kurzak and Mark Gates and Ichitaro Yamazaki and Hartwig Anzt and Azzam Haidar and Ahmad Abdelfattah}
}
@article {1342,
	title = {Accelerating Tensor Contractions for High-Order FEM on CPUs, GPUs, and KNLs},
	year = {2016},
	month = {2016-09},
	publisher = {moky Mountains Computational Sciences and Engineering Conference (SMC16), Poster},
	address = {Gatlinburg, TN},
	author = {Azzam Haidar and Ahmad Abdelfattah and Veselin Dobrev and Ian Karlin and Tzanio Kolev and Stanimire Tomov and Jack Dongarra}
}
@article {994,
	title = {Accelerating the Conjugate Gradient Algorithm with GPU in CFD Simulations},
	journal = {VECPAR},
	year = {2016},
	abstract = {This paper illustrates how GPU computing can be used to accelerate computational fluid dynamics (CFD) simulations. For sparse linear systems arising from finite volume discretization, we evaluate and optimize the performance of Conjugate Gradient (CG) routines designed for manycore accelerators and compare against an industrial CPU-based implementation. We also investigate how the recent advances in preconditioning, such as iterative Incomplete Cholesky (IC, as symmetric case of ILU) preconditioning, match the requirements for solving real world problems.},
	url = {http://hgpu.org/?p=16264},
	author = {Hartwig Anzt and Marc Baboulin and Jack Dongarra and Yvan Fournier and Frank Hulsemann and Amal Khabou and Yushan Wang}
}
@article {924,
	title = {Assessing the Cost of Redistribution followed by a Computational Kernel: Complexity and Performance Results},
	journal = {Parallel Computing},
	volume = {52},
	year = {2016},
	month = {2016-02},
	pages = {22-41},
	abstract = {The classical redistribution problem aims at optimally scheduling communications when reshuffling from an initial data distribution to a target data distribution. This target data distribution is usually chosen to optimize some objective for the algorithmic kernel under study (good computational balance or low communication volume or cost), and therefore to provide high efficiency for that kernel. However, the choice of a distribution minimizing the target objective is not unique. This leads to generalizing the redistribution problem as follows: find a re-mapping of data items onto processors such that the data redistribution cost is minimal, and the operation remains as efficient. This paper studies the complexity of this generalized problem. We compute optimal solutions and evaluate, through simulations, their gain over classical redistribution. We also show the NP-hardness of the problem to find the optimal data partition and processor permutation (defined by new subsets) that minimize the cost of redistribution followed by a simple computational kernel. Finally, experimental validation of the new redistribution algorithms are conducted on a multicore cluster, for both a 1D-stencil kernel and a more compute-intensive dense linear algebra routine.},
	keywords = {Data partition, linear algebra, parsec, QR factorization, Redistribution, Stencil},
	doi = {doi:10.1016/j.parco.2015.09.005},
	author = {Julien Herrmann and George Bosilca and Thomas Herault and Loris Marchal and Yves Robert and Jack Dongarra}
}
@inproceedings {991,
	title = {Batched Generation of Incomplete Sparse Approximate Inverses on GPUs},
	year = {2016},
	month = {2016-11},
	pages = {49{\textendash}56},
	abstract = {Incomplete Sparse Approximate Inverses (ISAI) have recently been shown to be an attractive alternative to exact sparse triangular solves in the context of incomplete factorization preconditioning. In this paper we propose a batched GPU-kernel for the efficient generation of ISAI matrices. Utilizing only thread-local memory allows for computing the ISAI matrix with very small memory footprint. We demonstrate that this strategy is faster than the existing strategy for generating ISAI matrices, and use a large number of test matrices to assess the algorithm{\textquoteright}s efficiency in an iterative solver setting.},
	isbn = {978-1-5090-5222-6},
	doi = {10.1109/ScalA.2016.11},
	author = {Hartwig Anzt and Edmond Chow and Thomas Huckle and Jack Dongarra}
}
@article {1344,
	title = {Cholesky Factorization on Batches of Matrices with Fixed and Variable Sizes},
	year = {2016},
	month = {2016-04},
	publisher = {GPU Technology Conference (GTC16), Poster},
	address = {San Jose, CA},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@techreport {978,
	title = {Context Identifier Allocation in Open MPI},
	number = { ICL-UT-16-01},
	year = {2016},
	month = {2016-01},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	abstract = {The concept of communicators is a central notion in Message Passing Interface, allowing on one side the MPI implemen- tation to specialize it{\textquoteright}s matching and deliver messages in the right context, and on the other side the library developers to contextualize their message exchanges, and scope different algorithms to well-defined groups of processes. More pre- cisely, all communication objects in MPI are derived from a communicator at some point. All MPI functions allowing the creation of new communicators have a collective mean- ing, either over the group of processes from the parent com- municator or those from the target communicator. Thus, the perfromance of the communicator creation is tied to the col- lective communication performance, as well as the amount of data needed to be exchanged in order to consistently create this new communicator. We introduce several communica- tor creation algorithms, and present their implementation in the context of Open MPI. We explore the performance of these new algorithms and compare them with state-of-the- art algorithms available in other MPI implementations.},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@conference {940,
	title = {On the Development of Variable Size Batched Computation for Heterogeneous Parallel Architectures},
	booktitle = {The 17th IEEE International Workshop on Parallel and Distributed Scientific and Engineering Computing (PDSEC 2016), IPDPS 2016},
	year = {2016},
	month = {2016-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {<p>
Many scientific applications, ranging from national security to medical advances, require solving a number of relatively small-size independent problems. As the size of each individual problem does not provide sufficient parallelism for the underlying hardware, especially accelerators, these problems must be solved concurrently as a batch in order to saturate the hardware with enough work, hence the name batched computation. A possible simplification is to assume a uniform size for all problems. However, real applications do not necessarily satisfy such assumption. Consequently, an efficient solution for variable-size batched computations is required.
</p>
<p>
This paper proposes a foundation for high performance variable-size batched matrix computation based on Graphics Processing Units (GPUs). Being throughput-oriented processors, GPUs favor regular computation and less divergence among threads, in order to achieve high performance. Therefore, the development of high performance numerical software for this kind of problems is challenging. As a case study, we developed efficient batched Cholesky factorization algorithms for relatively small matrices of different sizes. However, most of the strategies and the software developed, and in particular a set of variable size batched BLAS kernels, can be used in many other dense matrix factorizations, large scale sparse direct multifrontal solvers, and applications. We propose new interfaces and mechanisms to handle the irregular computation pattern on the GPU. According to the authors{\textquoteright} knowledge, this is the first attempt to develop high performance software for this class of problems. Using a K40c GPU, our performance tests show speedups of up to 2:5 against two Sandy Bridge CPUs (8-core each) running Intel MKL library.
</p>},
	keywords = {batched computation, GPUs, variable small sizes},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {979,
	title = {Failure Detection and Propagation in HPC Systems},
	year = {2016},
	month = {2016-11},
	pages = {27:1-27:11},
	publisher = {IEEE Press},
	address = {Salt Lake City, Utah},
	keywords = {failure detection, fault-tolerance, MPI},
	isbn = {978-1-4673-8815-3},
	url = {http://dl.acm.org/citation.cfm?id=3014904.3014941},
	author = {George Bosilca and Aurelien Bouteiller and Amina Guermouche and Thomas Herault and Yves Robert and Pierre Sens and Jack Dongarra}
}
@conference {939,
	title = {Heterogeneous Streaming},
	booktitle = {The Sixth International Workshop on Accelerators and Hybrid Exascale Systems (AsHES), IPDPS 2016},
	year = {2016},
	month = {2016-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {This paper introduces a new heterogeneous streaming library called hetero Streams (hStreams). We show how a simple FIFO streaming model can be applied to heterogeneous systems that include manycore coprocessors and multicore CPUs. This model supports concurrency across nodes, among tasks within a node, and between data transfers and computation. We give examples for different approaches, show how the implementation can be layered, analyze overheads among layers, and apply those models to parallelize applications using simple, intuitive interfaces. We compare the features and versatility of hStreams, OpenMP, CUDA Streams1 and OmpSs. We show how the use of hStreams makes it easier for scientists to identify tasks and easily expose concurrency among them, and how it enables tuning experts and runtime systems to tailor execution for different heterogeneous targets. Practical application examples are taken from the field of numerical linear algebra, commercial structural simulation software, and a seismic processing application.},
	keywords = {plasma},
	author = {Chris J. Newburn and Gaurav Bansal and Michael Wood and Luis Crivelli and Judit Planas and Alejandro Duran and Paulo Souza and Leonardo Borges and Piotr Luszczek and Stanimire Tomov and Jack Dongarra and Hartwig Anzt and Mark Gates and Azzam Haidar and Yulu Jia and Khairul Kabir and Ichitaro Yamazaki and Jesus Labarta}
}
@article {984,
	title = {High Performance Conjugate Gradient Benchmark: A new Metric for Ranking High Performance Computing Systems},
	journal = {International Journal of High Performance Computing Applications},
	volume = {30},
	year = {2016},
	month = {2016-02},
	pages = {3 - 10},
	issn = {1094-3420},
	doi = {10.1177/1094342015593158},
	url = {http://hpc.sagepub.com/cgi/doi/10.1177/1094342015593158},
	author = {Jack Dongarra and Michael A. Heroux and Piotr Luszczek}
}
@conference {964,
	title = {High-performance Matrix-matrix Multiplications of Very Small Matrices},
	booktitle = {22nd International European Conference on Parallel and Distributed Computing (Euro-Par{\textquoteright}16)},
	year = {2016},
	month = {2016-08},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Grenoble, France},
	abstract = {The use of the general dense matrix-matrix multiplication (GEMM) is fundamental for obtaining high performance in many scientific computing applications. GEMMs for small matrices (of sizes less than 32) however, are not sufficiently optimized in existing libraries. In this paper we consider the case of many small GEMMs on either CPU or GPU architectures. This is a case that often occurs in applications like big data analytics, machine learning, high-order FEM, and others. The GEMMs are grouped together in a single batched routine. We present specialized for these cases algorithms and optimization techniques to obtain performance that is within 90\% of the optimal. We show that these results outperform currently available state-of-the-art implementations and vendor-tuned math libraries.},
	author = {Ian Masliah and Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jo{\"e}l Falcou and Jack Dongarra}
}
@techreport {929,
	title = {High-Performance Tensor Contractions for GPUs},
	number = {UT-EECS-16-738},
	year = {2016},
	month = {2016-01},
	publisher = {University of Tennessee},
	abstract = {We present a computational framework for high-performance tensor contractions on GPUs. High-performance is difficult to obtain using existing libraries, especially for many  independent contractions where each contraction is very small, e.g., sub-vector/warp in size. However, using our framework to batch contractions plus application-specifics, we demonstrate close to peak performance results. In particular, to accelerate large scale tensor-formulated high-order finite element method (FEM) simulations, which is the main focus and motivation for this work, we represent contractions as tensor index reordering plus matrix-matrix multiplications (GEMMs). This is a key factor to achieve algorithmically many-fold acceleration (vs. not using it) due to possible reuse of data loaded in fast memory. In addition to using this context knowledge, we design tensor data-structures, tensor algebra interfaces, and new tensor contraction algorithms and implementations to achieve 90+\% of a theoretically derived peak on GPUs. On a K40c GPU for contractions resulting in GEMMs on square matrices of size 8 for example, we are 2.8{\texttimes} faster than CUBLAS, and 8.5{\texttimes} faster than MKL on 16 cores of Intel Xeon ES-2670 (Sandy Bridge) 2.60GHz CPUs. Finally, we apply autotuning and code generation techniques to simplify tuning and provide an architecture-aware, user-friendly interface.},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Christopher Earl and Jo{\"e}l Falcou and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@conference {942,
	title = {High-Performance Tensor Contractions for GPUs},
	booktitle = {International Conference on Computational Science (ICCS{\textquoteright}16)},
	year = {2016},
	month = {2016-06},
	address = {San Diego, CA},
	abstract = {We present a computational framework for high-performance tensor contractions on GPUs. High-performance is difficult to obtain using existing libraries, especially for many independent contractions where each contraction is very small, e.g., sub-vector/warp in size. However, using our framework to batch contractions plus application-specifics, we demonstrate close to peak performance results. In particular, to accelerate large scale tensor-formulated high-order finite element method (FEM) simulations, which is the main focus and motivation for this work, we represent contractions as tensor index reordering plus matrix-matrix multiplications (GEMMs). This is a key factor to achieve algorithmically many-fold acceleration (vs. not using it) due to possible reuse of data loaded in fast memory. In addition to using this context knowledge, we design tensor data-structures, tensor algebra interfaces, and new tensor contraction algorithms and implementations to achieve 90+\% of a theoretically derived peak on GPUs. On a K40c GPU for contractions resulting in GEMMs on square matrices of size 8 for example, we are 2.8{\texttimes} faster than CUBLAS, and 8.5{\texttimes} faster than MKL on 16 cores of Intel Xeon E5-2670 (Sandy Bridge) 2.60GHz CPUs. Finally, we apply autotuning and code generation techniques to simplify tuning and provide an architecture-aware, user-friendly interface.},
	keywords = {Applications, Batched linear algebra, FEM, gpu, Tensor contractions, Tensor HPC},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Christopher Earl and Jo{\"e}l Falcou and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@article {1472,
	title = {Linear Algebra Software for Large-Scale Accelerated Multicore Computing},
	journal = {Acta Numerica},
	volume = {25},
	year = {2016},
	month = {2016-05},
	pages = {1-160},
	abstract = {Many crucial scientific computing applications, ranging from national security to medical advances, rely on high-performance linear algebra algorithms and technologies, underscoring their importance and broad impact. Here we present the state-of-the-art design and implementation practices for the acceleration of the predominant linear algebra algorithms on large-scale accelerated multicore systems. Examples are given with fundamental dense linear algebra algorithms {\textendash} from the LU, QR, Cholesky, and LDLT factorizations needed for solving linear systems of equations, to eigenvalue and singular value decomposition (SVD) problems. The implementations presented are readily available via the open-source PLASMA and MAGMA libraries, which represent the next generation modernization of the popular LAPACK library for accelerated multicore systems. To generate the extreme level of parallelism needed for the efficient use of these systems, algorithms of interest are redesigned and then split into well-chosen computational tasks. The task execution is scheduled over the computational components of a hybrid system of multicore CPUs with GPU accelerators and/or Xeon Phi coprocessors, using either static scheduling or light-weight runtime systems. The use of light-weight runtime systems keeps scheduling overheads low, similar to static scheduling, while enabling the expression of parallelism through sequential-like code. This simplifies the development effort and allows exploration of the unique strengths of the various hardware components. Finally, we emphasize the development of innovative linear algebra algorithms using three technologies {\textendash} mixed precision arithmetic, batched operations, and asynchronous iterations {\textendash} that are currently of high interest for accelerated multicore systems.},
	doi = {10.1017/S0962492916000015},
	author = {Ahmad Abdelfattah and Hartwig Anzt and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and undefined and Asim YarKhan}
}
@conference {963,
	title = {LU, QR, and Cholesky Factorizations: Programming Model, Performance Analysis and Optimization Techniques for the Intel Knights Landing Xeon Phi},
	booktitle = {IEEE High Performance Extreme Computing Conference (HPEC{\textquoteright}16)},
	year = {2016},
	month = {2016-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {A wide variety of heterogeneous compute resources, ranging from multicore CPUs to GPUs and coprocessors, are available to modern computers, making it challenging to design unified numerical libraries that efficiently and productively use all these varied resources. For example, in order to efficiently use Intel{\textquoteright}s Knights Langing (KNL) processor, the next-generation of Xeon Phi architectures, one must design and schedule an application in multiple degrees of parallelism and task grain sizes in order to obtain efficient performance. We propose a productive and portable programming model that allows us to write a serial-looking code, which, however, achieves parallelism and scalability by using a lightweight runtime environment to manage the resource-specific workload, and to control the dataflow and the parallel execution. This is done through multiple techniques ranging from multi-level data partitioning to adaptive task grain sizes, and dynamic task scheduling. In addition, our task abstractions enable unified algorithmic development across all the heterogeneous resources. Finally, we outline the strengths and the effectiveness of this approach {\textendash} especially in regards to hardware trends and ease of programming high-performance numerical software that current applications need {\textendash} in order to motivate current work and future directions for the next generation of parallel programming models for high-performance linear algebra libraries on heterogeneous systems.},
	author = {Azzam Haidar and Stanimire Tomov and Konstantin Arturov and Murat Guney and Shane Story and Jack Dongarra}
}
@techreport {,
	title = {MAGMA Batched: A Batched BLAS Approach for Small Matrix Factorizations and Applications on GPUs},
	number = {ICL-UT-16-02},
	year = {2016},
	month = {2016-08},
	publisher = {University of Tennessee},
	abstract = {A particularly challenging class of problems arising in many applications, called batched problems, involves linear algebra operations on many small-sized matrices. We proposed and designed batched BLAS (Basic Linear Algebra Subroutines), Level-2 GEMV and Level-3 GEMM, to solve them. We illustrate how batched GEMV and GEMM to be able to assist batched advance factorization (e.g. bi-diagonalization) and other BLAS routines (e.g. triangular solve) to achieve optimal performance on GPUs. Our solutions achieved up to 2.8-3{\texttimes} speedups compared to CUBLAS and MKL solutions, wherever possible. We illustrated the batched methodology on a real-world Hydrodynamic application by reformulating the tensor operations into batched BLAS GEMV and GEMM operations. A 2.5{\texttimes} speedup and a 1.4{\texttimes} greenup are obtained by changing 10\% of the code. We accelerated and scaled it on Titan supercomputer to 4096 nodes.},
	author = {Tingxing Dong and Azzam Haidar and Piotr Luszczek and Stanimire Tomov and Ahmad Abdelfattah and Jack Dongarra}
}
@article {,
	title = {A New Metric for Ranking High-Performance Computing Systems},
	journal = {National Science Review},
	volume = {3},
	year = {2016},
	month = {2016-01},
	pages = {30-35},
	doi = {https://doi.org/10.1093/nsr/nwv084},
	author = {Jack Dongarra and Michael A. Heroux and Piotr Luszczek}
}
@conference {968,
	title = {Performance Analysis and Acceleration of Explicit Integration for Large Kinetic Networks using Batched GPU Computations},
	booktitle = {2016 IEEE High Performance Extreme Computing Conference (HPEC {\textquoteleft}16)},
	year = {2016},
	month = {2016-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {We demonstrate the systematic implementation of recently-developed fast explicit kinetic integration algorithms that solve efficiently N coupled ordinary differential equations (subject to initial conditions) on modern GPUs. We take representative test cases (Type Ia supernova explosions) and demonstrate two or more orders of magnitude increase in efficiency for solving such systems (of realistic thermonuclear networks coupled to fluid dynamics). This implies that important coupled, multiphysics problems in various scientific and technical disciplines that were intractable, or could be simulated only with highly schematic kinetic networks, are now computationally feasible. As examples of such applications we present the computational techniques developed for our ongoing deployment of these new methods on modern GPU accelerators. We show that similarly to many other scientific applications, ranging from national security to medical advances, the computation can be split into many independent computational tasks, each of relatively small-size. As the size of each individual task does not provide sufficient parallelism for the underlying hardware, especially for accelerators, these tasks must be computed concurrently as a single routine, that we call batched routine, in order to saturate the hardware with enough work.},
	author = {Azzam Haidar and Benjamin Brock and Stanimire Tomov and Michael Guidry and Jay Jay Billings and Daniel Shyles and Jack Dongarra}
}
@mastersthesis {960,
	title = {Performance Analysis and Modeling of Task-Based Runtimes},
	volume = {PhD},
	year = {2016},
	month = {2016-05},
	school = {University of Tennessee},
	address = {Knoxville},
	abstract = {The shift toward multicore processors has transformed the software and hardware landscape in the last decade. As a result, software developers must adopt parallelism in order to efficiently make use of multicore CPUs. Task-based scheduling has emerged as one method to reduce the complexity of parallel computing. Although task-based scheduling has been around for many years, the inclusion of task dependencies in OpenMP 4.0 suggests the paradigm will be around for the foreseeable future. While task-based schedulers simplify the process of parallel software development, they can obfuscate the performance characteristics of the execution of an algorithm. Additionally, they can create a challenge for users to analyze the performance of their software and tune algorithmic parameters accordingly.
<br /><br />
We will present the basic principles of task-based runtimes as well as two new tools developed to assist engineers developing these runtimes and users employing them to parallelize their workloads. The first is a tool allowing users to simulate the execution of their algorithm. The second is an extension to the common execution trace which includes information about task dependencies.},
	author = {Blake Haugen}
}
@techreport {934,
	title = {Performance, Design, and Autotuning of Batched GEMM for GPUs},
	number = {UT-EECS-16-739},
	year = {2016},
	month = {2016-02},
	publisher = {University of Tennessee},
	abstract = {Abstract. The general matrix-matrix multiplication (GEMM) is the most important numerical kernel in dense linear algebra. It is the key
component for obtaining high performance in most LAPACK routines. As batched computations on relatively small problems continue to gain interest in many scientific applications, there becomes a need to have a high performance GEMM kernel for a batch of small matrices. Such kernel should be well designed and tuned to handle small sizes, and to maintain high performance for realistic test cases found in the higher level LAPACK routines, and scientific computing applications in general. This paper presents a high performance batched GEMM kernel on Graphics Processing Units (GPUs). We address batched problems with both xed and variable sizes, and show that specialized GEMM designs and a comprehensive autotuning process are needed to handle problems of small sizes. For most performance test reported in this paper, the proposed kernels outperform state-of-the-art approaches using a K40c GPU.},
	keywords = {Autotuning, Batched GEMM, GEMM, GPU computing, HPC},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {944,
	title = {Performance, Design, and Autotuning of Batched GEMM for GPUs},
	booktitle = {The International Supercomputing Conference (ISC High Performance 2016)},
	year = {2016},
	month = {2016-06},
	address = {Frankfurt, Germany},
	abstract = {The general matrix-matrix multiplication (GEMM) is the most important numerical kernel in dense linear algebra, and is the key component for obtaining high performance in most LAPACK routines. As batched computations on relatively small problems continue to gain interest in many scientific applications, a need arises for a high performance GEMM kernel for batches of small matrices. Such a kernel should be well designed and tuned to handle small sizes, and to maintain high performance for realistic test cases found in the higher level LAPACK routines, and scientific computing applications in general. This paper presents a high performance batched GEMM kernel on Graphics Processing Units (GPUs). We address batched problems with both fixed and variable sizes, and show that specialized GEMM designs and
a comprehensive autotuning process are needed to handle problems of small sizes. For most performance tests reported in this paper, the proposed kernels outperform state-of-the-art approaches using a K40c GPU.},
	keywords = {Autotuning, Batched GEMM, GEMM, GPU computing, HPC},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@inbook {997,
	title = {Performance, Design, and Autotuning of Batched GEMM for GPUs},
	booktitle = {High Performance Computing: 31st International Conference, ISC High Performance 2016, Frankfurt, Germany, June 19-23, 2016, Proceedings},
	number = {9697},
	year = {2016},
	pages = {21{\textendash}38},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {The general matrix-matrix multiplication (GEMM) is the most important numerical kernel in dense linear algebra, and is the key component for obtaining high performance in most LAPACK routines. As batched computations on relatively small problems continue to gain interest in many scientific applications, a need arises for a high performance GEMM kernel for batches of small matrices. Such a kernel should be well designed and tuned to handle small sizes, and to maintain high performance for realistic test cases found in the higher level LAPACK routines, and scientific computing applications in general.

This paper presents a high performance batched GEMM kernel on Graphics Processing Units (GPUs). We address batched problems with both fixed and variable sizes, and show that specialized GEMM designs and a comprehensive autotuning process are needed to handle problems of small sizes. For most performance tests reported in this paper, the proposed kernels outperform state-of-the-art approaches using a K40c GPU.},
	isbn = {978-3-319-41321-1},
	doi = {10.1007/978-3-319-41321-1_2},
	url = {http://dx.doi.org/10.1007/978-3-319-41321-1_2},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra},
	editor = {Julian M. Kunkel and Pavan Balaji and Jack Dongarra}
}
@conference {943,
	title = {Performance Tuning and Optimization Techniques of Fixed and Variable Size Batched Cholesky Factorization on GPUs},
	booktitle = {International Conference on Computational Science (ICCS{\textquoteright}16)},
	year = {2016},
	month = {2016-06},
	address = {San Diego, CA},
	abstract = {<p>Solving a large number of relatively small linear systems has recently drawn more attention in the HPC community, due to the importance of such computational workloads in many scientific applications, including sparse multifrontal solvers. Modern hardware accelerators and their architecture require a set of optimization techniques that are very different from the ones used in solving one relatively large matrix. In order to impose concurrency on such throughput-oriented architectures, a common practice is to batch the solution of these matrices as one task offloaded to the underlying hardware, rather than solving them individually.</p>

<p>This paper presents a high performance batched Cholesky factorization on large sets of relatively small matrices using Graphics Processing Units (GPUs), and addresses both fixed and variable size batched problems. We investigate various algorithm designs and optimization techniques, and show that it is essential to combine kernel design with performance tuning in order to achieve the best possible performance. We compare our approaches against state-of-the-art CPU solutions as well as GPU-based solutions using existing libraries, and show that, on a K40c GPU for example, our kernels are more than 2 faster.</p>},
	keywords = {batched computation, Cholesky Factorization, GPUs, Tuning},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1343,
	title = {A Standard for Batched BLAS Routines},
	year = {2016},
	month = {2016-04},
	publisher = {17th SIAM Conference on Parallel Processing for Scientific Computing (SIAM PP16)},
	address = {Paris, France},
	author = {Pedro Valero-Lara and Jack Dongarra and Azzam Haidar and Samuel D. Relton and Stanimire Tomov and Mawussi Zounon}
}
@conference {977,
	title = {Towards Achieving Performance Portability Using Directives for Accelerators},
	booktitle = {The International Conference for High Performance Computing, Networking, Storage and Analysis (SC{\textquoteright}16), Third Workshop on Accelerator Programming Using Directives (WACCPD)},
	year = {2016},
	month = {2016-11},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	organization = {Innovative Computing Laboratory, University of Tennessee},
	address = {Salt Lake City, Utah},
	abstract = {In this paper we explore the performance portability of directives provided by OpenMP 4 and OpenACC to program various types of node architectures with attached accelerators, both self-hosted multicore and offload multicore/GPU. Our goal is to examine how successful OpenACC and the newer of- fload features of OpenMP 4.5 are for moving codes between architectures, how much tuning might be required and what lessons we can learn from this experience. To do this, we use examples of algorithms with varying computational intensities for our evaluation, as both compute and data access efficiency are important considerations for overall application performance. We implement these kernels using various methods provided by newer OpenACC and OpenMP implementations, and we evaluate their performance on various platforms including both X86 64 with attached NVIDIA GPUs, self-hosted Intel Xeon Phi KNL, as well as an X86 64 host system with Intel Xeon Phi coprocessors. In this paper, we explain what factors affected the performance portability such as how to pick the right programming model, its programming style, its availability on different platforms, and how well compilers can optimize and target to multiple platforms.},
	author = {M. Graham Lopez and Larrea, V and Joubert, W and Hernandez, O and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {820,
	title = {Algorithm-based Fault Tolerance for Dense Matrix Factorizations, Multiple Failures, and Accuracy},
	journal = {ACM Transactions on Parallel Computing},
	volume = {1},
	number = {10},
	year = {2015},
	month = {2015-01},
	pages = {10:1-10:28},
	abstract = {Dense matrix factorizations, such as LU, Cholesky and QR, are widely used for scientific applications that
require solving systems of linear equations, eigenvalues and linear least squares problems. Such computations
are normally carried out on supercomputers, whose ever-growing scale induces a fast decline of the
Mean Time To Failure (MTTF). This paper proposes a new hybrid approach, based on Algorithm-Based Fault
Tolerance (ABFT), to help matrix factorizations algorithms survive fail-stop failures. We consider extreme
conditions, such as the absence of any reliable node and the possibility of losing both data and checksum from a single failure. We will present a generic solution for protecting the right factor, where the updates are applied, of all above mentioned factorizations. For the left factor, where the panel has been applied, we propose a scalable checkpointing algorithm. This algorithm features high degree of checkpointing parallelism and cooperatively utilizes the checksum storage leftover from the right factor protection. The fault-tolerant algorithms derived from this hybrid solution is applicable to a wide range of dense matrix factorizations, with minor modifications. Theoretical analysis shows that the fault tolerance overhead decreases inversely to the scaling in the number of computing units and the problem size. Experimental results of LU and QR factorization on the Kraken (Cray XT5) supercomputer validate the theoretical evaluation and confirm negligible overhead, with- and without-errors. Applicability to tolerate multiple failures and accuracy after multiple recovery is also considered.},
	keywords = {ABFT, algorithms, fault-tolerance, High Performance Computing, linear algebra},
	doi = {10.1145/2686892},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Peng Du and Jack Dongarra},
	editor = {Phillip B. Gibbons}
}
@conference {843,
	title = {Batched Matrix Computations on Hardware Accelerators},
	booktitle = {EuroMPI/Asia 2015 Workshop},
	year = {2015},
	month = {2015-09},
	address = {Bordeaux, France},
	abstract = {Scientific applications require solvers that work on many small size problems that are independent from each other. At the same time, the high-end hardware evolves rapidly and becomes ever more throughput-oriented and thus there is an increasing need for effective approach to develop energy efficient, high-performance codes for these small matrix problems that we call batched factorizations. The many applications that need this functionality could especially benefit from the use of GPUs, which currently are four to five times more energy efficient than multicore CPUs on important scientific workloads. This paper, consequently, describes the development of the most common, one-sided factorizations: Cholesky, LU, and QR for a set of small dense matrices. The algorithms we present together with their implementations are, by design, inherently parallel. In particular, our approach is based on representing the process as a sequence of batched BLAS routines that are executed entirely on a GPU. Importantly, this is unlike the LAPACK and the hybridMAGMAfactorization algorithms that work under drastically different assumptions of hardware design and efficiency of execution of the various computational kernels involved in the implementation. Thus, our approach is more efficient than what works for a combination of multicore CPUs and GPUs for the problems sizes of interest of the application use cases. The paradigm where upon a single chip (a GPU or a CPU) factorizes a single problem at a time is not at all efficient for in our applications{\textquoteright} context. We illustrate all these claims through a detailed performance analysis. With the help of profiling and tracing tools, we guide our development of batched factorizations to achieve up to two-fold speedup and three-fold better energy efficiency as compared against our highly optimized batched CPU implementations based on MKL library. The tested system featured two sockets of Intel Sandy Bridge CPUs and we compared to a batched LU factorizations featured in the CUBLAS library for GPUs, we achieve as high as 2.5x speedup on the NVIDIA K40 GPU.},
	author = {Azzam Haidar and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@article {858,
	title = {Batched matrix computations on hardware accelerators based on GPUs},
	journal = {International Journal of High Performance Computing Applications},
	year = {2015},
	month = {2015-02},
	abstract = {Scientific applications require solvers that work on many small size problems that are independent from each other. At the same time, the high-end hardware evolves rapidly and becomes ever more throughput-oriented and thus there is an increasing need for an effective approach to develop energy-efficient, high-performance codes for these small matrix problems that we call batched factorizations. The many applications that need this functionality could especially benefit from the use of GPUs, which currently are four to five times more energy efficient than multicore CPUs on important scientific workloads. This paper, consequently, describes the development of the most common, one-sided factorizations, Cholesky, LU, and QR, for a set of small dense matrices. The algorithms we present together with their implementations are, by design, inherently parallel. In particular, our approach is based on representing the process as a sequence of batched BLAS routines that are executed entirely on a GPU. Importantly, this is unlike the LAPACK and the hybrid MAGMA factorization algorithms that work under drastically different assumptions of hardware design and efficiency of execution of the various computational kernels involved in the implementation. Thus, our approach is more efficient than what works for a combination of multicore CPUs and GPUs for the problems sizes of interest of the application use cases. The paradigm where upon a single chip (a GPU or a CPU) factorizes a single problem at a time is not at all efficient in our applications{\textquoteright} context. We illustrate all of these claims through a detailed performance analysis. With the help of profiling and tracing tools, we guide our development of batched factorizations to achieve up to two-fold speedup and three-fold better energy efficiency as compared against our highly optimized batched CPU implementations based on MKL library. The tested system featured two sockets of Intel Sandy Bridge CPUs and we compared with a batched LU factorizations featured in the CUBLAS library for GPUs, we achieve as high as 2.5{\texttimes} speedup on the NVIDIA K40 GPU.},
	keywords = {batched factorization, hardware accelerators, numerical linear algebra, numerical software libraries, one-sided factorization algorithms},
	doi = {10.1177/1094342014567546},
	author = {Azzam Haidar and Tingxing Dong and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@conference {895,
	title = {Batched Matrix Computations on Hardware Accelerators Based on GPUs},
	booktitle = {2015 SIAM Conference on Applied Linear Algebra (SIAM LA)},
	year = {2015},
	month = {2015-10},
	publisher = {SIAM},
	organization = {SIAM},
	address = {Atlanta, GA},
	abstract = {We will present techniques for small matrix computations on GPUs and their use for energy efficient, high-performance solvers. Work on small problems delivers high performance through improved data reuse. Many numerical libraries and applications need this functionality further developed. We describe the main factorizations LU, QR, and Cholesky for a set of small dense matrices in parallel. We achieve significant acceleration and reduced energy consumption against other solutions. Our techniques are of interest to GPU application developers in general.},
	author = {Azzam Haidar and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@conference {928,
	title = {Cholesky Across Accelerators},
	booktitle = {17th IEEE International Conference on High Performance Computing and Communications (HPCC 2015)},
	year = {2015},
	month = {2015-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Elizabeth, NJ},
	author = {Asim YarKhan and Azzam Haidar and Chongxiao Cao and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@conference {896,
	title = {Comparing Hybrid CPU-GPU and Native GPU-only Acceleration for Linear Algebra},
	booktitle = {2015 SIAM Conference on Applied Linear Algebra},
	year = {2015},
	month = {2015-10},
	publisher = {SIAM},
	organization = {SIAM},
	address = {Atlanta, GA},
	abstract = {Accelerating dense linear algebra using GPUs admits two models: hybrid CPU-GPU and GPU-only. The hybrid model factors the panel on the CPU while updating the trailing matrix on the GPU, concentrating the GPU on high-performance matrix multiplies. The GPU-only model performs the entire computation on the GPU, avoiding costly data transfers to the CPU. We compare these two approaches for three QR-based algorithms: QR factorization, rank revealing QR, and reduction to Hessenberg.},
	author = {Mark Gates and Stanimire Tomov and Azzam Haidar}
}
@article {852,
	title = {Composing Resilience Techniques: ABFT, Periodic, and Incremental Checkpointing},
	journal = {International Journal of Networking and Computing},
	volume = {5},
	number = {1},
	year = {2015},
	month = {2015-01},
	pages = {2-15},
	abstract = {Algorithm Based Fault Tolerant (ABFT) approaches promise unparalleled scalability and performance in failure-prone environments. Thanks to recent advances in the understanding of the involved mechanisms, a growing number of important algorithms (including all widely used factorizations) have been proven ABFT-capable. In the context of larger applications, these algorithms provide a temporal section of the execution, where the data is protected by its own intrinsic properties, and can therefore be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only practical fault-tolerance approach for these applications is checkpoint/restart. In this paper we propose a model to investigate the efficiency of a composite protocol, that alternates between ABFT and checkpoint/restart for the effective protection of an iterative application composed of ABFT- aware and ABFT-unaware sections. We also consider an incremental checkpointing composite approach in which the algorithmic knowledge is leveraged by a novel optimal dynamic program- ming to compute checkpoint dates. We validate these models using a simulator. The model and simulator show that the composite approach drastically increases the performance delivered by an execution platform, especially at scale, by providing the means to increase the interval between checkpoints while simultaneously decreasing the volume of each checkpoint. },
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, model, performance evaluation, resilience},
	issn = {ISSN 2185-2839},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@conference {842,
	title = { A Data Flow Divide and Conquer Algorithm for Multicore Architecture},
	booktitle = {29th IEEE International Parallel \& Distributed Processing Symposium (IPDPS)},
	year = {2015},
	month = {2015-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Hyderabad, India},
	abstract = {Computing eigenpairs of a symmetric matrix is a problem arising in many industrial applications, including quantum physics and finite-elements computation for automobiles. A classical approach is to reduce the matrix to tridiagonal form before computing eigenpairs of the tridiagonal matrix. Then, a back-transformation allows one to obtain the final solution. Parallelism issues of the reduction stage have already been tackled in different shared-memory libraries. In this article, we focus on solving the tridiagonal eigenproblem, and we describe a novel implementation of the Divide and Conquer algorithm. The algorithm is expressed as a sequential task-flow, scheduled in an out-of-order fashion by a dynamic runtime which allows the programmer to play with tasks granularity. The resulting implementation is between two and five times faster than the equivalent routine from the Intel MKL library, and outperforms the best MRRR implementation for many matrices.},
	keywords = {Eigensolver, lapack, Multicore, plasma, task-based programming},
	author = {Azzam Haidar and Jakub Kurzak and Gregoire Pichon and Mathieu Faverge}
}
@article {1345,
	title = {On the Design, Autotuning, and Optimization of GPU Kernels for Kinetic Network Simulations Using Fast Explicit Integration and GPU Batched Computation},
	year = {2015},
	month = {2015-09},
	publisher = {Joint Institute for Computational Sciences Seminar Series, Presentation},
	address = {Oak Ridge, TN},
	author = {Michael Guidry and Azzam Haidar}
}
@conference {862,
	title = {On the Design, Development, and Analysis of Optimized Matrix-Vector Multiplication Routines for Coprocessors},
	booktitle = {ISC High Performance 2015},
	year = {2015},
	month = {2015-07},
	address = {Frankfurt, Germany},
	abstract = {The dramatic change in computer architecture due to the manycore paradigm shift, made the development of numerical routines that are optimal extremely challenging. In this work, we target the development of numerical algorithms and implementations for Xeon Phi coprocessor architecture designs. In particular, we examine and optimize the general and symmetric matrix-vector multiplication routines (gemv/symv), which are some of the most heavily used linear algebra kernels in many important engineering and physics applications. We describe a successful approach on how to address the challenges for this problem, starting from our algorithm design, performance analysis and programing model, to kernel optimization. Our goal, by targeting low-level, easy to understand fundamental kernels, is to develop new optimization strategies that can be effective elsewhere for the use on manycore coprocessors, and to show significant performance improvements compared to the existing state-of-the-art implementations. Therefore, in addition to the new optimization strategies, analysis, and optimal performance results, we finally present the significance of using these routines/strategies to accelerate higher-level numerical algorithms for the eigenvalue problem (EVP) and the singular value decomposition (SVD) that by themselves are foundational for many important applications.},
	author = {Khairul Kabir and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {840,
	title = {Design for a Soft Error Resilient Dynamic Task-based Runtime},
	booktitle = {29th IEEE International Parallel \& Distributed Processing Symposium (IPDPS)},
	year = {2015},
	month = {2015-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Hyderabad, India},
	abstract = {As the scale of modern computing systems grows, failures will happen more frequently. On the way to Exascale a generic, low-overhead, resilient extension becomes a desired aptitude of any programming paradigm. In this paper we explore three additions to a dynamic task-based runtime to build a generic framework providing soft error resilience to task-based programming paradigms. The first recovers the application by re-executing the minimum required sub-DAG, the second takes critical checkpoints of the data flowing between tasks to minimize the necessary re-execution, while the last one takes advantage of algorithmic properties to recover the data without re-execution. These mechanisms have been implemented in the PaRSEC task-based runtime framework. Experimental results validate our approach and quantify the overhead introduced by such mechanisms.},
	author = {Chongxiao Cao and George Bosilca and Thomas Herault and Jack Dongarra}
}
@conference {894,
	title = {Efficient Eigensolver Algorithms on Accelerator Based Architectures},
	booktitle = {2015 SIAM Conference on Applied Linear Algebra (SIAM LA)},
	year = {2015},
	month = {2015-10},
	publisher = {SIAM},
	organization = {SIAM},
	address = {Atlanta, GA},
	abstract = {The enormous gap between the high-performance capabilities of GPUs and the slow interconnect between them has made the development of numerical software that is scalable across multiple GPUs extremely challenging. We describe a successful methodology on how to address the challenges -starting from our algorithm design, kernel optimization and tuning, to our programming model- in the development of a scalable high-performance symmetric eigenvalue and singular value solver.},
	author = {Azzam Haidar and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@conference {886,
	title = {Efficient Implementation Of Quantum Materials Simulations On Distributed CPU-GPU Systems},
	booktitle = {The International Conference for High Performance Computing, Networking, Storage and Analysis (SC15)},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {We present a scalable implementation of the Linearized Augmented Plane Wave method for distributed memory systems, which relies on an efficient distributed, block-cyclic setup of the Hamiltonian and overlap matrices and allows us to turn around highly accurate 1000+ atom all-electron quantum materials simulations on clusters with a few hundred nodes. The implementation runs efficiently on standard multicore CPU nodes, as well as hybrid CPU-GPU nodes. The key for the latter is a novel algorithm to solve the generalized eigenvalue problem for dense, complex Hermitian matrices on distributed hybrid CPU-GPU systems. Performance tests for Li-intercalated CoO2 supercells containing 1501 atoms demonstrate that high-accuracy, transferable quantum simulations can now be used in throughput materials search problems. While our application can benefit and get scalable performance through CPU-only libraries like ScaLAPACK or ELPA2, our new hybrid solver enables the efficient use of GPUs and shows that a hybrid CPU-GPU architecture scales to a desired performance using substantially fewer cluster nodes, and notably, is considerably more energy efficient than the traditional multicore CPU only systems for such complex applications.},
	author = {Raffaele Solc{\`a} and Anton Kozhevnikov and Azzam Haidar and Stanimire Tomov and Thomas C. Schulthess and Jack Dongarra}
}
@article {873,
	title = {Experiences in autotuning matrix multiplication for energy minimization on GPUs},
	journal = {Concurrency in Computation: Practice and Experience},
	volume = {27},
	year = {2015},
	month = {2015-12},
	pages = {5096-5113},
	doi = {10.1002/cpe.3516},
	author = {Hartwig Anzt and Blake Haugen and Jakub Kurzak and Piotr Luszczek and Jack Dongarra}
}
@article {982,
	title = {Experiences in Autotuning Matrix Multiplication for Energy Minimization on GPUs},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {27},
	year = {2015},
	month = {12-Oct},
	pages = {5096 - 5113},
	abstract = {In this paper, we report extensive results and analysis of autotuning the computationally intensive graphics processing units kernel for dense matrix{\textendash}matrix multiplication in double precision. In contrast to traditional autotuning and/or optimization for runtime performance only, we also take the energy efficiency into account. For kernels achieving equal performance, we show significant differences in their energy balance. We also identify the memory throughput as the most influential metric that trades off performance and energy efficiency. As a result, the performance optimal case ends up not being the most efficient kernel in overall resource use.},
	keywords = {Autotuning, energy efficiency, hardware accelerators, matrix multiplication, power},
	doi = {10.1002/cpe.3516},
	url = {http://doi.wiley.com/10.1002/cpe.3516https://api.wiley.com/onlinelibrary/tdm/v1/articles/10.1002\%2Fcpe.3516},
	author = {Hartwig Anzt and Blake Haugen and Jakub Kurzak and Piotr Luszczek and Jack Dongarra}
}
@techreport {919,
	title = {Fault Tolerance Techniques for High-performance Computing},
	number = {UT-EECS-15-734},
	year = {2015},
	month = {2015-05},
	publisher = {University of Tennessee},
	abstract = {This report provides an introduction to resilience methods. The emphasis is on checkpointing, the de-facto standard technique for resilience in High Performance Computing. We present the main two protocols, namely coordinated checkpointing and hierarchical checkpointing. Then we introduce performance models and use them to assess the performance of theses protocols. We cover the Young/Daly formula for the optimal period and much more! Next we explain how the efficiency of checkpointing can be improved via fault prediction or replication. Then we move to application-specific methods, such as ABFT. We conclude the report by discussing techniques to cope with silent errors (or silent data corruption).},
	url = {http://www.netlib.org/lapack/lawnspdf/lawn289.pdf},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@conference {887,
	title = {Flexible Linear Algebra Development and Scheduling with Cholesky Factorization},
	booktitle = {17th IEEE International Conference on High Performance Computing and Communications},
	year = {2015},
	month = {2015-08},
	address = {Newark, NJ},
	abstract = {Modern high performance computing environments are composed of networks of compute nodes that often contain a variety of heterogeneous compute resources, such as multicore-CPUs, GPUs, and coprocessors. One challenge faced by domain scientists is how to efficiently use all these distributed, heterogeneous resources. In order to use the GPUs effectively, the workload parallelism needs to be much greater than the parallelism for a multicore-CPU. On the other hand, a Xeon Phi coprocessor will work most effectively with degree of parallelism between GPUs and multicore-CPUs. Additionally, effectively using distributed memory nodes brings out another level of complexity where the workload must be carefully partitioned over the nodes. In this work we are using a lightweight runtime environment to handle many of the complexities in such distributed, heterogeneous systems. The runtime environment uses task-superscalar concepts to enable the developer to write serial code while providing parallel execution. The task-programming model allows the developer to write resource-specialization code, so that each resource gets the appropriate sized workload-grain. Our task programming abstraction enables the developer to write a single algorithm that will execute efficiently across the distributed heterogeneous machine. We demonstrate the effectiveness of our approach with performance results for dense linear algebra applications, specifically the Cholesky factorization.},
	author = {Azzam Haidar and Asim YarKhan and Chongxiao Cao and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@conference {864,
	title = {Framework for Batched and GPU-resident Factorization Algorithms to Block Householder Transformations},
	booktitle = {ISC High Performance},
	year = {2015},
	month = {2015-07},
	publisher = {Springer},
	organization = {Springer},
	address = {Frankfurt, Germany},
	author = {Azzam Haidar and Tingxing Dong and Stanimire Tomov and Piotr Luszczek and Jack Dongarra}
}
@inproceedings {1310,
	title = {From MPI to OpenSHMEM: Porting LAMMPS},
	year = {2015},
	pages = {121{\textendash}137},
	publisher = {Springer International Publishing},
	address = {Annapolis, MD, USA},
	abstract = {This work details the opportunities and challenges of porting a Petascale, MPI-based application {\textendash}-LAMMPS{\textendash}- to OpenSHMEM. We investigate the major programming challenges stemming from the differences in communication semantics, address space organization, and synchronization operations between the two programming models. This work provides several approaches to solve those challenges for representative communication patterns in LAMMPS, e.g., by considering group synchronization, peer{\textquoteright}s buffer status tracking, and unpacked direct transfer of scattered data. The performance of LAMMPS is evaluated on the Titan HPC system at ORNL. The OpenSHMEM implementations are compared with MPI versions in terms of both strong and weak scaling. The results outline that OpenSHMEM provides a rich semantic to implement scalable scientific applications. In addition, the experiments demonstrate that OpenSHMEM can compete with, and often improve on, the optimized MPI implementation.},
	isbn = {978-3-319-26428-8},
	doi = {10.1007/978-3-319-26428-8_8},
	author = {Tang, Chunyan and Aurelien Bouteiller and Thomas Herault and Manjunath Gorentla Venkata and George Bosilca},
	editor = {Manjunath Gorentla Venkata and Shamis, Pavel and Imam, Neena and M. Graham Lopez}
}
@inbook {927,
	title = {High-Performance Computing},
	booktitle = {The Princeton Companion to Applied Mathematics},
	year = {2015},
	pages = {839-842},
	publisher = {Princeton University Press},
	organization = {Princeton University Press},
	address = {Princeton, New Jersey},
	isbn = {9781400874477},
	author = {Jack Dongarra and Nicholas J. Higham and Mark R. Dennis and Paul Glendinning and Paul A. Martin and Fadil Santosa and Jared Tanner}
}
@article {923,
	title = {High-Performance Conjugate-Gradient Benchmark: A New Metric for Ranking High-Performance Computing Systems},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2015},
	abstract = {We describe a new high-performance conjugate-gradient (HPCG) benchmark. HPCG is composed of computations and data-access patterns commonly found in scientific applications. HPCG strives for a better correlation to existing codes from the computational science domain and to be representative of their performance. HPCG is meant to help drive the computer system design and implementation in directions that will better impact future performance improvement.},
	keywords = {Additive Schwarz, HPC Benchmarking, Multigrid smoothing, Preconditioned Conjugate Gradient, Validation and Verification},
	doi = {10.1177/1094342015593158},
	author = {Jack Dongarra and Michael A. Heroux and Piotr Luszczek}
}
@article {829,
	title = {HPC Programming on Intel Many-Integrated-Core Hardware with MAGMA Port to Xeon Phi},
	journal = {Scientific Programming},
	volume = {23},
	year = {2015},
	month = {2015-01},
	abstract = {This paper presents the design and implementation of several fundamental dense linear algebra (DLA) algorithms for multicore with Intel Xeon Phi Coprocessors. In particular, we consider algorithms for solving linear systems. Further, we give an overview of the MAGMA MIC library, an open source, high performance library that incorporates the developments presented, and in general provides to heterogeneous architectures of multicore with coprocessors the DLA functionality of the popular LAPACK library. The LAPACK-compliance simplifies the use of the MAGMA MIC library in applications, while providing them with portably performant DLA. High performance is obtained through use of the high-performance BLAS, hardware-specific tuning, and a hybridization methodology where we split the algorithm into computational tasks of various granularities. Execution of those tasks is properly scheduled over the heterogeneous hardware components by minimizing data movements and mapping algorithmic requirements to the architectural strengths of the various heterogeneous hardware components. Our methodology and programming techniques are incorporated into the MAGMA MIC API, which abstracts the application developer from the specifics of the Xeon Phi architecture and is therefore applicable to algorithms beyond the scope of DLA.},
	keywords = {communication and computation overlap, dynamic runtime scheduling using dataflow dependences, hardware accelerators and coprocessors, Intel Xeon Phi processor, Many Integrated Cores, numerical linear algebra},
	issn = {1058-9244},
	doi = {10.3233/SPR-140404},
	author = {Azzam Haidar and Jack Dongarra and Khairul Kabir and Mark Gates and Piotr Luszczek and Stanimire Tomov and Yulu Jia}
}
@techreport {920,
	title = {HPCG Benchmark: a New Metric for Ranking High Performance Computing Systems},
	number = {ut-eecs-15-736},
	year = {2015},
	month = {2015-01},
	publisher = {University of Tennessee},
	abstract = {We describe a new high performance conjugate gradient (HPCG) benchmark. HPCG is composed of computations and data access patterns commonly found in scientific applications. HPCG strives for a better correlation to existing codes from the computational science domain and be representative of their performance. HPCG is meant to help drive the computer system design and implementation in directions that will better impact future performance improvement.},
	keywords = {Additive Schwarz, HPC Benchmarking, Multigrid smoothing, Preconditioned Conjugate Gradient, Validation and Verification},
	url = {http://www.eecs.utk.edu/resources/library/file/1047/ut-eecs-15-736.pdf},
	author = {Jack Dongarra and Michael A. Heroux and Piotr Luszczek}
}
@conference {888,
	title = {MAGMA Embedded: Towards a Dense Linear Algebra Library for Energy Efficient Extreme Computing},
	booktitle = { 2015 IEEE High Performance Extreme Computing Conference (HPEC {\textquoteright}15), (Best Paper Award)},
	year = {2015},
	month = {2015-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {Embedded computing, not only in large systems like drones and hybrid vehicles, but also in small portable devices like smart phones and watches, gets more extreme to meet ever increasing demands for extended and improved functionalities. This, combined with the typical constrains for low power consumption and small sizes, makes the design of numerical libraries for embedded systems challenging. In this paper, we present the design and implementation of embedded system aware algorithms, that target these challenges in the area of dense linear algebra. We consider the fundamental problems of solving linear systems of equations and least squares problems, using the LU, QR, and Cholesky factorizations, and illustrate our results, both in terms of performance and energy efficiency, on the Jetson TK1 development kit. We developed performance optimizations for both small and large problems. In contrast to the corresponding LAPACK algorithms, the new designs target the use of many-cores, readily available now even in mobile devices like the Jetson TK1, e.g., featuring 192 CUDA cores. The implementations presented will form the core of a MAGMA Embedded library, to be released as part of the MAGMA libraries.
},
	author = {Azzam Haidar and Stanimire Tomov and Piotr Luszczek and Jack Dongarra}
}
@article {1347,
	title = {MAGMA MIC: Optimizing Linear Algebra for Intel Xeon Phi},
	year = {2015},
	month = {2015-06},
	publisher = {ISC High Performance (ISC15), Intel Booth Presentation},
	address = {Frankfurt, Germany},
	author = {Hartwig Anzt and Jack Dongarra and Mark Gates and Azzam Haidar and Khairul Kabir and Piotr Luszczek and Stanimire Tomov and Ichitaro Yamazaki}
}
@article {917,
	title = {Mixing LU-QR Factorization Algorithms to Design High-Performance Dense Linear Algebra Solvers},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {85},
	year = {2015},
	month = {2015-11},
	pages = {32-46},
	abstract = {This paper introduces hybrid LU{\textendash}QR algorithms for solving dense linear systems of the form Ax=b. Throughout a matrix factorization, these algorithms dynamically alternate LU with local pivoting and QR elimination steps based upon some robustness criterion. LU elimination steps can be very efficiently parallelized, and are twice as cheap in terms of floating-point operations, as QR steps. However, LU steps are not necessarily stable, while QR steps are always stable. The hybrid algorithms execute a QR step when a robustness criterion detects some risk for instability, and they execute an LU step otherwise. The choice between LU and QR steps must have a small computational overhead and must provide a satisfactory level of stability with as few QR steps as possible. In this paper, we introduce several robustness criteria and we establish upper bounds on the growth factor of the norm of the updated matrix incurred by each of these criteria. In addition, we describe the implementation of the hybrid algorithms through an extension of the PaRSEC software to allow for dynamic choices during execution. Finally, we analyze both stability and performance results compared to state-of-the-art linear solvers on parallel distributed multicore platforms. A comprehensive set of experiments shows that hybrid LU{\textendash}QR algorithms provide a continuous range of trade-offs between stability and performances.},
	keywords = {lu factorization, Numerical algorithms, QR factorization, Stability; Performance},
	doi = {doi:10.1016/j.jpdc.2015.06.007},
	author = {Mathieu Faverge and Julien Herrmann and Julien Langou and Bradley Lowery and Yves Robert and Jack Dongarra}
}
@conference {916,
	title = {Optimization for Performance and Energy for Batched Matrix Computations on GPUs},
	booktitle = {8th Workshop on General Purpose Processing Using GPUs (GPGPU 8)},
	year = {2015},
	month = {2015-02},
	publisher = {ACM},
	organization = {ACM},
	address = {San Francisco, CA},
	abstract = {As modern hardware keeps evolving, an increasingly effective approach to develop energy efficient and high-performance solvers is to design them to work on many small size independent problems. Many applications already need this functionality, especially for GPUs, which are known to be currently about four to five times more energy efficient than multicore CPUs. We describe the development of the main one-sided factorizations that work for a set of small dense matrices in parallel, and we illustrate our techniques on the LU and Cholesky factorizations. We refer to this mode of operation as a batched factorization. Our approach is based on representing the  algorithms as a sequence of batched BLAS routines for GPU-only execution. The goal of avoiding multicore CPU use, e.g., as in the hybrid CPU-GPU algorithms, is to exclusively benefit from the GPU{\textquoteright}s significantly higher energy efficiency, as well as from the removal of the  costly CPU-to-GPU communications. Furthermore, we do not use a single symmetric multiprocessor (on the GPU) to factorize a single problem at a time. We illustrate how our performance analysis and the use of profiling and tracing tools guided the development and optimization of batched factorizations to achieve up to 2-fold speedup and 3-fold better energy efficiency compared to our highly optimized batched CPU implementations based on the MKL library (when using two sockets of Intel Sandy Bridge CPUs). Compared to a batched LU factorization featured in the CUBLAS library for GPUs, we achieved up to 2.5 speedup on the K40 GPU.},
	keywords = {batched factorization, hardware accelerators, numerical linear algebra, numerical software libraries, one-sided factorization algorithms},
	doi = {10.1145/2716282.2716288},
	author = {Azzam Haidar and Tingxing Dong and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@article {936,
	title = {Parallel Programming Models for Dense Linear Algebra on Heterogeneous Systems},
	journal = {Supercomputing Frontiers and Innovations},
	volume = {2},
	number = {4},
	year = {2015},
	month = {2015-10},
	abstract = {We present a review of the current best practices in parallel programming models for dense linear algebra (DLA) on heterogeneous architectures. We consider multicore CPUs, stand alone manycore coprocessors, GPUs, and combinations of these. Of interest is the evolution of the programming models for DLA libraries {\textendash} in particular, the evolution from the popular LAPACK and ScaLAPACK libraries to their modernized counterparts PLASMA (for multicore CPUs) and MAGMA (for heterogeneous architectures), as well as other programming models and libraries. Besides providing insights into the programming techniques of the libraries considered, we outline our view of the current strengths and weaknesses of their programming models {\textendash} especially in regards to hardware trends and ease of programming high-performance numerical software that current applications need {\textendash} in order to motivate work and future directions for the next generation of parallel programming models for high-performance linear algebra libraries on heterogeneous systems.},
	keywords = {dense linear algebra, gpu, HPC, Multicore, plasma, Programming models, runtime},
	doi = {10.14529/jsfi1504},
	author = {Maksims Abalenkovs and Ahmad Abdelfattah and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Ichitaro Yamazaki and Asim YarKhan}
}
@conference {860,
	title = {Performance Analysis and Design of a Hessenberg Reduction using Stabilized Blocked Elementary Transformations for New Architectures},
	booktitle = {The Spring Simulation Multi-Conference 2015 (SpringSim{\textquoteright}15), Best Paper Award},
	year = {2015},
	month = {2015-04},
	address = {Alexandria, VA},
	abstract = {The solution of nonsymmetric eigenvalue problems, Ax = λx, can be accelerated substantially by first reducing A to an upper Hessenberg matrix H that has the same eigenvalues as A. This can be done using Householder orthogonal transformations, which is a well established standard, or stabilized elementary transformations. The latter approach, although having half the flops of the former, has been used less in practice, e.g., on computer architectures with well developed hierarchical memories, because of its memory-bound operations and the complexity in stabilizing it. In this paper we revisit the stabilized elementary transformations approach in the context of new architectures {\textendash} both multicore CPUs and Xeon Phi coprocessors. We derive for a first time a blocking version of the algorithm. The blocked version reduces the memory-bound operations and we analyze its performance. A performance model is developed that shows the limitations of both approaches. The competitiveness of using stabilized elementary transformations has been quantified, highlighting that it can be 20 to 30\% faster on current high-end multicore CPUs and Xeon Phi coprocessors.},
	keywords = {Eigenvalues problem, Hessenberg reduction, Multi/Many-core, Stabilized Elementary Transformations},
	author = {Khairul Kabir and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {861,
	title = {Performance Analysis and Optimization of Two-Sided Factorization Algorithms for Heterogeneous Platform},
	booktitle = {International Conference on Computational Science (ICCS 2015)},
	year = {2015},
	month = {2015-06},
	address = {Reykjav{\'\i}k, Iceland},
	author = {Khairul Kabir and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@techreport {872,
	title = {Practical Scalable Consensus for Pseudo-Synchronous Distributed Systems: Formal Proof},
	number = {ICL-UT-15-01},
	year = {2015},
	month = {2015-04},
	author = {Thomas Herault and Aurelien Bouteiller and George Bosilca and Marc Gamell and Keita Teranishi and Manish Parashar and Jack Dongarra}
}
@conference {889,
	title = {Practical Scalable Consensus for Pseudo-Synchronous Distributed Systems},
	booktitle = {The International Conference for High Performance Computing, Networking, Storage and Analysis (SC15)},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {The ability to consistently handle faults in a distributed environment requires, among a small set of basic routines, an agreement algorithm allowing surviving entities to reach a consensual decision between a bounded set of volatile resources. This paper presents an algorithm that implements an Early Returning Agreement (ERA) in pseudo-synchronous systems, which optimistically allows a process to resume its activity while guaranteeing strong progress. We prove the correctness of our ERA algorithm, and expose its logarithmic behavior, which is an extremely desirable property for any algorithm which targets future exascale platforms. We detail a practical implementation of this consensus algorithm in the context of an MPI library, and evaluate both its efficiency and scalability through a set of benchmarks and two fault tolerant scientific applications.},
	author = {Thomas Herault and Aurelien Bouteiller and George Bosilca and Marc Gamell and Keita Teranishi and Manish Parashar and Jack Dongarra}
}
@article {1346,
	title = {Towards a High-Performance Tensor Algebra Package for Accelerators},
	year = {2015},
	month = {2015-09},
	publisher = {moky Mountains Computational Sciences and Engineering Conference (SMC15)},
	address = {Gatlinburg, TN},
	author = {Marc Baboulin and Veselin Dobrev and Jack Dongarra and Christopher Earl and Jo{\"e}l Falcou and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@conference {844,
	title = {Towards Batched Linear Solvers on Accelerated Hardware Platforms},
	booktitle = {8th Workshop on General Purpose Processing Using GPUs (GPGPU 8) co-located with PPOPP 2015},
	year = {2015},
	month = {2015-02},
	publisher = {ACM},
	organization = {ACM},
	address = {San Francisco, CA},
	abstract = {As hardware evolves, an increasingly effective approach to develop energy efficient, high-performance solvers, is to design them to work on many small and independent problems. Indeed, many applications already need this functionality, especially for GPUs, which are known to be currently about four to five times more energy efficient than multicore CPUs for every floating-point operation. In this paper, we describe the development of the main one-sided factorizations: LU, QR, and Cholesky; that are needed for a set of small dense matrices to work in parallel. We refer to such algorithms as batched factorizations. Our approach is based on representing the algorithms as a sequence of batched BLAS routines for GPU-contained execution. Note that this is similar in functionality to the LAPACK and the hybrid MAGMA algorithms for large-matrix factorizations. But it is different from a straightforward approach, whereby each of GPU{\textquoteright}s symmetric multiprocessors factorizes a single problem at a time.We illustrate how our performance analysis together with the profiling and tracing tools guided the development of batched factorizations to achieve up to 2-fold speedup and 3-fold better energy efficiency compared to our highly optimized batched CPU implementations based on the MKL library on a two-sockets, Intel Sandy Bridge server. Compared to a batched LU factorization featured in the NVIDIA{\textquoteright}s CUBLAS library for GPUs, we achieves up to 2.5-fold speedup on the K40 GPU.},
	keywords = {batched factorization, hardware accelerators, numerical linear algebra, numerical software libraries, one-sided factorization algorithms},
	author = {Azzam Haidar and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {1309,
	title = {UCX: An Open Source Framework for HPC Network APIs and Beyond},
	year = {2015},
	month = {Aug},
	pages = {40-43},
	publisher = {IEEE},
	address = {Santa Clara, CA, USA},
	abstract = {This paper presents Unified Communication X (UCX), a set of network APIs and their implementations for high throughput computing. UCX comes from the combined effort of national laboratories, industry, and academia to design and implement a high-performing and highly-scalable network stack for next generation applications and systems. UCX design provides the ability to tailor its APIs and network functionality to suit a wide variety of application domains and hardware. We envision these APIs to satisfy the networking needs of many programming models such as Message Passing Interface (MPI), OpenSHMEM, Partitioned Global Address Space (PGAS) languages, task-based paradigms and I/O bound applications. To evaluate the design we implement the APIs and protocols, and measure the performance of overhead-critical network primitives fundamental for implementing many parallel programming models and system libraries. Our results show that the latency, bandwidth, and message rate achieved by the portable UCX prototype is very close to that of the underlying driver. With UCX, we achieved a message exchange latency of 0.89 us, a bandwidth of 6138.5 MB/s, and a message rate of 14 million messages per second. As far as we know, this is the highest bandwidth and message rate achieved by any network stack (publicly known) on this hardware.},
	keywords = {application program interfaces, Bandwidth, Electronics packaging, Hardware, high throughput computing, highly-scalable network stack, HPC, HPC network APIs, I/O bound applications, Infiniband, input-output programs, Libraries, Memory management, message passing, message passing interface, Middleware, MPI, open source framework, OpenSHMEM, parallel programming, parallel programming models, partitioned global address space languages, PGAS, PGAS languages, Programming, protocols, public domain software, RDMA, system libraries, task-based paradigms, UCX, Unified Communication X},
	isbn = {978-1-4673-9160-3},
	doi = {10.1109/HOTI.2015.13},
	author = {P. Shamis and Manjunath Gorentla Venkata and M. Graham Lopez and M. B. Baker and O. Hernandez and Y. Itigin and M. Dubman and G. Shainer and R. L. Graham and L. Liss and Y. Shahar and S. Potluri and D. Rossetti and D. Becker and D. Poole and C. Lamb and S. Kumar and C. Stunkel and George Bosilca and Aurelien Bouteiller}
}
@conference {892,
	title = {Visualizing Execution Traces with Task Dependencies},
	booktitle = {2nd Workshop on Visual Performance Analysis (VPA {\textquoteright}15)},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {Task-based scheduling has emerged as one method to reduce the complexity of parallel computing. When using task-based schedulers, developers must frame their computation as a series of tasks with various data dependencies. The scheduler can take these tasks, along with their input and output dependencies, and schedule the task in parallel across a node or cluster. While these schedulers simplify the process of parallel software development, they can obfuscate the performance characteristics of the execution of an algorithm. The execution trace has been used for many years to give developers a visual representation of how their computations are performed. These methods can be employed to visualize when and where each of the tasks in a task-based algorithm is scheduled. In addition, the task dependencies can be used to create a directed acyclic graph (DAG) that can also be visualized to demonstrate the dependencies of the various tasks that make up a workload. The work presented here aims to combine these two data sets and extend execution trace visualization to better suit task-based workloads. This paper presents a brief description of task-based schedulers and the performance data they produce. It will then describe an interactive extension to the current trace visualization methods that combines the trace and DAG data sets. This new tool allows users to gain a greater understanding of how their tasks are scheduled. It also provides a simplified way for developers to evaluate and debug the performance of their scheduler. },
	author = {Blake Haugen and Stephen Richmond and Jakub Kurzak and Chad A. Steed and Jack Dongarra}
}
@inproceedings {959,
	title = {Weighted Dynamic Scheduling with Many Parallelism Grains for Offloading of Numerical Workloads to Multiple Varied Accelerators},
	volume = {No. 5},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	address = {Austin, TX},
	abstract = {A wide variety of heterogeneous compute resources are available to modern computers, including multiple sockets containing multicore CPUs, one-or-more GPUs of varying power, and coprocessors such as the Intel Xeon Phi. The challenge faced by domain scientists is how to efficiently and productively use these varied resources. For example, in order to use GPUs effectively, the workload must have a greater degree of parallelism than a workload designed for a multicore-CPU. The domain scientist would have to design and schedule an application in multiple degrees of parallelism and task grain sizes in order to obtain efficient performance from the resources. We propose a productive programming model starting from serial code, which achieves parallelism and scalability by using a task-superscalar runtime environment to adapt the computation to the available resources. The adaptation is done at multiple points, including multi-level data partitioning, adaptive task grain sizes, and dynamic task scheduling. The effectiveness of this approach for utilizing multi-way heterogeneous hardware resources is demonstrated by implementing dense linear algebra applications.},
	keywords = {dataflow scheduling, hardware accelerators, multi-grain parallelism},
	author = {Azzam Haidar and Yulu Jia and Piotr Luszczek and Stanimire Tomov and Asim YarKhan and Jack Dongarra}
}
@conference {766,
	title = {Accelerating Eigenvector Computation in the Nonsymmetric Eigenvalue Problem},
	booktitle = {VECPAR 2014},
	year = {2014},
	month = {2014-06},
	address = {Eugene, OR},
	abstract = {In the nonsymmetric eigenvalue problem, work has focused on the Hessenberg reduction and QR iteration, using efficient algorithms and fast, Level 3 BLAS routines. Comparatively, computation of eigenvectors performs poorly, limited to slow, Level 2 BLAS performance with little speedup on multi-core systems. It has thus become a dominant cost in the eigenvalue problem. To address this, we present improvements for the eigenvector computation to use Level 3 BLAS where applicable and parallelize the remaining triangular solves, achieving good parallel scaling and accelerating the overall eigenvalue problem more than three-fold.},
	author = {Mark Gates and Azzam Haidar and Jack Dongarra}
}
@inbook {780,
	title = {Accelerating Numerical Dense Linear Algebra Calculations with GPUs},
	booktitle = {Numerical Computations with GPUs},
	year = {2014},
	pages = {3-28},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	chapter = {1},
	isbn = {978-3-319-06547-2},
	doi = {10.1007/978-3-319-06548-9_1},
	author = {Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Ichitaro Yamazaki}
}
@conference {708,
	title = {Assessing the Impact of ABFT and Checkpoint Composite Strategies},
	booktitle = {16th Workshop on Advances in Parallel and Distributed Computational Models, IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Algorithm-specific fault tolerant approaches promise unparalleled scalability and performance in failure-prone environments. With the advances in the theoretical and practical understanding of algorithmic traits enabling such approaches, a growing number of frequently used algorithms (including all widely used factorization kernels) have been proven capable of such properties. These algorithms provide a temporal section of the execution when the data is protected by it{\textquoteright}s own intrinsic properties, and can be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only fault-tolerance approach that is currently used for these applications is checkpoint/restart. In this paper we propose a model and a simulator to investigate the behavior of a composite protocol, that alternates between ABFT and checkpoint/restart protection for effective protection of each phase of an iterative application composed of ABFT-aware and ABFTunaware sections. We highlight this approach drastically increases the performance delivered by the system, especially at scale, by providing means to rarefy the checkpoints while simultaneously decreasing the volume of data needed to be checkpointed.},
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, resilience},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@techreport {822,
	title = {Design for a Soft Error Resilient Dynamic Task-based Runtime},
	number = {ICL-UT-14-04},
	year = {2014},
	month = {2014-11},
	publisher = {University of Tennessee},
	abstract = {Abstract{\textemdash}As the scale of modern computing systems grows, failures will happen more frequently. On the way to Exascale a generic, low-overhead, resilient extension becomes a desired aptitude of any programming paradigm. In this paper we explore three additions to a dynamic task-based runtime to build a generic framework providing soft error resilience to task-based programming paradigms. The first recovers the application by re-executing the minimum required sub-DAG, the second takes critical checkpoints of the data flowing between tasks to minimize the necessary re-execution, while the last one takes advantage of algorithmic properties to recover the data without re-execution. These mechanisms have been implemented in the PaRSEC task-based runtime framework. Experimental results validate our approach and quantify the overhead introduced by such mechanisms.},
	author = {Chongxiao Cao and Thomas Herault and George Bosilca and Jack Dongarra}
}
@conference {813,
	title = {Designing LU-QR Hybrid Solvers for Performance and Stability},
	booktitle = {IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {This paper introduces hybrid LU-QR algorithms for solving dense linear systems of the form Ax = b. Throughout a matrix factorization, these algorithms dynamically alternate LU with local pivoting and QR elimination steps, based upon some robustness criterion. LU elimination steps can be very efficiently parallelized, and are twice as cheap in terms of operations, as QR steps. However, LU steps are not necessarily stable, while QR steps are always stable. The hybrid algorithms execute a QR step when a robustness criterion detects some risk for instability, and they execute an LU step otherwise. Ideally, the choice between LU and QR steps must have a small computational overhead and must provide a satisfactory level of stability with as few QR steps as possible. In this paper, we introduce several robustness criteria and we establish upper bounds on the growth factor of the norm of the updated matrix incurred by each of these criteria. In addition, we describe the implementation of the hybrid algorithms through an extension of the Parsec software to allow for dynamic choices during execution. Finally, we analyze both stability and performance results compared to state-of-the-art linear solvers on parallel distributed multicore platforms.},
	keywords = {plasma},
	isbn = {978-1-4799-3800-1},
	doi = {10.1109/IPDPS.2014.108},
	author = {Mathieu Faverge and Julien Herrmann and Julien Langou and Bradley Lowery and Yves Robert and Jack Dongarra}
}
@conference {816,
	title = {Domain Decomposition Preconditioners for Communication-Avoiding Krylov Methods on a Hybrid CPU/GPU Cluster},
	booktitle = {The International Conference for High Performance Computing, Networking, Storage and Analysis (SC 14)},
	year = {2014},
	month = {2014-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {New Orleans, LA},
	author = {Ichitaro Yamazaki and Sivasankaran Rajamanickam and Eric G. Boman and Mark Hoemmen and Michael A. Heroux and Stanimire Tomov}
}
@conference {715,
	title = {A Fast Batched Cholesky Factorization on a GPU},
	booktitle = {International Conference on Parallel Processing (ICPP-2014)},
	year = {2014},
	month = {2014-09},
	address = {Minneapolis, MN},
	abstract = {Currently, state of the art libraries, like MAGMA, focus on very large linear algebra problems, while solving many small independent problems, which is usually referred to as batched problems, is not given adequate attention. In this paper, we proposed a batched Cholesky factorization on a GPU. Three algorithms {\textendash} nonblocked, blocked, and recursive blocked {\textendash} were examined. The left-looking version of the Cholesky factorization is used to factorize the panel, and the right-looking Cholesky version is used to update the trailing matrix in the recursive blocked algorithm. Our batched Cholesky achieves up to 1:8 speedup compared to the optimized parallel implementation in the MKL library on two sockets of Intel Sandy Bridge CPUs. Further, we use the new routines to develop a single Cholesky factorization solver which targets large matrix sizes. Our approach differs from MAGMA by having an entirely GPU implementation where both the panel factorization and the trailing matrix updates are on the GPU. Such an implementation does not depend on the speed of the CPU. Compared to the MAGMA library, our full GPU solution achieves 85\% of the hybrid MAGMA performance which uses 16 Sandy Bridge cores, in addition to a K40 Nvidia GPU. Moreover, we achieve 80\% of the practical dgemm peak of the machine, while MAGMA achieves only 75\%, and finally, in terms of energy consumption, we outperform MAGMA by 1.5 in performance-per-watt for large matrices.},
	author = {Tingxing Dong and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {765,
	title = {Heterogeneous Acceleration for Linear Algebra in Mulit-Coprocessor Environments},
	booktitle = {VECPAR 2014},
	year = {2014},
	month = {2014-06},
	address = {Eugene, OR},
	abstract = {We present an efficient and scalable programming model for the development of linear algebra in heterogeneous multi-coprocessor environments. The model incorporates some of the current best design and implementation practices for the heterogeneous acceleration of dense linear algebra (DLA). Examples are given as the basis for solving linear systems{\textquoteright} algorithms {\textendash} the LU, QR, and Cholesky factorizations. To generate the extreme level of parallelism needed for the efficient use of coprocessors, algorithms of interest are redesigned and then split into well-chosen computational tasks. The tasks execution is scheduled over the computational components of a hybrid system of multi-core CPUs and coprocessors using a light-weight runtime system. The use of light-weight runtime systems keeps scheduling overhead low, while enabling the expression of parallelism through otherwise sequential code. This simplifies the development efforts and allows the exploration of the unique strengths of the various hardware components.},
	keywords = {Computer science, factorization, Heterogeneous systems, Intel Xeon Phi, linear algebra},
	author = {Azzam Haidar and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@conference {807,
	title = {Improving the performance of CA-GMRES on multicores with multiple GPUs},
	booktitle = {IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Abstract{\textemdash}The Generalized Minimum Residual (GMRES) method is one of the most widely-used iterative methods for solving nonsymmetric linear systems of equations. In recent years, techniques to avoid communication in GMRES have gained attention because in comparison to floating-point operations, communication is becoming increasingly expensive on modern computers. Since graphics processing units (GPUs) are now becoming crucial component in computing, we investigate the effectiveness of these techniques on multicore CPUs with multiple GPUs. While we present the detailed performance studies of a matrix powers kernel on multiple GPUs, we particularly focus on orthogonalization strategies that have a great impact on both the numerical stability and performance of GMRES, especially as the matrix becomes sparser or ill-conditioned. We present the experimental results on two eight-core Intel Sandy Bridge CPUs with three NDIVIA Fermi GPUs and demonstrate that significant speedups can be obtained by avoiding communication, either on a GPU or between the GPUs. As part of our study, we investigate several optimization techniques for the GPU kernels that can also be used in other iterative solvers besides GMRES. Hence, our studies not only emphasize the importance of avoiding communication on GPUs, but they also provide insight about the effects of these optimization techniques on the performance of the sparse solvers, and may have greater impact beyond GMRES.},
	author = {Ichitaro Yamazaki and Hartwig Anzt and Stanimire Tomov and Mark Hoemmen and Jack Dongarra}
}
@conference {859,
	title = {LU Factorization of Small Matrices: Accelerating Batched DGETRF on the GPU},
	booktitle = {16th IEEE International Conference on High Performance Computing and Communications (HPCC)},
	year = {2014},
	month = {2014-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Paris, France},
	abstract = {Gaussian Elimination is commonly used to solve dense linear systems in scientific models. In a large number of applications, a need arises to solve many small size problems, instead of few large linear systems. The size of each of these small linear systems depends, for example, on the number of the ordinary differential equations (ODEs) used in the model, and can be on the order of hundreds of unknowns. To efficiently exploit the computing power of modern accelerator hardware, these linear systems are processed in batches. To improve the numerical stability of the Gaussian Elimination, at least partial pivoting is required, most often accomplished with row pivoting. However, row pivoting can result in a severe performance penalty on GPUs because it brings in thread divergence and non-coalesced memory accesses. The state-of-the-art libraries for linear algebra that target GPUs, such as MAGMA, focus on large matrix sizes. They change the data layout by transposing the matrix to avoid these divergence and non-coalescing penalties. However, the data movement associated with transposition is very expensive for small matrices. In this paper, we propose a batched LU factorization for GPUs by using a multi-level blocked right looking algorithm that preserves the data layout but minimizes the penalty of partial pivoting. Our batched LU achieves up to 2:5-fold speedup when compared to the alternative CUBLAS solutions on a K40c GPU and 3:6-fold speedup over MKL on a node of the Titan supercomputer at ORNL in a nuclear reaction network simulation.},
	author = {Tingxing Dong and Azzam Haidar and Piotr Luszczek and James Harris and Stanimire Tomov and Jack Dongarra}
}
@article {831,
	title = {Model-Driven One-Sided Factorizations on Multicore, Accelerated Systems},
	journal = {Supercomputing Frontiers and Innovations},
	volume = {1},
	year = {2014},
	abstract = {Hardware heterogeneity of the HPC platforms is no longer considered unusual but instead have become the most viable way forward towards Exascale.  In fact, the multitude of the heterogeneous resources available to modern computers are designed for different workloads and their efficient use is closely aligned with the specialized role envisaged by their design.  Commonly in order to efficiently use such GPU resources, the workload in question must have a much greater degree of parallelism than workloads often associated with multicore processors (CPUs).  Available GPU variants differ in their internal architecture and, as a result, are capable of handling workloads of varying degrees of complexity and a range of computational patterns.  This vast array of applicable workloads will likely lead to an ever accelerated mixing of multicore-CPUs and GPUs in multi-user environments with the ultimate goal of offering adequate computing facilities for a wide range of scientific and technical workloads.  In the following paper, we present a research prototype that uses a lightweight runtime environment to manage the resource-specific workloads, and to control the dataflow and parallel execution in hybrid systems.  Our lightweight runtime environment uses task superscalar concepts to enable the developer to write serial code while providing parallel execution.  This concept is reminiscent of dataflow and systolic architectures in its conceptualization of a workload as a set of side-effect-free tasks that pass data items whenever the associated work assignment have been completed.  Additionally, our task abstractions and their parametrization enable uniformity in the algorithmic development across all the heterogeneous resources without sacrificing precious compute cycles.  We include performance results for dense linear algebra functions which demonstrate the practicality and effectiveness of our approach that is aptly capable of full utilization of a wide range of accelerator hardware.},
	keywords = {dense linear algebra, hardware accelerators, task superscalar scheduling},
	doi = {http://dx.doi.org/10.14529/jsfi1401},
	author = {Jack Dongarra and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Asim YarKhan}
}
@conference {867,
	title = {A Multithreaded Communication Substrate for OpenSHMEM},
	booktitle = {8th International Conference on Partitioned Global Address Space Programming Models (PGAS)},
	year = {2014},
	month = {2014-10},
	address = {Eugene, OR},
	abstract = {OpenSHMEM scalability is strongly dependent on the capa- bility of its communication layer to efficiently handle multi- ple threads. In this paper, we present an early evaluation of the thread safety specification in the Unified Common Com- munication Substrate (UCCS) employed in OpenSHMEM. Results demonstrate that thread safety can be provided at an acceptable cost and can improve efficiency for some op- erations, compared to serializing communication.},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca}
}
@conference {839,
	title = {New Algorithm for Computing Eigenvectors of the Symmetric Eigenvalue Problem},
	booktitle = {Workshop on Parallel and Distributed Scientific and Engineering Computing, IPDPS 2014 (Best Paper)},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {We describe a design and implementation of a multi-stage algorithm for computing eigenvectors of a dense symmetric matrix. We show that reformulating the existing algorithms is beneficial in terms of performance even if that doubles the computational complexity. Through detailed analysis, we show that the effect of the increase in the asymptotic operation count may be compensated by a much improved performance rate. Our performance results indicate that using our approach achieves very good speedup and scalability even when directly compared with the existing state-of-the-art software.},
	doi = {10.1109/IPDPSW.2014.130},
	author = {Azzam Haidar and Piotr Luszczek and Jack Dongarra}
}
@article {758,
	title = {A Novel Hybrid CPU-GPU Generalized Eigensolver for Electronic Structure Calculations Based on Fine Grained Memory Aware Tasks},
	journal = {International Journal of High Performance Computing Applications},
	volume = {28},
	year = {2014},
	month = {2014-05},
	pages = {196-209},
	chapter = {196},
	abstract = {The adoption of hybrid CPU{\textendash}GPU nodes in traditional supercomputing platforms such as the Cray-XK6 opens acceleration opportunities for electronic structure calculations in materials science and chemistry applications, where medium-sized generalized eigenvalue problems must be solved many times. These eigenvalue problems are too small to effectively solve on distributed systems, but can benefit from the massive computing power concentrated on a single-node, hybrid CPU{\textendash}GPU system. However, hybrid systems call for the development of new algorithms that efficiently exploit heterogeneity and massive parallelism of not just GPUs, but of multicore/manycore CPUs as well. Addressing these demands, we developed a generalized eigensolver featuring novel algorithms of increased computational intensity (compared with the standard algorithms), decomposition of the computation into fine-grained memory aware tasks, and their hybrid execution. The resulting eigensolvers are state-of-the-art in high-performance computing, significantly outperforming existing libraries. We describe the algorithm and analyze its performance impact on applications of interest when different fractions of eigenvectors are needed by the host electronic structure code. },
	keywords = {Eigensolver, electronic structure calculations, generalized eigensolver, gpu, high performance, hybrid, Multicore, two-stage},
	doi = {10.1177/1094342013502097 },
	author = {Azzam Haidar and Raffaele Solc{\`a} and Mark Gates and Stanimire Tomov and Thomas C. Schulthess and Jack Dongarra}
}
@conference {828,
	title = {Performance and Portability with OpenCL for Throughput-Oriented HPC Workloads Across Accelerators, Coprocessors, and Multicore Processors},
	booktitle = {5th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems (ScalA {\textquoteright}14)},
	year = {2014},
	month = {2014-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {New Orleans, LA},
	abstract = {Ever since accelerators and coprocessors became the mainstream hardware for throughput-oriented HPC workloads, various programming techniques have been proposed to increase productivity in terms of both the performance and ease-of-use. We evaluate these aspects of OpenCL on a number of hardware platforms for an important subset of dense linear algebra operations that are relevant to a wide range of scientific applications. Our findings indicate that OpenCL portability has improved since our previous publication and many new and surprising usage scenarios are possible that rival those available after decades of software development on the CPUs. The combined performance-portability metric, even though not promised by the OpenCL standard, reflects the need for tuning performance-critical operations during the porting process and we show how a large portion of the available efficiency is lost if the tuning is not done correctly.},
	doi = {10.1109/ScalA.2014.8},
	author = {Azzam Haidar and Chongxiao Cao and Ichitaro Yamazaki and Jack Dongarra and Mark Gates and Piotr Luszczek and Stanimire Tomov}
}
@article {814,
	title = {Performance and Reliability Trade-offs for the Double Checkpointing Algorithm},
	journal = {International Journal of Networking and Computing},
	volume = {4},
	number = {1},
	year = {2014},
	month = {2014},
	pages = {32-41},
	chapter = {32},
	abstract = {Fast checkpointing algorithms require distributed access to stable storage. This paper revisits the approach based upon double checkpointing, and compares the blocking algorithm of Zheng, Shi and Kal{\'e} [23], with the non-blocking algorithm of Ni, Meneses and Kal{\'e} [15] in terms of both performance and risk. We also extend the model proposedcan provide a better efficiency in [23, 15] to assess the impact of the overhead associated to non-blocking communications. In addition, we deal with arbitrary failure distributions (as opposed to uniform distributions in [23]). We then provide a new peer-to-peer checkpointing algorithm, called the triple checkpointing algorithm, that can work without additional memory, and achieves both higher efficiency and better risk handling than the double checkpointing algorithm. We provide performance and risk models for all the evaluated protocols, and compare them through comprehensive simulations.},
	keywords = {communication contention, in-memory checkpoint, performance, resilience, risk},
	issn = {2185-2847},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@conference {871,
	title = {PTG: An Abstraction for Unhindered Parallelism},
	booktitle = {International Workshop on Domain-Specific Languages and High-Level Frameworks for High Performance Computing (WOLFHPC)},
	year = {2014},
	month = {2014-11},
	publisher = {IEEE Press},
	organization = {IEEE Press},
	address = {New Orleans, LA},
	abstract = {<p>
Increased parallelism and use of heterogeneous computing resources is now an established trend in High Performance Computing (HPC), a trend that, looking forward to Exascale, seems bound to intensify. Despite the evolution of hardware over the past decade, the programming paradigm of choice was invariably derived from Coarse Grain Parallelism with explicit data movements. We argue that message passing has remained the de facto standard in HPC because, until now, the ever increasing challenges that application developers had to address to create efficient portable applications remained manageable for expert programmers.
</p>
<p>
Data-flow based programming is an alternative approach with significant potential. In this paper, we discuss the Parameterized Task Graph (PTG) abstraction and present the specialized input language that we use to specify PTGs in our data-flow task-based runtime system, PaRSEC. This language and the corresponding execution model are in contrast with the execution model of explicit message passing as well as the model of alternative task based runtime systems. The Parameterized Task Graph language decouples the expression of the parallelism in the algorithm from the control-flow ordering, load balance, and data distribution. Thus, programs are more adaptable and map more efficiently on challenging hardware, as well as maintain portability across diverse architectures. To support these claims, we discuss the different challenges of HPC programming and how PaRSEC can address them, and we demonstrate that in today{\textquoteright}s large scale supercomputers, PaRSEC can significantly outperform state-of-the-art MPI applications and libraries, a trend that will increase with future architectural evolution.
</p>},
	keywords = {dte, parsec, plasma},
	author = {Anthony Danalis and George Bosilca and Aurelien Bouteiller and Thomas Herault and Jack Dongarra}
}
@conference {879,
	title = {Search Space Pruning Constraints Visualization},
	booktitle = {VISSOFT{\textquoteright}14: 2nd IEEE Working Conference on Software Visualization},
	year = {2014},
	month = {2014-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Victoria, BC, Canada},
	abstract = {The field of software optimization, among others, is interested in finding an optimal solution in a large search space. These search spaces are often large, complex, non-linear and even non-continuous at times. The size of the search space makes a brute force solution intractable. As a result, one or more search space pruning constraints are often used to reduce the number of candidate configurations that must be evaluated in order to solve the optimization problem.

If more than one pruning constraint is employed, it can be challenging to understand how the pruning constraints interact and overlap. This work presents a visualization technique based on a radial, space-filling technique that allows the user to gain a better understanding of how the pruning constraints remove candidates from the search space. The technique is then demonstrated using a search space pruning data set derived from the optimization of a matrix multiplication code for NVIDIA CUDA accelerators.},
	author = {Blake Haugen and Jakub Kurzak}
}
@conference {809,
	title = {Unified Development for Mixed Multi-GPU and Multi-Coprocessor Environments using a Lightweight Runtime Environment},
	booktitle = {IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Many of the heterogeneous resources available to modern computers are designed for different workloads. In order to efficiently use GPU resources, the workload must have a greater degree of parallelism than a workload designed for multicore-CPUs. And conceptually, the Intel Xeon Phi coprocessors are capable of handling workloads somewhere in between the two. This multitude of applicable workloads will likely lead to mixing multicore-CPUs, GPUs, and Intel coprocessors in multi-user environments that must offer adequate computing facilities for a wide range of workloads. In this work, we are using a lightweight runtime environment to manage the resourcespecific workload, and to control the dataflow and parallel execution in two-way hybrid systems. The lightweight runtime environment uses task superscalar concepts to enable the developer to write serial code while providing parallel execution. In addition, our task abstractions enable unified algorithmic development across all the heterogeneous resources. We provide performance results for dense linear algebra applications, demonstrating the effectiveness of our approach and full utilization of a wide variety of accelerator hardware.},
	keywords = {algorithms, Computer science, CUDA, Heterogeneous systems, Intel Xeon Phi, linear algebra, nVidia, Tesla K20, Tesla M2090},
	author = {Azzam Haidar and Chongxiao Cao and Jack Dongarra and Piotr Luszczek and Stanimire Tomov}
}
@article {icl:721,
	title = {Accelerating Linear System Solutions Using Randomization Techniques},
	journal = {ACM Transactions on Mathematical Software (also LAWN 246)},
	volume = {39},
	year = {2013},
	month = {2013-02},
	abstract = {We illustrate how linear algebra calculations can be enhanced by statistical techniques in the case of a square linear system Ax = b. We study a random transformation of A that enables us to avoid pivoting and then to reduce the amount of communication. Numerical experiments show that this randomization can be performed at a very affordable computational price while providing us with a satisfying accuracy when compared to partial pivoting. This random transformation called Partial Random Butterfly Transformation (PRBT) is optimized in terms of data storage and flops count. We propose a solver where PRBT and the LU factorization with no pivoting take advantage of the current hybrid multicore/GPU machines and we compare its Gflop/s performance with a solver implemented in a current parallel library.},
	keywords = {algorithms, dense linear algebra, experimentation, graphics processing units, linear systems, lu factorization, multiplicative preconditioning, numerical linear algebra, performance, plasma, randomization},
	doi = {10.1145/2427023.2427025},
	url = {http://dl.acm.org/citation.cfm?id=2427025},
	author = {Marc Baboulin and Jack Dongarra and Julien Herrmann and Stanimire Tomov}
}
@techreport {694,
	title = {Assessing the impact of ABFT and Checkpoint composite strategies},
	number = {ICL-UT-13-03},
	year = {2013},
	abstract = {Algorithm-specific fault tolerant approaches promise unparalleled scalability and performance in failure-prone environments. With the advances in the theoretical and practical understanding of algorithmic traits enabling such approaches, a growing number of frequently used algorithms (including all widely used factorization kernels) have been proven capable of such properties. These algorithms provide a temporal section of the execution when the data is protected by it{\textquoteright}s own intrinsic properties, and can be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only fault-tolerance approach that is currently used for these applications is checkpoint/restart. In this paper we propose a model and a simulator to investigate the behavior of a composite protocol, that alternates between ABFT and checkpoint/restart protection for effective protection of each phase of an iterative application composed of ABFT-aware and ABFT-unaware sections. We highlight this approach drastically increases the performance delivered by the system, especially at scale, by providing means to rarefy the checkpoints while simultaneously decreasing the volume of data needed to be checkpointed.},
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, resilience},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@article {icl:719,
	title = {A Block-Asynchronous Relaxation Method for Graphics Processing Units},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {73},
	year = {2013},
	month = {2013-12},
	pages = {1613{\textendash}1626},
	abstract = {In this paper, we analyze the potential of asynchronous relaxation methods on Graphics Processing Units (GPUs). We develop asynchronous iteration algorithms in CUDA and compare them with parallel implementations of synchronous relaxation methods on CPU- or GPU-based systems. For a set of test matrices from UFMC we investigate convergence behavior, performance and tolerance to hardware failure. We observe that even for our most basic asynchronous relaxation scheme, the method can efficiently leverage the GPUs computing power and is, despite its lower convergence rate compared to the Gauss{\textendash}Seidel relaxation, still able to provide solution approximations of certain accuracy in considerably shorter time than Gauss{\textendash}Seidel running on CPUs- or GPU-based Jacobi. Hence, it overcompensates for the slower convergence by exploiting the scalability and the good fit of the asynchronous schemes for the highly parallel GPU architectures. Further, enhancing the most basic asynchronous approach with hybrid schemes{\textendash}using multiple iterations within the {\textquoteleft}{\textquoteleft}subdomain{\textquoteright}{\textquoteright} handled by a GPU thread block{\textendash}we manage to not only recover the loss of global convergence but often accelerate convergence of up to two times, while keeping the execution time of a global iteration practically the same. The combination with the advantageous properties of asynchronous iteration methods with respect to hardware failure identifies the high potential of the asynchronous methods for Exascale computing.},
	doi = {http://dx.doi.org/10.1016/j.jpdc.2013.05.008},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra and Vincent Heuveline}
}
@techreport {684,
	title = {On the Combination of Silent Error Detection and Checkpointing},
	year = {2013},
	month = {2013-06},
	publisher = {University of Tennessee Computer Science Technical Report},
	abstract = {In this paper, we revisit traditional checkpointing and rollback recovery strategies, with a focus on silent data corruption errors. Contrarily to fail-stop failures, such latent errors cannot be detected immediately, and a mechanism to detect them must be provided. We consider two models: (i) errors are detected after some delays following a probability distribution (typically, an Exponential distribution); (ii) errors are detected through some verification mechanism. In both cases, we compute the optimal period in order to minimize the waste, i.e., the fraction of time where nodes do not perform useful computations. In practice, only a fixed number of checkpoints can be kept in memory, and the first model may lead to an irrecoverable failure. In this case, we compute the minimum period required for an acceptable risk. For the second model, there is no risk of irrecoverable failure, owing to the verification mechanism, but the corresponding overhead is included in the waste. Finally, both models are instantiated using realistic scenarios and application/architecture parameters.},
	keywords = {checkpointing, error recovery, High-performance computing, silent data corruption, verification},
	url = {http://www.netlib.org/lapack/lawnspdf/lawn278.pdf},
	author = {Guillaume Aupy and Anne Benoit and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@article {750,
	title = {Correlated Set Coordination in Fault Tolerant Message Logging Protocols},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {25},
	year = {2013},
	month = {2013-03},
	pages = {572-585},
	abstract = {With our current expectation for the exascale systems, composed of hundred of thousands of many-core nodes, the mean time between failures will become small, even under the most optimistic assumptions. One of the most scalable checkpoint restart techniques, the message logging approach, is the most challenged when the number of cores per node increases because of the high overhead of saving the message payload. Fortunately, for two processes on the same node, the failure probability is correlated, meaning that coordinated recovery is free. In this paper, we propose an intermediate approach that uses coordination between correlated processes but retains the scalability advantage of message logging between independent ones. The algorithm still belongs to the family of event logging protocols but eliminates the need for costly payload logging between coordinated processes.},
	doi = {10.1002/cpe.2859},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {icl:698,
	title = {Dense Linear Algebra on Distributed Heterogeneous Hardware with a Symbolic DAG Approach},
	journal = {Scalable Computing and Communications: Theory and Practice},
	year = {2013},
	month = {2013-03},
	pages = {699-735},
	publisher = {John Wiley \& Sons},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Piotr Luszczek and Jack Dongarra},
	editor = {Samee Khan and Lin-Wang Wang and Albert Zomaya}
}
@techreport {703,
	title = {Designing LU-QR hybrid solvers for performance and stability},
	number = {ut-eecs-13-719},
	year = {2013},
	month = {2013-10},
	publisher = {University of Tennessee},
	author = {Mathieu Faverge and Julien Herrmann and Julien Langou and Bradley Lowery and Yves Robert and Jack Dongarra}
}
@article {icl:538,
	title = {Enabling Workflows in GridSolve: Request Sequencing and Service Trading},
	journal = {Journal of Supercomputing},
	volume = {64},
	year = {2013},
	month = {2013-06},
	pages = {1133-1152},
	chapter = {1133},
	abstract = {GridSolve employs a RPC-based client-agent-server model for solving computational problems. There are two deficiencies associated with GridSolve when a computational problem essentially forms a workflow consisting of a sequence of tasks with data dependencies between them. First, intermediate results are always passed through the client, resulting in unnecessary data transport. Second, since the execution of each individual task is a separate RPC session, it is difficult to enable any potential parallelism among tasks. This paper presents a request sequencing technique that addresses these deficiencies and enables workflow executions. Building on the request sequencing work, one way to generate workflows is by taking higher level service requests and decomposing them into a sequence of simpler service requests using a technique called service trading. A service trading component is added to GridSolve to take advantage of the new dynamic request sequencing. The features described here include automatic DAG construction and data dependency analysis, direct interserver data transfer, parallel task execution capabilities, and a service trading component.},
	keywords = {grid computing, gridpac, netsolve, service trading, workflow applications},
	issn = {1573-0484},
	doi = {10.1007/s11227-010-0549-1},
	author = {Yinan Li and Asim YarKhan and Jack Dongarra and Keith Seymour and Aurlie Hurault}
}
@article {869,
	title = {An evaluation of User-Level Failure Mitigation support in MPI},
	journal = {Computing},
	volume = {95},
	year = {2013},
	month = {2013-12},
	pages = {1171-1184},
	abstract = {As the scale of computing platforms becomes increasingly extreme, the requirements for application fault tolerance are increasing as well. Techniques to address this problem by improving the resilience of algorithms have been developed, but they currently receive no support from the programming model, and without such support, they are bound to fail. This paper discusses the failure-free overhead and recovery impact of the user-level failure mitigation proposal presented in the MPI Forum. Experiments demonstrate that fault-aware MPI has little or no impact on performance for a range of applications, and produces satisfactory recovery times when there are failures.},
	keywords = {Fault tolerance, MPI, User-level fault mitigation},
	doi = {10.1007/s00607-013-0331-3},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and Joshua Hursey and George Bosilca and Jack Dongarra}
}
@article {691,
	title = {Extending the scope of the Checkpoint-on-Failure protocol for forward recovery in standard MPI},
	journal = {Concurrency and Computation: Practice and Experience},
	year = {2013},
	month = {2013-07},
	abstract = {Most predictions of exascale machines picture billion ways parallelism, encompassing not only millions of cores but also tens of thousands of nodes. Even considering extremely optimistic advances in hardware reliability, probabilistic amplification entails that failures will be unavoidable. Consequently, software fault tolerance is paramount to maintain future scientific productivity. Two major problems hinder ubiquitous adoption of fault tolerance techniques: (i) traditional checkpoint-based approaches incur a steep overhead on failure free operations and (ii) the dominant programming paradigm for parallel applications (the message passing interface (MPI) Standard) offers extremely limited support of software-level fault tolerance approaches. In this paper, we present an approach that relies exclusively on the features of a high quality implementation, as defined by the current MPI Standard, to enable advanced forward recovery techniques, without incurring the overhead of customary periodic checkpointing. With our approach, when failure strikes, applications regain control to make a checkpoint before quitting execution. This checkpoint is in reaction to the failure occurrence rather than periodic. This checkpoint is reloaded in a new MPI application, which restores a sane environment for the forward, application-based recovery technique to repair the failure-damaged dataset. The validity and performance of this approach are evaluated on large-scale systems, using the QR factorization as an example. Published 2013. This article is a US Government work and is in the public domain in the USA.},
	doi = {10.1002/cpe.3100},
	url = {http://doi.wiley.com/10.1002/cpe.3100},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {752,
	title = {Hierarchical QR Factorization Algorithms for Multi-core Cluster Systems},
	journal = {Parallel Computing},
	volume = {39},
	year = {2013},
	month = {2013-05},
	pages = {212-232},
	abstract = {This paper describes a new QR factorization algorithm which is especially designed for massively parallel platforms combining parallel distributed nodes, where a node is a multi-core processor. These platforms represent the present and the foreseeable future of high-performance computing. Our new QR factorization algorithm falls in the category of the tile algorithms which naturally enables good data locality for the sequential kernels executed by the cores (high sequential performance), low number of messages in a parallel distributed setting (small latency term), and fine granularity (high parallelism). Each tile algorithm is uniquely characterized by its sequence of reduction trees. In the context of a cluster of nodes, in order to minimize the number of inter-processor communications (aka, {\textquoteleft}{\textquoteleft}communication-avoiding{\textquoteright}{\textquoteright}), it is natural to consider hierarchical trees composed of an {\textquoteleft}{\textquoteleft}inter-node{\textquoteright}{\textquoteright} tree which acts on top of {\textquoteleft}{\textquoteleft}intra-node{\textquoteright}{\textquoteright} trees. At the intra-node level, we propose a hierarchical tree made of three levels: (0) {\textquoteleft}{\textquoteleft}TS level{\textquoteright}{\textquoteright} for cache-friendliness, (1) {\textquoteleft}{\textquoteleft}low-level{\textquoteright}{\textquoteright} for decoupled highly parallel inter-node reductions, (2) {\textquoteleft}{\textquoteleft}domino level{\textquoteright}{\textquoteright} to efficiently resolve interactions between local reductions and global reductions. Our hierarchical algorithm and its implementation are flexible and modular, and can accommodate several kernel types, different distribution layouts, and a variety of reduction trees at all levels, both inter-node and intra-node. Numerical experiments on a cluster of multi-core nodes (i) confirm that each of the four levels of our hierarchical tree contributes to build up performance and (ii) build insights on how these levels influence performance and interact within each other. Our implementation of the new algorithm with the DAGUE scheduling tool significantly outperforms currently available QR factorization software for all matrix shapes, thereby bringing a new advance in numerical linear algebra for petascale and exascale platforms.},
	keywords = {Cluster, Distributed memory, Hierarchical architecture, multi-core, numerical linear algebra, QR factorization},
	author = {Jack Dongarra and Mathieu Faverge and Thomas Herault and Mathias Jacquelin and Julien Langou and Yves Robert}
}
@conference {696,
	title = {An Improved Parallel Singular Value Algorithm and Its Implementation for Multicore Hardware},
	booktitle = {Supercomputing 2013},
	year = {2013},
	month = {2013-11},
	address = {Denver, CO},
	author = {Azzam Haidar and Piotr Luszczek and Jakub Kurzak and Jack Dongarra}
}
@techreport {704,
	title = {An Improved Parallel Singular Value Algorithm and Its Implementation for Multicore Hardware},
	number = {ut-eecs-13-720},
	year = {2013},
	month = {2013-10},
	publisher = {University of Tennessee},
	keywords = {lapack, plasma, scalapack},
	author = {Azzam Haidar and Piotr Luszczek and Jakub Kurzak and Jack Dongarra}
}
@inbook {762,
	title = {Keeneland: Computational Science Using Heterogeneous GPU Computing},
	booktitle = {Contemporary High Performance Computing: From Petascale Toward Exascale},
	series = {CRC Computational Science Series},
	year = {2013},
	publisher = {Taylor and Francis},
	organization = {Taylor and Francis},
	chapter = {7},
	address = {Boca Raton, FL},
	abstract = {The Keeneland Project is a five year Track 2D grant awarded by the National Science Foundation (NSF) under solicitation NSF 08-573 in August 2009 for the development and deployment of an innovative high performance computing system. The Keeneland project is led by the Georgia Institute of Technology (Georgia Tech) in collaboration with the University of Tennessee at Knoxville, National Institute of Computational Sciences, and Oak Ridge National Laboratory.},
	author = {Jeffrey Vetter and Richard Glassbrook and Karsten Schwan and Sudha Yalamanchili and Mitch Horton and Ada Gavrilovska and Magda Slawinska and Jack Dongarra and Jeremy Meredith and Philip Roth and Kyle Spafford and Stanimire Tomov and John Wynkoop}
}
@inproceedings {757,
	title = {Leading Edge Hybrid Multi-GPU Algorithms for Generalized Eigenproblems in Electronic Structure Calculations},
	volume = {7905},
	year = {2013},
	month = {2013-06},
	pages = {67-80},
	publisher = {Springer Berlin Heidelberg},
	edition = {Lecture Notes in Computer Science},
	address = {Leipzig, Germany},
	abstract = {Today{\textquoteright}s high computational demands from engineering fields and complex hardware development make it necessary to develop and optimize new algorithms toward achieving high performance and good scalability on the next generation of computers. The enormous gap between the high-performance capabilities of GPUs and the slow interconnect between them has made the development of numerical software that is scalable across multiple GPUs extremely challenging. We describe and analyze a successful methodology to address the challenges{\textemdash}starting from our algorithm design, kernel optimization and tuning, to our programming model{\textemdash}in the development of a scalable high-performance generalized eigenvalue solver in the context of electronic structure calculations in materials science applications. We developed a set of leading edge dense linear algebra algorithms, as part of a generalized eigensolver, featuring fine grained memory aware kernels, a task based approach and hybrid execution/scheduling. The goal of the new design is to increase the computational intensity of the major compute kernels and to reduce synchronization and data transfers between GPUs. We report the performance impact on the generalized eigensolver when different fractions of eigenvectors are needed. The algorithm described provides an enormous performance boost compared to current GPU-based solutions, and performance comparable to state-of-the-art distributed solutions, using a single node with multiple GPUs.},
	isbn = {978-3-642-38750-0},
	doi = {10.1007/978-3-642-38750-0_6},
	author = {Azzam Haidar and Stanimire Tomov and Jack Dongarra and Raffaele Solc{\`a} and Thomas C. Schulthess}
}
@article {756,
	title = {Level-3 Cholesky Factorization Routines Improve Performance of Many Cholesky Algorithms},
	journal = {ACM Transactions on Mathematical Software (TOMS)},
	volume = {39},
	year = {2013},
	month = {2013-02},
	abstract = {Four routines called DPOTF3i, i = a,b,c,d, are presented. DPOTF3i are a novel type of level-3 BLAS for use by BPF (Blocked Packed Format) Cholesky factorization and LAPACK routine DPOTRF. Performance of routines DPOTF3i are still increasing when the performance of Level-2 routine DPOTF2 of LAPACK starts decreasing. This is our main result and it implies, due to the use of larger block size nb, that DGEMM, DSYRK, and DTRSM performance also increases! The four DPOTF3i routines use simple register blocking. Different platforms have different numbers of registers. Thus, our four routines have different register blocking sizes.

BPF is introduced. LAPACK routines for POTRF and PPTRF using BPF instead of full and packed format are shown to be trivial modifications of LAPACK POTRF source codes. We call these codes BPTRF. There are two variants of BPF: lower and upper. Upper BPF is {\textquotedblleft}identical{\textquotedblright} to Square Block Packed Format (SBPF). {\textquotedblleft}LAPACK{\textquotedblright} implementations on multicore processors use SBPF. Lower BPF is less efficient than upper BPF. Vector inplace transposition converts lower BPF to upper BPF very efficiently. Corroborating performance results for DPOTF3i versus DPOTF2 on a variety of common platforms are given for n ≈ nb as well as results for large n comparing DBPTRF versus DPOTRF.},
	doi = {10.1145/2427023.2427026},
	author = {Fred G. Gustavson and Jerzy Wasniewski and Jack Dongarra and Jos{\'e} Herrero and Julien Langou}
}
@techreport {icl:733,
	title = {Multi-criteria checkpointing strategies: optimizing response-time versus resource utilization},
	number = {ICL-UT-13-01},
	year = {2013},
	month = {2013-02},
	abstract = {Failures are increasingly threatening the eciency of HPC systems, and current projections of Exascale platforms indicate that rollback recovery, the most convenient method for providing fault tolerance to generalpurpose applications, reaches its own limits at such scales. One of the reasons explaining this unnerving situation comes from the focus that has been given to per-application completion time, rather than to platform efficiency. In this paper, we discuss the case of uncoordinated rollback recovery where the idle time spent waiting recovering processors is used to progress a different, independent application from the system batch queue. We then propose an extended model of uncoordinated checkpointing that can discriminate between idle time and wasted computation. We instantiate this model in a simulator to demonstrate that, with this strategy, uncoordinated checkpointing per application completion time is unchanged, while it delivers near-perfect platform efficiency.},
	author = {Aurelien Bouteiller and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert}
}
@conference {868,
	title = {Multi-criteria Checkpointing Strategies: Response-Time versus Resource Utilization},
	booktitle = {Euro-Par 2013},
	year = {2013},
	month = {2013-08},
	publisher = {Springer},
	organization = {Springer},
	address = {Aachen, Germany},
	abstract = {Failures are increasingly threatening the efficiency of HPC systems, and current projections of Exascale platforms indicate that roll- back recovery, the most convenient method for providing fault tolerance to general-purpose applications, reaches its own limits at such scales. One of the reasons explaining this unnerving situation comes from the focus that has been given to per-application completion time, rather than to platform efficiency. In this paper, we discuss the case of uncoordinated rollback recovery where the idle time spent waiting recovering processors is used to progress a different, independent application from the sys- tem batch queue. We then propose an extended model of uncoordinated checkpointing that can discriminate between idle time and wasted com- putation. We instantiate this model in a simulator to demonstrate that, with this strategy, uncoordinated checkpointing per application comple- tion time is unchanged, while it delivers near-perfect platform efficiency.},
	author = {Aurelien Bouteiller and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert}
}
@techreport {702,
	title = {Optimal Checkpointing Period: Time vs. Energy},
	number = {ut-eecs-13-718},
	year = {2013},
	month = {2013-10},
	publisher = {University of Tennessee},
	author = {Guillaume Aupy and Anne Benoit and Thomas Herault and Yves Robert and Jack Dongarra}
}
@article {749,
	title = {PaRSEC: Exploiting Heterogeneity to Enhance Scalability},
	journal = {IEEE Computing in Science and Engineering},
	volume = {15},
	year = {2013},
	month = {2013-11},
	pages = {36-45},
	abstract = {New high-performance computing system designs with steeply escalating processor and core counts, burgeoning heterogeneity and accelerators, and increasingly unpredictable memory access times call for dramatically new programming paradigms. These new approaches must react and adapt quickly to unexpected contentions and delays, and they must provide the execution environment with sufficient intelligence and flexibility to rearrange the execution to improve resource utilization.},
	doi = {10.1109/MCSE.2013.98},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Thomas Herault and Jack Dongarra}
}
@conference {753,
	title = {Portable HPC Programming on Intel Many-Integrated-Core Hardware with MAGMA Port to Xeon Phi},
	booktitle = {PPAM 2013},
	year = {2013},
	month = {2013-09},
	address = {Warsaw, Poland},
	abstract = {This paper presents the design and implementation of several fundamental dense linear algebra (DLA) algorithms for multicore with Intel Xeon Phi Coprocessors. In particular, we consider algorithms for solving linear systems. Further, we give an overview of the MAGMA MIC library, an open source, high performance library that incorporates the developments presented, and in general provides to heterogeneous architectures of multicore with coprocessors the DLA functionality of the popular LAPACK library. The LAPACK-compliance simplifies the use of the MAGMA MIC library in applications, while providing them with portably performant DLA. High performance is obtained through use of the high-performance BLAS, hardware-specific tuning, and a hybridization methodology where we split the algorithm into computational tasks of various granularities. Execution of those tasks is properly scheduled over the heterogeneous hardware components by minimizing data movements and mapping algorithmic requirements to the architectural strengths of the various heterogeneous hardware components. Our methodology and programming techniques are incorporated into the MAGMA MIC API, which abstracts the application developer from the specifics of the Xeon Phi architecture and is therefore applicable to algorithms beyond the scope of DLA.},
	keywords = {magma, mic, xeon phi},
	author = {Jack Dongarra and Mark Gates and Azzam Haidar and Yulu Jia and Khairul Kabir and Piotr Luszczek and Stanimire Tomov}
}
@article {693,
	title = {Post-failure recovery of MPI communication capability: Design and rationale},
	journal = {International Journal of High Performance Computing Applications},
	volume = {27},
	year = {2013},
	month = {2013-01},
	pages = {244 - 254},
	abstract = {As supercomputers are entering an era of massive parallelism where the frequency of faults is increasing, the MPI Standard remains distressingly vague on the consequence of failures on MPI communications. Advanced fault-tolerance techniques have the potential to prevent full-scale application restart and therefore lower the cost incurred for each failure, but they demand from MPI the capability to detect failures and resume communications afterward. In this paper, we present a set of extensions to MPI that allow communication capabilities to be restored, while maintaining the extreme level of performance to which MPI users have become accustomed. The motivation behind the design choices are weighted against alternatives, a task that requires simultaneously considering MPI from the viewpoint of both the user and the implementor. The usability of the interfaces for expressing advanced recovery techniques is then discussed, including the difficult issue of enabling separate software layers to coordinate their recovery. },
	issn = {1094-3420},
	doi = {10.1177/1094342013488238},
	url = {http://hpc.sagepub.com/cgi/doi/10.1177/1094342013488238},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@techreport {icl:735,
	title = {Revisiting the Double Checkpointing Algorithm},
	number = {ut-cs-13-705},
	year = {2013},
	month = {2013-01},
	abstract = {Fast checkpointing algorithms require distributed access to stable storage. This paper revisits the approach base upon double checkpointing, and compares the blocking algorithm of Zheng, Shi and Kal{\'e}, with the non-blocking algorithm of Ni, Meneses and Kal{\'e} in terms of both performance and risk. We also extend the model that they have proposed to assess the impact of the overhead associated to non-blocking communications. We then provide a new peer-to-peer checkpointing algorithm, called the triple checkpointing algorithm, that can work at constant memory, and achieves both higher efficiency and better risk handling than the double checkpointing algorithm. We provide performance and risk models for all the evaluated protocols, and compare them through comprehensive simulations.},
	keywords = {checkpoint algorithm, communication overlap, fault-tolerance, performance model, resilience},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@conference {717,
	title = {Revisiting the Double Checkpointing Algorithm},
	booktitle = {15th Workshop on Advances in Parallel and Distributed Computational Models, at the IEEE International Parallel \& Distributed Processing Symposium},
	year = {2013},
	month = {2013-05},
	address = {Boston, MA},
	abstract = {Abstract{\textemdash}Fast checkpointing algorithms require distributed access to stable storage. This paper revisits the approach base upon double checkpointing, and compares the blocking algorithm of Zheng, Shi and Kale [1], with the non-blocking algorithm of Ni, Meneses and Kale [2] in terms of both performance and risk. We also extend the model proposed in [1], [2] to assess the impact of the overhead associated to non-blocking communications. We then provide a new peer-topeer checkpointing algorithm, called the triple checkpointing algorithm, that can work at constant memory, and achieves both higher efficiency and better risk handling than the double checkpointing algorithm. We provide performance and risk models for all the evaluated protocols, and compare them through comprehensive simulations.},
	author = {Jack Dongarra and Thomas Herault and Yves Robert}
}
@inbook {695,
	title = {Scalable Dense Linear Algebra on Heterogeneous Hardware},
	booktitle = {HPC: Transition Towards Exascale Processing, in the series Advances in Parallel Computing},
	year = {2013},
	abstract = {Abstract. Design of systems exceeding 1 Pflop/s and the push toward 1 Eflop/s, forced a dramatic shift in hardware design. Various physical and engineering constraints resulted in introduction of massive parallelism and functional hybridization with the use of accelerator units. This paradigm change brings about a serious challenge for application developers, as the management of multicore proliferation and heterogeneity rests on software. And it is reasonable to expect, that this situation will not change in the foreseeable future. This chapter presents a methodology of dealing with this issue in three common scenarios. In the context of shared-memory multicore installations, we show how high performance and scalability go hand in hand, when the well-known linear algebra algorithms are recast in terms of Direct Acyclic Graphs (DAGs), which are then transparently scheduled at runtime inside the Parallel Linear Algebra Software for Multicore Architectures (PLASMA) project. Similarly, Matrix Algebra on GPU and Multicore Architectures (MAGMA) schedules DAG-driven computations on multicore processors and accelerators. Finally, Distributed PLASMA (DPLASMA), takes the approach to distributed-memory machines with the use of automatic dependence analysis and the Direct Acyclic Graph Engine (DAGuE) to deliver high performance at the scale of many thousands of cores.},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@conference {761,
	title = {Standards for Graph Algorithm Primitives},
	booktitle = {17th IEEE High Performance Extreme Computing Conference (HPEC {\textquoteright}13)},
	year = {2013},
	month = {2013-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {It is our view that the state of the art in constructing a large collection of graph algorithms in terms of linear algebraic operations is mature enough to support the emergence of a standard set of primitive building blocks. This paper is a position paper defining the problem and announcing our intention to launch an open effort to define this standard.},
	keywords = {algorithms, graphs, linear algebra, software standards},
	doi = {10.1109/HPEC.2013.6670338},
	author = {Tim Mattson and David Bader and Jon Berry and Aydin Buluc and Jack Dongarra and Christos Faloutsos and John Feo and John Gilbert and Joseph Gonzalez and Bruce Hendrickson and Jeremy Kepner and Charles Lieserson and Andrew Lumsdaine and David Padua and Steve W. Poole and Steve Reinhardt and Mike Stonebraker and Steve Wallach and Andrew Yoo}
}
@techreport {683,
	title = {Toward a New Metric for Ranking High Performance Computing Systems},
	year = {2013},
	month = {2013-06},
	abstract = {The High Performance Linpack (HPL), or Top 500, benchmark is the most widely recognized and discussed metric for ranking high performance computing systems. However, HPL is increasingly unreliable as a true measure of system performance for a growing collection of important science and engineering applications. In this paper we describe a new high performance conjugate gradient (HPCG) benchmark. HPCG is composed of computations and data access patterns more commonly found in applications. Using HPCG we strive for a better correlation to real scientific application performance and expect to drive computer system design and implementation in directions that will better impact performance improvement.},
	url = {http://www.netlib.org/utk/people/JackDongarra/PAPERS/HPCG-Benchmark-utk.pdf},
	author = {Michael A. Heroux and Jack Dongarra}
}
@conference {686,
	title = {Toward a scalable multi-GPU eigensolver via compute-intensive kernels and efficient communication},
	booktitle = {Proceedings of the 27th ACM International Conference on Supercomputing (ICS {\textquoteright}13)},
	year = {2013},
	month = {2013-06},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Eugene, Oregon, USA},
	abstract = {The enormous gap between the high-performance capabilities of GPUs and the slow interconnect between them has made the development of numerical software that is scalable across multiple GPUs extremely challenging. We describe a successful methodology on how to address the challenges---starting from our algorithm design, kernel optimization and tuning, to our programming model---in the development of a scalable high-performance tridiagonal reduction algorithm for the symmetric eigenvalue problem. This is a fundamental linear algebra problem with many engineering and physics applications. We use a combination of a task-based approach to parallelism and a new algorithmic design to achieve high performance. The goal of the new design is to increase the computational intensity of the major compute kernels and to reduce synchronization and data transfers between GPUs. This may increase the number of flops, but the increase is offset by the more efficient execution and reduced data transfers. Our performance results are the best available, providing an enormous performance boost compared to current state-of-the-art solutions. In particular, our software scales up to 1070 Gflop/s using 16 Intel E5-2670 cores and eight M2090 GPUs, compared to 45 Gflop/s achieved by the optimized Intel Math Kernel Library (MKL) using only the 16 CPU cores.},
	keywords = {eigenvalue, gpu communication, gpu computation, heterogeneous programming model, performance, reduction to tridiagonal, singular value decomposiiton, task parallelism},
	isbn = {9781450321303},
	doi = {10.1145/2464996.2465438},
	url = {http://dl.acm.org/citation.cfm?doid=2464996.2465438},
	author = {Azzam Haidar and Mark Gates and Stanimire Tomov and Jack Dongarra},
	editor = {Allen D. Malony and Nemirovsky, Mario and Midkiff, Sam}
}
@article {748,
	title = {Unified Model for Assessing Checkpointing Protocols at Extreme-Scale},
	journal = {Concurrency and Computation: Practice and Experience},
	year = {2013},
	month = {2013-11},
	abstract = {In this paper, we present a unified model for several well-known checkpoint/restart protocols. The proposed model is generic enough to encompass both extremes of the checkpoint/restart space, from coordinated approaches to a variety of uncoordinated checkpoint strategies (with message logging). We identify a set of crucial parameters, instantiate them, and compare the expected efficiency of the fault tolerant protocols, for a given application/platform pair. We then propose a detailed analysis of several scenarios, including some of the most powerful currently available high performance computing platforms, as well as anticipated Exascale designs. The results of this analytical comparison are corroborated by a comprehensive set of simulations. Altogether, they outline comparative behaviors of checkpoint strategies at very large scale, thereby providing insight that is hardly accessible to direct experimentation.},
	doi = {10.1002/cpe.3173},
	author = {George Bosilca and Aurelien Bouteiller and Elisabeth Brunet and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@inproceedings {icl:672,
	title = {Algorithm-Based Fault Tolerance for Dense Matrix Factorization},
	year = {2012},
	month = {2012-02},
	pages = {225-234},
	publisher = {ACM},
	address = {New Orleans, LA, USA},
	abstract = {Dense matrix factorizations, such as LU, Cholesky and QR, are widely used for scientific applications that require solving systems of linear equations, eigenvalues and linear least squares problems. Such computations are normally carried out on supercomputers, whose ever-growing scale induces a fast decline of the Mean Time To Failure (MTTF). This paper proposes a new hybrid approach, based on Algorithm-Based Fault Tolerance (ABFT), to help matrix factorizations algorithms survive fail-stop failures. We consider extreme conditions, such as the absence of any reliable component and the possibility of loosing both data and checksum from a single failure. We will present a generic solution for protecting the right factor, where the updates are applied, of all above mentioned factorizations. For the left factor, where the panel has been applied, we propose a scalable checkpointing algorithm. This algorithm features high degree of checkpointing parallelism and cooperatively utilizes the checksum storage leftover from the right factor protection. The fault-tolerant algorithms derived from this hybrid solution is applicable to a wide range of dense matrix factorizations, with minor modifications. Theoretical analysis shows that the fault tolerance overhead sharply decreases with the scaling in the number of computing units and the problem size. Experimental results of LU and QR factorization on the Kraken (Cray XT5) supercomputer validate the theoretical evaluation and confirm negligible overhead, with- and without-errors.},
	keywords = {ft-la, ftmpi},
	doi = {10.1145/2145816.2145845},
	author = {Peng Du and Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra},
	editor = {J. Ramanujam and P. Sadayappan}
}
@article {icl:697,
	title = {Block-asynchronous Multigrid Smoothers for GPU-accelerated Systems},
	journal = {ICCS 2012},
	year = {2012},
	month = {2012-06},
	address = {Omaha, NE},
	author = {Hartwig Anzt and Stanimire Tomov and Mark Gates and Jack Dongarra and Vincent Heuveline}
}
@inproceedings {icl:679,
	title = {A Checkpoint-on-Failure Protocol for Algorithm-Based Recovery in Standard MPI},
	year = {2012},
	month = {2012-08},
	publisher = {Springer-Verlag},
	address = {Rhodes, Greece},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra},
	editor = {Christos Kaklamanis and Theodore Papatheodorou and Paul Spirakis}
}
@article {icl:695,
	title = {A Comprehensive Study of Task Coalescing for Selecting Parallelism Granularity in a Two-Stage Bidiagonal Reduction},
	journal = {IPDPS 2012},
	year = {2012},
	month = {2012-05},
	address = {Shanghai, China},
	author = {Azzam Haidar and Hatem Ltaeif and Piotr Luszczek and Jack Dongarra}
}
@article {icl:670,
	title = {DAGuE: A generic distributed DAG Engine for High Performance Computing.},
	journal = {Parallel Computing},
	volume = {38},
	number = {1-2},
	year = {2012},
	month = {2012-00},
	pages = {27-51},
	publisher = {Elsevier},
	keywords = {dague, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@inproceedings {icl:680,
	title = {An Evaluation of User-Level Failure Mitigation Support in MPI},
	year = {2012},
	month = {2012-09},
	publisher = {Springer},
	address = {Vienna, Austria},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and Joshua Hursey and George Bosilca and Jack Dongarra}
}
@techreport {icl:724,
	title = {Extending the Scope of the Checkpoint-on-Failure Protocol for Forward Recovery in Standard MPI},
	number = {ut-cs-12-702},
	year = {2012},
	month = {2012-00},
	keywords = {ftmpi},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@conference {icl:699,
	title = {From Serial Loops to Parallel Execution on Distributed Systems},
	booktitle = {International European Conference on Parallel and Distributed Computing (Euro-Par {\textquoteright}12)},
	year = {2012},
	month = {2012-08},
	address = {Rhodes, Greece},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Jack Dongarra}
}
@article {icl:723,
	title = {GPU-Accelerated Asynchronous Error Correction for Mixed Precision Iterative Refinement},
	journal = {EuroPar 2012 (also LAWN 260)},
	year = {2012},
	month = {2012-08},
	address = {Rhodes Island, Greece},
	author = {Hartwig Anzt and Piotr Luszczek and Jack Dongarra and Vincent Heuveline}
}
@inproceedings {icl:687,
	title = {Hierarchical QR Factorization Algorithms for Multi-Core Cluster Systems},
	year = {2012},
	month = {2012-05},
	publisher = {IEEE Computer Society Press},
	address = {Shanghai, China},
	author = {Jack Dongarra and Mathieu Faverge and Thomas Herault and Julien Langou and Yves Robert}
}
@techreport {icl:717,
	title = {How LAPACK library enables Microsoft Visual Studio support with CMake and LAPACKE},
	number = {UT-CS-12-698},
	year = {2012},
	month = {2012-07},
	author = {Julien Langou and Bill Hoffman and Brad King}
}
@article {1355,
	title = {MAGMA: A Breakthrough in Solvers for Eigenvalue Problems},
	year = {2012},
	month = {2012-05},
	publisher = {GPU Technology Conference (GTC12), Presentation},
	address = {San Jose, CA},
	author = {Stanimire Tomov and Jack Dongarra and Azzam Haidar and Ichitaro Yamazaki and Tingxing Dong and Thomas Schulthess and Raffaele Solc{\`a}}
}
@article {1349,
	title = {MAGMA: A New Generation of Linear Algebra Library for GPU and Multicore Architectures},
	year = {2012},
	month = {2012-11},
	publisher = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC12), Presentation},
	address = {Salt Lake City, UT},
	author = {Jack Dongarra and Tingxing Dong and Mark Gates and Azzam Haidar and Stanimire Tomov and Ichitaro Yamazaki}
}
@article {icl:729,
	title = {A Novel Hybrid CPU-GPU Generalized Eigensolver for Electronic Structure Calculations Based on Fine Grained Memory Aware Tasks},
	journal = {Supercomputing {\textquoteright}12 (poster)},
	year = {2012},
	month = {2012-11},
	address = {Salt Lake City, Utah},
	author = {Raffaele Solc{\`a} and Azzam Haidar and Stanimire Tomov and Jack Dongarra and Thomas C. Schulthess}
}
@techreport {icl:667,
	title = {A Proposal for User-Level Failure Mitigation in the MPI-3 Standard},
	number = {ut-cs-12-693},
	year = {2012},
	month = {2012-02},
	publisher = {University of Tennessee},
	keywords = {ftmpi},
	author = {Wesley Bland and George Bosilca and Aurelien Bouteiller and Thomas Herault and Jack Dongarra}
}
@article {icl:726,
	title = {Toward High Performance Divide and Conquer Eigensolver for Dense Symmetric Matrices},
	journal = {SIAM Journal on Scientific Computing (Accepted)},
	year = {2012},
	month = {2012-07},
	author = {Azzam Haidar and Hatem Ltaeif and Jack Dongarra}
}
@techreport {icl:716,
	title = {Unified Model for Assessing Checkpointing Protocols at Extreme-Scale},
	number = {UT-CS-12-697},
	year = {2012},
	month = {2012-06},
	author = {George Bosilca and Aurelien Bouteiller and Elisabeth Brunet and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@inproceedings {icl:713,
	title = {Weighted Block-Asynchronous Iteration on GPU-Accelerated Systems},
	year = {2012},
	month = {2012-08},
	address = {Rhodes Island, Greece},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra and Vincent Heuveline}
}
@article {icl:701,
	title = {Weighted Block-Asynchronous Relaxation for GPU-Accelerated Systems},
	journal = {SIAM Journal on Computing (submitted)},
	year = {2012},
	month = {2012-03},
	author = {Hartwig Anzt and Jack Dongarra and Vincent Heuveline}
}
@inproceedings {icl:607,
	title = {3-D parallel frequency-domain visco-acoustic wave modelling based on a hybrid direct/iterative solver},
	year = {2011},
	month = {2011-00},
	author = {Azzam Haidar and Luc Giraud and Hafedh Ben-Hadj-Ali and Florent Sourbier and St{\'e}phane Operto and Jean Virieux}
}
@article {icl:637,
	title = {Accelerating Linear System Solutions Using Randomization Techniques},
	journal = {INRIA RR-7616 / LAWN $\#$246 (presented at International AMMCS{\textquoteright}11)},
	year = {2011},
	month = {2011-07},
	address = {Waterloo, Ontario, Canada},
	keywords = {magma},
	author = {Marc Baboulin and Jack Dongarra and Julien Herrmann and Stanimire Tomov}
}
@inproceedings {icl:605,
	title = {Algebraic Schwarz Preconditioning for the Schur Complement: Application to the Time-Harmonic Maxwell Equations Discretized by a Discontinuous Galerkin Method.},
	year = {2011},
	month = {2011-02},
	address = {La Jolla, California},
	url = {http://hal.inria.fr/inria-00577639},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Stephane Lanteri and Jean Roman}
}
@techreport {icl:626,
	title = {Algorithm-based Fault Tolerance for Dense Matrix Factorizations},
	number = {UT-CS-11-676},
	year = {2011},
	month = {2011-08},
	address = {Knoxville, TN},
	keywords = {ft-la},
	author = {Peng Du and Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra}
}
@techreport {icl:631,
	title = {Analysis of Dynamically Scheduled Tile Algorithms for Dense Linear Algebra on Multicore Architectures},
	year = {2011},
	month = {2011-03},
	keywords = {plasma, quark},
	author = {Azzam Haidar and Hatem Ltaeif and Asim YarKhan and Jack Dongarra}
}
@article {icl:661,
	title = {Block-asynchronous Multigrid Smoothers for GPU-accelerated Systems},
	number = {UT-CS-11-689},
	year = {2011},
	month = {2011-12},
	keywords = {magma},
	author = {Hartwig Anzt and Stanimire Tomov and Mark Gates and Jack Dongarra and Vincent Heuveline}
}
@techreport {icl:656,
	title = {A Block-Asynchronous Relaxation Method for Graphics Processing Units},
	number = {UT-CS-11-687 / LAWN 258},
	year = {2011},
	month = {2011-11},
	keywords = {magma},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra and Vincent Heuveline}
}
@inproceedings {icl:640,
	title = {A Class of Hybrid LAPACK Algorithms for Multicore and GPU Architectures},
	year = {2011},
	month = {2011-07},
	address = {Knoxville, TN},
	keywords = {magma, quark},
	author = {Mitch Horton and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {icl:673,
	title = {Correlated Set Coordination in Fault Tolerant Message Logging Protocols},
	volume = {6853},
	year = {2011},
	month = {2011-08},
	pages = {51-64},
	publisher = {Springer},
	address = {Bordeaux, France},
	keywords = {ftmpi},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra},
	editor = {Emmanuel Jeannot and Raymond Namyst and Jean Roman}
}
@inproceedings {icl:675,
	title = {DAGuE: A Generic Distributed DAG Engine for High Performance Computing},
	year = {2011},
	month = {2011-00},
	pages = {1151-1158},
	publisher = {IEEE},
	address = {Anchorage, Alaska, USA},
	keywords = {dague, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@inproceedings {icl:676,
	title = {Flexible Development of Dense Linear Algebra Algorithms on Massively Parallel Architectures with DPLASMA},
	year = {2011},
	month = {2011-05},
	pages = {1432-1441},
	publisher = {IEEE},
	address = {Anchorage, Alaska, USA},
	keywords = {dague, dplasma, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@techreport {icl:662,
	title = {GPU-Accelerated Asynchronous Error Correction for Mixed Precision Iterative Refinement},
	year = {2011},
	month = {2011-12},
	keywords = {magma},
	author = {Hartwig Anzt and Piotr Luszczek and Jack Dongarra and Vincent Heuveline}
}
@techreport {icl:645,
	title = {Hierarchical QR Factorization Algorithms for Multi-Core Cluster Systems},
	number = {UT-CS-11-684},
	year = {2011},
	month = {2011-10},
	keywords = {magma, plasma},
	author = {Jack Dongarra and Mathieu Faverge and Thomas Herault and Julien Langou and Yves Robert}
}
@article {icl:653,
	title = {A Hybridization Methodology for High-Performance Linear Algebra Software for GPUs},
	journal = {in GPU Computing Gems, Jade Edition},
	volume = {2},
	year = {2011},
	month = {2011-00},
	pages = {473-484},
	publisher = {Elsevier},
	keywords = {magma, morse},
	author = {Emmanuel Agullo and Cedric Augonnet and Jack Dongarra and Hatem Ltaeif and Raymond Namyst and Samuel Thibault and Stanimire Tomov},
	editor = {Wen-mei W. Hwu}
}
@article {icl:643,
	title = {The International Exascale Software Project Roadmap},
	journal = {International Journal of High Performance Computing},
	volume = {25},
	number = {1},
	year = {2011},
	month = {2011-01},
	pages = {3-60},
	abstract = {Over the last 20 years, the open-source community has provided more and more software on which the world{\textquoteright}s high-performance computing systems depend for performance and productivity. The community has invested millions of dollars and years of effort to build key components. However, although the investments in these separate software elements have been tremendously valuable, a great deal of productivity has also been lost because of the lack of planning, coordination, and key integration of technologies necessary to make them work together smoothly and efficiently, both within individual petascale systems and between different systems. It seems clear that this completely uncoordinated development model will not provide the software needed to support the unprecedented parallelism required for peta/ exascale computation on millions of cores, or the flexibility required to exploit new hardware models and features, such as transactional memory, speculative execution, and graphics processing units. This report describes the work of the community to prepare for the challenges of exascale computing, ultimately combing their efforts in a coordinated International Exascale Software Project.},
	doi = {https://doi.org/10.1177/1094342010391989},
	author = {Jack Dongarra and Pete Beckman and Terry Moore and Patrick Aerts and Giovanni Aloisio and Jean-Claude Andre and David Barkai and Jean-Yves Berthou and Taisuke Boku and Bertrand Braunschweig and Franck Cappello and Barbara Chapman and Xuebin Chi and Alok Choudhary and Sudip Dosanjh and Thom Dunning and Sandro Fiore and Al Geist and Bill Gropp and Robert Harrison and Mark Hereld and Michael Heroux and Adolfy Hoisie and Koh Hotta and Zhong Jin and Yutaka Ishikawa and Fred Johnson and Sanjay Kale and Richard Kenway and David Keyes and Bill Kramer and Jesus Labarta and Alain Lichnewsky and Thomas Lippert and Bob Lucas and Barney MacCabe and Satoshi Matsuoka and Paul Messina and Peter Michielse and Bernd Mohr and Matthias S. Mueller and Wolfgang E. Nagel and Hiroshi Nakashima and Michael E. Papka and Dan Reed and Mitsuhisa Sato and Ed Seidel and John Shalf and David Skinner and Marc Snir and Thomas Sterling and Rick Stevens and Fred Streitz and Bob Sugar and Shinji Sumimoto and William Tang and John Taylor and Rajeev Thakur and Anne Trefethen and Mateo Valero and Aad van der Steen and Jeffrey Vetter and Peg Williams and Robert Wisniewski and Kathy Yelick}
}
@article {icl:606,
	title = {Parallel algebraic domain decomposition solver for the solution of augmented systems.},
	journal = {Parallel, Distributed, Grid and Cloud Computing for Engineering, Ajaccio, Corsica, France, 12-15 April},
	year = {2011},
	month = {2011-00},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Jean Roman}
}
@techreport {icl:627,
	title = {Parallel Reduction to Condensed Forms for Symmetric Eigenvalue Problems using Aggregated Fine-Grained and Memory-Aware Kernels},
	year = {2011},
	month = {2011-08},
	author = {Azzam Haidar and Hatem Ltaeif and Jack Dongarra}
}
@inproceedings {icl:657,
	title = {Parallel Reduction to Condensed Forms for Symmetric Eigenvalue Problems using Aggregated Fine-Grained and Memory-Aware Kernels},
	year = {2011},
	month = {2011-11},
	address = {Seattle, WA},
	keywords = {plasma, quark},
	author = {Azzam Haidar and Hatem Ltaeif and Jack Dongarra}
}
@article {icl:636,
	title = {Performance Portability of a GPU Enabled Factorization with the DAGuE Framework},
	journal = {IEEE Cluster: workshop on Parallel Programming on Accelerator Clusters (PPAC)},
	year = {2011},
	month = {2011-06},
	keywords = {dague, magma, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Narapat Ohm Saengpatsa and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {icl:648,
	title = {Process Distance-aware Adaptive MPI Collective Communications},
	year = {2011},
	month = {2011-00},
	address = {Austin, Texas},
	author = {Teng Ma and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {icl:677,
	title = {QCG-OMPI: MPI Applications on Grids.},
	journal = {Future Generation Computer Systems},
	volume = {27},
	number = {4},
	year = {2011},
	month = {2011-01},
	pages = {435-369},
	author = {Emmanuel Agullo and Camille Coti and Thomas Herault and Julien Langou and Sylvain Peyronnet and A. Rezmerita and Franck Cappello and Jack Dongarra}
}
@techreport {icl:612,
	title = {On Scalability for MPI Runtime Systems},
	number = {ICL-UT-11-05},
	year = {2011},
	month = {2011-05},
	address = {Knoxville, TN},
	author = {George Bosilca and Thomas Herault and A. Rezmerita and Jack Dongarra}
}
@inproceedings {icl:671,
	title = {On Scalability for MPI Runtime Systems},
	year = {2011},
	month = {2011-09},
	pages = {187-195},
	publisher = {IEEEE},
	address = {Austin, TX, USA},
	keywords = {harness},
	author = {George Bosilca and Thomas Herault and A. Rezmerita and Jack Dongarra}
}
@inproceedings {icl:674,
	title = {Scalable Runtime for MPI: Efficiently Building the Communication Infrastructure},
	volume = {6960},
	year = {2011},
	month = {2011-09},
	pages = {342-344},
	publisher = {Springer},
	address = {Santorini, Greece},
	keywords = {ftmpi},
	author = {George Bosilca and Thomas Herault and Pierre Lemariner and Jack Dongarra and A. Rezmerita},
	editor = {Yiannis Cotronis and Anthony Danalis and Dimitrios S. Nikolopoulos and Jack Dongarra}
}
@article {icl:603,
	title = {Three-dimensional parallel frequency-domain visco-acoustic wave modelling based on a hybrid direct/iterative solver.},
	journal = {To appear in Geophysical Prospecting journal.},
	year = {2011},
	month = {2011-00},
	author = {Florent Sourbier and Azzam Haidar and Luc Giraud and Hafedh Ben-Hadj-Ali and St{\'e}phane Operto and Jean Virieux}
}
@article {icl:604,
	title = {Toward High Performance Divide and Conquer Eigensolver for Dense Symmetric Matrices.},
	journal = {Submitted to SIAM Journal on Scientific Computing (SISC)},
	year = {2011},
	month = {2011-00},
	author = {Azzam Haidar and Hatem Ltaeif and Jack Dongarra}
}
@inproceedings {icl:593,
	title = {A Unified HPC Environment for Hybrid Manycore/GPU Distributed Systems},
	year = {2011},
	month = {2011-05},
	address = {Anchorage, AK},
	keywords = {dague},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Narapat Ohm Saengpatsa and Stanimire Tomov and Jack Dongarra}
}
@article {icl:533,
	title = {Analysis of Dynamically Scheduled Tile Algorithms for Dense Linear Algebra on Multicore Architectures},
	journal = {Submitted to Concurrency and Computations: Practice and Experience},
	year = {2010},
	month = {2010-11},
	keywords = {plasma, quark},
	author = {Azzam Haidar and Hatem Ltaeif and Asim YarKhan and Jack Dongarra}
}
@article {icl:555,
	title = {Constructing Resiliant Communication Infrastructure for Runtime Environments in Advances in Parallel Computing},
	journal = {Advances in Parallel Computing - Parallel Computing: From Multicores and GPU{\textquoteright}s to Petascale},
	volume = {19},
	year = {2010},
	pages = {441-451},
	doi = {10.3233/978-1-60750-530-3-441},
	author = {George Bosilca and Camille Coti and Thomas Herault and Pierre Lemariner and Jack Dongarra},
	editor = {Barbara Chapman and Frederic Desprez and Gerhard R. Joubert and Alain Lichnewsky and Frans Peters and T. Priol}
}
@techreport {icl:528,
	title = {DAGuE: A generic distributed DAG engine for high performance computing},
	number = {ICL-UT-10-01},
	year = {2010},
	month = {2010-04},
	keywords = {dague},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@techreport {icl:563,
	title = {Distributed Dense Numerical Linear Algebra Algorithms on Massively Parallel Architectures: DPLASMA},
	year = {2010},
	month = {2010-09},
	keywords = {dague, dplasma, parsec, plasma},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@techreport {icl:529,
	title = {Distributed-Memory Task Execution and Dependence Tracking within DAGuE and the DPLASMA Project},
	number = {ICL-UT-10-02},
	year = {2010},
	month = {2010-00},
	keywords = {dague, plasma},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@inproceedings {icl:534,
	title = {Dodging the Cost of Unavoidable Memory Copies in Message Logging Protocols},
	year = {2010},
	month = {2010-09},
	publisher = {Springer},
	address = {Stuttgart, Germany},
	keywords = {ftmpi},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Jack Dongarra},
	editor = {Jack Dongarra and Michael Resch and Rainer Keller and Edgar Gabriel}
}
@article {icl:539,
	title = {Intelligent Service Trading and Brokering for Distributed Network Services in GridSolve},
	journal = {VECPAR 2010, 9th International Meeting on High Performance Computing for Computational Science},
	year = {2010},
	month = {2010-06},
	address = {Berkeley, CA},
	keywords = {gridpac, netsolve},
	author = {Aurlie Hurault and Asim YarKhan}
}
@article {icl:553,
	title = {MaPHyS or the Development of a Parallel Algebraic Domain Decomposition Solver in the Course of the Solstice Project},
	journal = {Sparse Days 2010 Meeting at CERFACS},
	year = {2010},
	month = {2010-06},
	address = {Toulouse, France},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Jean Roman and Yohan Lee-Tin-Yien}
}
@article {icl:574,
	title = {QCG-OMPI: MPI Applications on Grids},
	journal = {Future Generation Computer Systems},
	volume = {27},
	number = {4},
	year = {2010},
	month = {2010-03},
	pages = {357-369},
	author = {Emmanuel Agullo and Camille Coti and Thomas Herault and Julien Langou and Sylvain Peyronnet and A. Rezmerita and Franck Cappello and Jack Dongarra}
}
@inproceedings {icl:532,
	title = {QR Factorization of Tall and Skinny Matrices in a Grid Computing Environment},
	year = {2010},
	month = {2010-04},
	address = {Atlanta, GA},
	author = {Emmanuel Agullo and Camille Coti and Jack Dongarra and Thomas Herault and Julien Langou}
}
@article {icl:554,
	title = {Scalability Study of a Quantum Simulation Code},
	journal = {PARA 2010},
	year = {2010},
	month = {2010-06},
	address = {Reykjavik, Iceland},
	author = {Jerzy Bernholc and Miroslav Hodak and Wenchang Lu and Shirley Moore and Stanimire Tomov}
}
@techreport {icl:530,
	title = {Scalable Tile Communication-Avoiding QR Factorization on Multicore Cluster Systems},
	volume = {{\textendash}10-653},
	year = {2010},
	month = {2010-04},
	keywords = {plasma},
	author = {Fengguang Song and Hatem Ltaeif and Bilel Hadri and Jack Dongarra}
}
@article {icl:559,
	title = {Scalable Tile Communication-Avoiding QR Factorization on Multicore Cluster Systems},
	journal = {SC{\textquoteright}10},
	year = {2010},
	month = {2010-11},
	publisher = {ACM SIGARCH/ IEEE Computer Society},
	address = {New Orleans, LA},
	keywords = {plasma},
	author = {Fengguang Song and Hatem Ltaeif and Bilel Hadri and Jack Dongarra}
}
@article {icl:545,
	title = {Sparse approximations of the Schur complement for parallel algebraic hybrid solvers in 3D},
	journal = {Numerical Mathematics: Theory, Methods and Applications},
	volume = {3},
	number = {3},
	year = {2010},
	month = {2010-00},
	pages = {64-82},
	publisher = {Golbal Science Press},
	address = {Beijing},
	author = {Luc Giraud and Azzam Haidar and Yousef Saad},
	editor = {C. Zhiming}
}
@inproceedings {icl:588,
	title = {Tile QR Factorization with Parallel Panel Processing for Multicore Architectures},
	year = {2010},
	month = {2010-00},
	author = {Bilel Hadri and Emmanuel Agullo and Jack Dongarra}
}
@article {icl:552,
	title = {Towards a Complexity Analysis of Sparse Hybrid Linear Solvers},
	journal = {PARA 2010},
	year = {2010},
	month = {2010-06},
	address = {Reykjavik, Iceland},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Jean Roman}
}
@article {icl:544,
	title = {Using multiple levels of parallelism to enhance the performance of domain decomposition solvers},
	journal = {Parallel Computing},
	volume = {36},
	number = {5-6},
	year = {2010},
	month = {2010-00},
	pages = {285-296},
	publisher = {Elsevier journals},
	author = {Luc Giraud and Azzam Haidar and Stephane Pralet},
	editor = {Costas Bekas and Pascua D{\textquoteright}Ambra and Ananth Grama and Yousef Saad and Petko Yanev}
}
@inproceedings {icl:488,
	title = {Comparative Study of One-Sided Factorizations with Multiple Software Packages on Multi-Core Hardware},
	year = {2009},
	month = {2009-00},
	author = {Emmanuel Agullo and Bilel Hadri and Hatem Ltaeif and Jack Dongarra}
}
@techreport {icl:484,
	title = {Constructing resiliant communication infrastructure for runtime environments},
	number = {ICL-UT-09-02},
	year = {2009},
	month = {2009-07},
	author = {George Bosilca and Camille Coti and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@article {icl:517,
	title = {Constructing Resilient Communication Infrastructure for Runtime Environments},
	journal = {ParCo 2009},
	year = {2009},
	month = {2009-09},
	address = {Lyon France},
	author = {Pierre Lemariner and George Bosilca and Camille Coti and Thomas Herault and Jack Dongarra}
}
@article {icl:524,
	title = {Enhancing Parallelism of Tile QR Factorization for Multicore Architectures},
	journal = {Submitted to Transaction on Parallel and Distributed Systems},
	year = {2009},
	month = {2009-12},
	keywords = {plasma},
	author = {Bilel Hadri and Hatem Ltaeif and Emmanuel Agullo and Jack Dongarra}
}
@inproceedings {icl:602,
	title = {Modeling the Office of Science Ten Year Facilities Plan: The PERI Architecture Tiger Team},
	volume = {180(2009)012039},
	year = {2009},
	month = {2009-07},
	publisher = {IOP Publishing},
	address = {San Diego, California},
	keywords = {test},
	author = {Bronis R. de Supinski and Sadaf Alam and David Bailey and Laura Carrington and Chris Daley and Anshu Dubey and Todd Gamblin and Dan Gunter and Paul D. Hovland and Heike Jagode and Karen Karavanic and Gabriel Marin and John Mellor-Crummey and Shirley Moore and Boyana Norris and Leonid Oliker and Catherine Olschanowsky and Philip C. Roth and Martin Schulz and Sameer Shende and Allan Snavely}
}
@inproceedings {icl:486,
	title = {Numerical Linear Algebra on Emerging Architectures: The PLASMA and MAGMA Projects},
	volume = {180},
	year = {2009},
	month = {2009-00},
	keywords = {magma, plasma},
	author = {Emmanuel Agullo and James Demmel and Jack Dongarra and Bilel Hadri and Jakub Kurzak and Julien Langou and Hatem Ltaeif and Piotr Luszczek and Stanimire Tomov}
}
@article {1352,
	title = {Numerical Linear Algebra on Emerging Architectures: The PLASMA and MAGMA Projects},
	year = {2009},
	month = {2009-11},
	publisher = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC09)},
	address = {Portland, OR},
	author = {Emmanuel Agullo and James Demmel and Jack Dongarra and Bilel Hadri and Jakub Kurzak and Julien Langou and Hatem Ltaeif and Piotr Luszczek and Rajib Nath and Stanimire Tomov and Asim YarKhan and Vasily Volkov}
}
@article {icl:420,
	title = {Reliability and Performance Modeling and Analysis for Grid Computing},
	journal = {in Handbook of Research on Scalable Computing Technologies (to appear)},
	year = {2009},
	month = {2009-00},
	pages = {219-245},
	publisher = {IGI Global},
	author = {Yuan-Shun Dai and Jack Dongarra},
	editor = {Kuan-Ching Li and Ching-Hsien Hsu and Laurence Yang and Jack Dongarra and Hans Zima}
}
@techreport {icl:487,
	title = {Tall and Skinny QR Matrix Factorization Using Tile Algorithms on Multicore Architectures},
	number = {ICL-UT-09-03},
	year = {2009},
	month = {2009-09},
	keywords = {plasma},
	author = {Bilel Hadri and Hatem Ltaeif and Emmanuel Agullo and Jack Dongarra}
}
@inproceedings {icl:522,
	title = {Tile QR Factorization with Parallel Panel Processing for Multicore Architectures},
	year = {2009},
	month = {2009-12},
	address = {Atlanta, GA},
	keywords = {plasma},
	author = {Bilel Hadri and Hatem Ltaeif and Emmanuel Agullo and Jack Dongarra}
}
@article {icl:506,
	title = {Towards Efficient MapReduce Using MPI},
	journal = {Lecture Notes in Computer Science, Recent Advances in Parallel Virtual Machine and Message Passing Interface - 16th European PVM/MPI Users{\textquoteright} Group Meeting},
	volume = {5759},
	year = {2009},
	month = {2009-00},
	pages = {240-249},
	publisher = {Springer Berlin / Heidelberg},
	address = {Espoo, Finland},
	author = {Torsten Hoefler and Yuan-Shun Dai and Jack Dongarra},
	editor = {M. Ropo and J Westerholm and Jack Dongarra}
}
@inproceedings {icl:519,
	title = {VGrADS: Enabling e-Science Workflows on Grids and Clouds with Fault Tolerance},
	year = {2009},
	month = {2009-00},
	address = {Portland, OR},
	keywords = {grads},
	author = {Lavanya Ramakrishan and Daniel Nurmi and Anirban Mandal and Charles Koelbel and Dennis Gannon and Mark Huang and Yang-Suk Kee and Graziano Obertelli and Kiran Thyagaraja and Rich Wolski and Asim YarKhan and Dmitrii Zagorodnov}
}
@inproceedings {icl:463,
	title = {Custom assignment of MPI ranks for parallel multi-dimensional FFTs: Evaluation of BG/P versus BG/L},
	year = {2008},
	month = {2008-01},
	pages = {271-283},
	publisher = {IEEE Computer Society},
	address = {Sydney, Australia},
	author = {Heike Jagode and Joachim Hein}
}
@article {icl:451,
	title = {DARPA{\textquoteright}s HPCS Program: History, Models, Tools, Languages},
	journal = {in Advances in Computers},
	volume = {72},
	year = {2008},
	month = {2008-01},
	publisher = {Elsevier},
	author = {Jack Dongarra and Robert Graybill and William Harrod and Robert Lucas and Ewing Lusk and Piotr Luszczek and Janice McMahon and Allan Snavely and Jeffrey Vetter and Katherine Yelick and Sadaf Alam and Roy Campbell and Laura Carrington and Tzu-Yi Chen and Omid Khalili and Jeremy Meredith and Mustafa Tikir},
	editor = {M. Zelkowitz}
}
@techreport {icl:427,
	title = {HPCS Library Study Effort},
	year = {2008},
	month = {2008-01},
	author = {Jack Dongarra and James Demmel and Parry Husbands and Piotr Luszczek}
}
@article {icl:408,
	title = {Interactive Grid-Access Using Gridsolve and Giggle},
	journal = {Computing and Informatics},
	volume = {27},
	number = {2},
	year = {2008},
	month = {2008-00},
	pages = {233-248,ISSN1335-9150},
	keywords = {netsolve},
	author = {Marcus Hardt and Keith Seymour and Jack Dongarra and Michael Zapf and Nicole Ruiter}
}
@article {icl:417,
	title = {Performance Instrumentation and Compiler Optimizations for MPI/OpenMP Applications},
	journal = {Lecture Notes in Computer Science, OpenMP Shared Memory Parallel Programming},
	volume = {4315},
	year = {2008},
	month = {2008-00},
	publisher = {Springer Berlin / Heidelberg},
	author = {Oscar Hernandez and Fengguang Song and Barbara Chapman and Jack Dongarra and Bernd Mohr and Shirley Moore and Felix Wolf}
}
@article {icl:462,
	title = {PERI Auto-tuning},
	journal = {Proc. SciDAC 2008},
	volume = {125},
	year = {2008},
	month = {2008-01},
	publisher = {Journal of Physics},
	address = {Seatlle, Washington},
	keywords = {gco},
	author = {David Bailey and Jacqueline Chame and Chun Chen and Jack Dongarra and Mary Hall and Jeffrey K. Hollingsworth and Paul D. Hovland and Shirley Moore and Keith Seymour and Jaewook Shin and Ananta Tiwari and Sam Williams and Haihang You}
}
@techreport {icl:411,
	title = {Task placement of parallel multi-dimensional FFTs on a mesh communication network},
	number = {UT-CS-08-613},
	year = {2008},
	month = {2008-01},
	author = {Heike Jagode and Joachim Hein and Arthur Trew}
}
@inproceedings {icl:412,
	title = {Usage of the Scalasca Toolset for Scalable Performance Analysis of Large-scale Parallel Applications},
	year = {2008},
	month = {2008-01},
	pages = {157-167},
	publisher = {Springer},
	address = {Stuttgart, Germany},
	keywords = {point},
	author = {Felix Wolf and Brian Wylie and Erika Abraham and Wolfgang Frings and Karl F{\"u}rlinger and Markus Geimer and Marc-Andre Hermanns and Bernd Mohr and Shirley Moore and Matthias Pfeifer},
	editor = {Michael Resch and Rainer Keller and Valentin Himmler and Bettina Krammer and A Schulz}
}
@inproceedings {icl:388,
	title = {Multithreading for synchronization tolerance in matrix factorization},
	volume = {78},
	number = {2007},
	year = {2007},
	month = {2007-01},
	author = {Alfredo Buttari and Jack Dongarra and Parry Husbands and Jakub Kurzak and Katherine Yelick}
}
@inbook {875,
	title = {A New Approach to MPI Collective Communication Implementations},
	booktitle = {Distributed and Parallel Systems},
	year = {2007},
	pages = {45-54},
	publisher = {Springer US},
	organization = {Springer US},
	abstract = {Recent research into the optimization of collective MPI operations has resulted in a wide variety of algorithms and corresponding implementations, each typically only applicable in a relatively narrow scope: on a specific architecture, on a specific network, with a specific number of processes, with a specific data size and/or data-type {\textendash} or any combination of these (or other) factors. This situation presents an enormous challenge to portable MPI implementations which are expected to provide optimized collective operation performance on all platforms. Many portable implementations have attempted to provide a token number of algorithms that are intended to realize good performance on most systems. However, many platform configurations are still left without well-tuned collective operations. This paper presents a proposal for a framework that will allow a wide variety of collective algorithm implementations and a flexible, multi-tiered selection process for choosing which implementation to use when an application invokes an MPI collective function.},
	keywords = {Automatic Selection, Collective Operation, Framework, Message Passing (MPI), Open MPI},
	isbn = {978-0-387-69857-1},
	doi = {10.1007/978-0-387-69858-8_5},
	author = {Torsten Hoefler and Jeffrey M. Squyres and Graham Fagg and George Bosilca and Wolfgang Rehm and Andrew Lumsdaine}
}
@inproceedings {icl:387,
	title = {Results of the PERI survey of SciDAC applications},
	volume = {78},
	number = {2007},
	year = {2007},
	month = {2007-01},
	author = {Bronis R. de Supinski and Jeffrey K. Hollingsworth and Shirley Moore and Patrick H. Worley}
}
@inproceedings {icl:319,
	title = {Performance Instrumentation and Compiler Optimizations for MPI/OpenMP Applications},
	year = {2006},
	month = {2006-01},
	address = {Reims, France},
	keywords = {kojak},
	author = {Oscar Hernandez and Fengguang Song and Barbara Chapman and Jack Dongarra and Bernd Mohr and Shirley Moore and Felix Wolf}
}
@article {icl:370,
	title = {Prospectus for the Next LAPACK and ScaLAPACK Libraries},
	journal = {PARA 2006},
	year = {2006},
	month = {2006-06},
	address = {Umea, Sweden},
	author = {James Demmel and Jack Dongarra and B. Parlett and William Kahan and Ming Gu and David Bindel and Yozo Hida and Xiaoye Li and Osni Marques and Jason E. Riedy and Christof Voemel and Julien Langou and Piotr Luszczek and Jakub Kurzak and Alfredo Buttari and Julien Langou and Stanimire Tomov}
}
@inproceedings {icl:272,
	title = {Event-based Measurement and Analysis of One-sided Communication},
	year = {2005},
	month = {2005-08},
	publisher = {Springer},
	address = {Lisbon, Portugal},
	keywords = {kojak},
	author = {Marc-Andre Hermanns and Bernd Mohr and Felix Wolf}
}
@article {icl:305,
	title = {Optimization Problem Solving System Using GridRPC},
	journal = {IEEE Transactions on Parallel and Distributed Systems (submitted)},
	year = {2005},
	month = {2005-01},
	author = {Hisashi Shimosaka and Tomoyuki Hiroyasu and Mitsunori Miki and Jack Dongarra}
}
@inproceedings {icl:287,
	title = {Performance Analysis of GYRO: A Tool Evaluation},
	year = {2005},
	month = {2005-06},
	address = {San Francisco, CA},
	keywords = {kojak},
	author = {Patrick H. Worley and Jeff Candy and Laura Carrington and Kevin Huck and Timothy Kaiser and Kumar Mahinthakumar and Allen D. Malony and Shirley Moore and Dan Reed and Philip C. Roth and H. Shan and Sameer Shende and Allan Snavely and S. Sreepathi and Felix Wolf and Y. Zhang}
}
@inproceedings {icl:295,
	title = {Performance Analysis of One-sided Communication Mechanisms},
	number = {ICL-UT-06-07},
	year = {2005},
	month = {2005-09},
	address = {Malaga, Spain},
	keywords = {kojak},
	author = {Bernd Mohr and Andrej K{\"u}hnal and Marc-Andre Hermanns and Felix Wolf}
}
@article {icl:244,
	title = {Self Adaptivity in Grid Computing},
	journal = {Concurrency and Computation: Practice and Experience, Special Issue: Grid Performance},
	volume = {17},
	number = {2-4},
	year = {2005},
	month = {2005-00},
	pages = {235-257},
	keywords = {netsolve, sans},
	author = {Sathish Vadhiyar and Jack Dongarra},
	editor = {John Gurd and Anthony Hey and Juri Papay and Graham Riley}
}
@inproceedings {icl:206,
	title = {Active Logistical State Management in the GridSolve/L},
	year = {2004},
	month = {2004-01},
	address = {Chicago, Illinois},
	keywords = {netsolve},
	author = {Micah Beck and Jack Dongarra and Jian Huang and Terry Moore and James Plank}
}
@article {icl:236,
	title = {Cray X1 Evaluation Status Report},
	journal = {Oak Ridge National Laboratory Report},
	volume = {/-2004/13},
	year = {2004},
	month = {2004-01},
	author = {Pratul Agarwal and R. A. Alexander and E. Apra and Satish Balay and Arthur S. Bland and James Colgan and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Tom Dunigan and Mark Fahey and Al Geist and M. Gordon and Robert Harrison and Dinesh Kaushik and M. Krishnakumar and Piotr Luszczek and Tony Mezzacapa and Jeff Nichols and Jarek Nieplocha and Leonid Oliker and T. Packwood and M. Pindzola and Thomas C. Schulthess and Jeffrey Vetter and James B White and T. Windus and Patrick H. Worley and Thomas Zacharia}
}
@article {icl:247,
	title = {An Overview of Heterogeneous High Performance and Grid Computing},
	journal = {Engineering the Grid (to appear)},
	year = {2004},
	month = {2004-00},
	publisher = {Nova Science Publishers, Inc.},
	author = {Jack Dongarra and Alexey Lastovetsky},
	editor = {Beniamino Di Martino and Jack Dongarra and Adolfy Hoisie and Laurence Yang and Hans Zima}
}
@article {icl:175,
	title = {Distributed Probablistic Model-Building Genetic Algorithm},
	journal = {Lecture Notes in Computer Science},
	volume = {2723},
	year = {2003},
	month = {2003-01},
	pages = {1015-1028},
	publisher = {Springer-Verlag, Heidelberg},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Masaki Sano and Hisashi Shimosaka and Shigeyoshi Tsutsui and Jack Dongarra}
}
@article {icl:131,
	title = {Energy Minimization of Protein Tertiary Structure by Parallel Simulated Annealing using Genetic Crossover},
	journal = {Special Issue on Biological Applications of Genetic and Evolutionary Computation (submitted)},
	year = {2003},
	month = {2003-03},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Shinya Ogura and Keiko Aoi and Takeshi Yoshida and Yuko Okamoto and Jack Dongarra}
}
@inproceedings {icl:172,
	title = {GrADSolve - RPC for High Performance Computing on the Grid},
	volume = {2790},
	year = {2003},
	month = {2003-01},
	pages = {394-403},
	publisher = {Springer-Verlag, Berlin},
	address = {Klagenfurt, Austria},
	keywords = {netsolve},
	doi = {10.1007/978-3-540-45209-6_58},
	author = {Sathish Vadhiyar and Jack Dongarra and Asim YarKhan},
	editor = {Harald Kosch and Laszlo Boszormenyi and Hermann Hellwagner}
}
@article {icl:168,
	title = {High Performance Computing for Computational Science},
	journal = {Lecture Notes in Computer Science},
	volume = {2565},
	year = {2003},
	month = {2003-01},
	publisher = {Springer-Verlag, Berlin},
	address = {VECPAR 2002, 5th International Conference June 26-28, 2002},
	author = {Jose Palma and Jack Dongarra and Vicente Hern{\'a}ndez},
	editor = {Antonio Augusto Sousa}
}
@article {icl:143,
	title = {NetSolve: Past, Present, and Future - A Look at a Grid Enabled Server},
	journal = {Making the Global Infrastructure a Reality},
	year = {2003},
	month = {2003-00},
	publisher = {Wiley Publishing},
	keywords = {netsolve},
	author = {Sudesh Agrawal and Jack Dongarra and Keith Seymour and Sathish Vadhiyar},
	editor = {Francine Berman and Geoffrey Fox and Anthony Hey}
}
@inproceedings {icl:179,
	title = {Optimization of Injection Schedule of Diesel Engine Using GridRPC},
	volume = {2003},
	number = {14},
	year = {2003},
	month = {2003-01},
	pages = {189-197},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Junji Sawada and Jack Dongarra}
}
@inproceedings {icl:128,
	title = {Optimization Problem Solving System using Grid RPC},
	year = {2003},
	month = {2003-03},
	address = {Tokyo, Japan},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Hisashi Shimosaka and Jack Dongarra}
}
@article {icl:205,
	title = {A Parallel Implementation of the Nonsymmetric QR Algorithm for Distributed Memory Architectures},
	journal = {SIAM Journal on Scientific Computing},
	volume = {24},
	number = {1},
	year = {2003},
	month = {2003-01},
	pages = {284-311},
	author = {Greg Henry and David Watkins and Jack Dongarra}
}
@article {icl:139,
	title = {The Semantic Conference Organizer},
	journal = {Statistical Data Mining and Knowledge Discovery},
	year = {2003},
	month = {2003-00},
	publisher = {CRC Press},
	keywords = {netsolve},
	author = {Kevin Heinrich and Michael Berry and Jack Dongarra and Sathish Vadhiyar},
	editor = {Hamparsum Bozdogan}
}
@inproceedings {icl:129,
	title = {A Simple Installation and Administration Tool for Large-scaled PC Cluster System},
	year = {2003},
	month = {2003-03},
	address = {San Jose, CA},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Kenzo Kodama and Junichi Uekawa and Jack Dongarra}
}
@inproceedings {icl:177,
	title = {Static Scheduling for ScaLAPACK on the Grid Using Genetic Algorithm},
	volume = {2003},
	number = {14},
	year = {2003},
	month = {2003-01},
	pages = {3-10},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Hiroki Saito and Yusuke Tanimura and Jack Dongarra}
}
@article {icl:92,
	title = {Active Netlib: An Active Mathematical Software Collection for Inquiry-based Computational Science and Engineering Education},
	journal = {Journal of Digital Information special issue on Interactivity in Digital Libraries},
	volume = {2},
	number = {4},
	year = {2002},
	month = {2002-00},
	keywords = {activenetlib, rib},
	author = {Shirley Moore and A.J. Baker and Jack Dongarra and Christian Halloy and Chung Ng}
}
@article {icl:113,
	title = {A Comparison of Parallel Solvers for General Narrow Banded Linear Systems},
	journal = {Parallel and Distributed Computing Practices},
	volume = {2},
	year = {2002},
	month = {2002-10},
	pages = {385-400},
	author = {Peter Arbenz and Andrew Cleary and Jack Dongarra and Markus Hegland}
}
@article {icl:104,
	title = {Optimization System Using Grid RPC},
	journal = {Meeting of the Japan Society of Mechanical Engineers},
	year = {2002},
	month = {2002-10},
	address = {Kyoto University, Kyoto, Japan},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Hisashi Shimosaka and Yusuke Tanimura and Jack Dongarra}
}
@article {icl:114,
	title = {A Parallel Implementation of the Nonsymmetric QR Algorithm for Disitributed Memory Architectures},
	journal = {SIAM Journal on Scientific Computing},
	volume = {16},
	number = {2},
	year = {2002},
	month = {2002-10},
	pages = {284-311},
	author = {Greg Henry and David Watkins and Jack Dongarra}
}
@article {icl:103,
	title = {Truss Structural Optimization Using NetSolve System},
	journal = {Meeting of the Japan Society of Mechanical Engineers},
	year = {2002},
	month = {2002-10},
	address = {Kyoto University, Kyoto, Japan},
	keywords = {netsolve},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Hisashi Shimosaka and Masaki Sano and Yusuke Tanimura and Yasunari Mimura and Shinobu Yoshimura and Jack Dongarra}
}
@article {icl:125,
	title = {An Updated Set of Basic Linear Algebra Subprograms (BLAS)},
	journal = {ACM Transactions on Mathematical Software},
	volume = {28},
	number = {2},
	year = {2002},
	month = {2002-12},
	pages = {135-151},
	doi = {10.1145/567806.567807},
	author = {Susan Blackford and James Demmel and Jack Dongarra and Iain Duff and Sven Hammarling and Greg Henry and Michael Heroux and Linda Kaufman and Andrew Lumsdaine and Antoine Petitet and Roldan Pozo and Karin Remington and Clint Whaley}
}
@article {icl:6,
	title = {Basic Linear Algebra Subprograms (BLAS)},
	journal = {(an update), submitted to ACM TOMS},
	year = {2001},
	month = {2001-02},
	author = {Susan Blackford and James Demmel and Jack Dongarra and Iain Duff and Sven Hammarling and Greg Henry and Michael Heroux and Linda Kaufman and Andrew Lumsdaine and Antoine Petitet and Roldan Pozo and Karin Remington and Clint Whaley}
}
@article {icl:118,
	title = {Message Passing Software Systems},
	journal = {Encyclopedia of Electrical and Engineering, Supplement 1},
	year = {2000},
	month = {2000-00},
	publisher = {John Wiley \& Sons, Inc.},
	keywords = {ftmpi},
	author = {Jack Dongarra and Graham Fagg and Rolf Hempel and David W. Walker},
	editor = {J. Webster}
}
@article {icl:31,
	title = {A Portable Programming Interface for Performance Evaluation on Modern Processors},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {14},
	number = {3},
	year = {2000},
	month = {2000-09},
	pages = {189-204},
	keywords = {papi},
	doi = {https://doi.org/10.1177/109434200001400303},
	author = {Shirley Browne and Jack Dongarra and Nathan Garner and George Ho and Phil Mucci}
}
@techreport {icl:51,
	title = {A Comparison of Parallel Solvers for Diagonally Dominant and General Narrow Banded Linear Systems II (LAPACK Working Note 143)},
	number = {UT-CS-99-415},
	year = {1999},
	month = {1999-01},
	author = {Peter Arbenz and Andrew Cleary and Jack Dongarra and Markus Hegland}
}
@techreport {icl:52,
	title = {A Comparison of Parallel Solvers for General Narrow Banded Linear Systems (LAPACK Working Note 142)},
	number = {UT-CS-99-414},
	year = {1999},
	month = {1999-01},
	author = {Peter Arbenz and Andrew Cleary and Jack Dongarra and Markus Hegland}
}
@article {icl:50,
	title = {LAPACK Users{\textquoteright} Guide, 3rd ed.},
	journal = {Philadelphia: Society for Industrial and Applied Mathematics},
	year = {1999},
	month = {1999-01},
	author = {Ed Anderson and Zhaojun Bai and Christian Bischof and Susan Blackford and James Demmel and Jack Dongarra and Jeremy Du Croz and Anne Greenbaum and Sven Hammarling and Alan McKenney and Danny Sorensen}
}
@inproceedings {icl:59,
	title = {PAPI: A Portable Interface to Hardware Performance Counters},
	year = {1999},
	month = {1999-06},
	keywords = {papi},
	author = {Shirley Browne and Christine Deane and George Ho and Phil Mucci}
}
@book {1469,
	title = {MPI - The Complete Reference, Volume 1: The MPI Core},
	year = {1998},
	month = {1998-08},
	pages = {426},
	publisher = {MIT Press},
	organization = {MIT Press},
	edition = {Second},
	address = {Cambridge, MA, USA},
	abstract = {Since its release in summer 1994, the Message Passing Interface (MPI) specification has become a standard for message-passing libraries for parallel computations. There exist more than a dozen implementations on a variety of computing platforms, from the IBM SP-2 supercomputer to PCs running Windows NT. The initial MPI Standard, known as MPI-1, has been modified over the last two years. This volume, the definitive reference manual for the latest version of MPI-1, contains a complete specification of the MPI Standard. It is annotated with comments that clarify complicated issues, including why certain design choices were made, how users are intended to use the interface, and how they should construct their version of MPI. The volume also provides many detailed, illustrative programming examples.},
	isbn = {978-0-262-69215-1},
	author = {Marc Snir and Steve Otto and Steven Huss-Lederman and David Walker and Jack Dongarra}
}
@article {icl:256,
	title = {National HPCC Software Exchange (NHSE): Uniting the High Performance Computing and Communications Community},
	journal = {D-Lib Magazine},
	year = {1998},
	month = {1998-01},
	keywords = {rib},
	author = {Shirley Browne and Jack Dongarra and Jeff Horner and Paul McMahan and Scott Wells}
}
