@conference {,
	title = {AI Benchmarking for Science: Efforts from the MLCommons Science Working Group},
	booktitle = {Lecture Notes in Computer Science},
	volume = {13387},
	year = {2023},
	month = {2023-01},
	pages = {47 - 64},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {With machine learning (ML) becoming a transformative tool for science, the scientific community needs a clear catalogue of ML techniques, and their relative benefits on various scientific problems, if they were to make significant advances in science using AI. Although this comes under the purview of benchmarking, conventional benchmarking initiatives are focused on performance, and as such, science, often becomes a secondary criteria.

In this paper, we describe a community effort from a working group, namely, MLCommons Science Working Group, in developing science-specific AI benchmarking for the international scientific community. Since the inception of the working group in 2020, the group has worked very collaboratively with a number of national laboratories, academic institutions and industries, across the world, and has developed four science-specific AI benchmarks. We will describe the overall process, the resulting benchmarks along with some initial results. We foresee that this initiative is likely to be very transformative for the AI for Science, and for performance-focused communities.},
	isbn = {978-3-031-23219-0},
	doi = {10.1007/978-3-031-23220-610.1007/978-3-031-23220-6_4},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-23220-6_4},
	author = {Thiyagalingam, Jeyan and von Laszewski, Gregor and Yin, Junqi and Emani, Murali and Papay, Juri and Barrett, Gregg and Luszczek, Piotr and Tsaris, Aristeidis and Kirkpatrick, Christine and Wang, Feiyi and Gibbs, Tom and Vishwanath, Venkatram and Shankar, Mallikarjun and Fox, Geoffrey and Hey, Tony},
	editor = {Anzt, Hartwig and Bienz, Amanda and Luszczek, Piotr and Baboulin, Marc}
}
@conference {,
	title = {Mixed Precision Algebraic Multigrid on GPUs},
	booktitle = { Parallel Processing and Applied Mathematics (PPAM 2022)},
	volume = {13826},
	year = {2023},
	month = {2023-04},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Cham},
	abstract = {In this paper, we present the first GPU-native platform-portable algebraic multigrid (AMG) implementation that allows the user to use different precision formats for the distinct multigrid levels. The AMG we present uses an aggregation size 2 parallel graph match as the AMG coarsening strategy. The implementation provides a high level of flexibility in terms of configuring the bottom-level solver and the precision format for the distinct levels. We present convergence and performance results on the GPUs from AMD, Intel, and NVIDIA, and compare against corresponding functionality available in other libraries.},
	keywords = {Algebraic multigrid, GPUs, mixed precision, Portability},
	isbn = {978-3-031-30441-5},
	doi = {10.1007/978-3-031-30442-2_9},
	url = {https://link.springer.com/10.1007/978-3-031-30442-2},
	author = {Tsai, Yu-Hsiang Mike and Natalie Beams and Anzt, Hartwig},
	editor = {Wyrzykowski, Roman and Dongarra, Jack and Deelman, Ewa and Karczewski, Konrad}
}
@conference {,
	title = {PAQR: Pivoting Avoiding QR factorization},
	booktitle = {2023 IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2023},
	publisher = {IEEE},
	organization = {IEEE},
	address = {St. Petersburg, FL, USA},
	doi = {10.1109/IPDPS54959.2023.00040},
	url = {https://ieeexplore.ieee.org/document/10177407/},
	author = {Sid-Lakhdar, Wissam and Cayrols, Sebastien and Bielich, Daniel and Abdelfattah, Ahmad and Luszczek, Piotr and Gates, Mark and Tomov, Stanimire and Johansen, Hans and Williams-Young, David and Davis, Timothy and Dongarra, Jack and Anzt, Hartwig}
}
@article {,
	title = {Evaluating Data Redistribution in PaRSEC},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {33},
	number = {8},
	year = {2022},
	month = {2022-08},
	pages = {1856-1872},
	doi = {10.1109/TPDS.2021.3131657},
	author = {Qinglei Cao and George Bosilca and Losada, Nuria and Wu, Wei and Zhong, Dong and Jack Dongarra}
}
@article {,
	title = {Evaluations of molecular modeling and machine learning for predictive capabilities in binding of lanthanum and actinium with carboxylic acids},
	journal = {Journal of Radioanalytical and Nuclear Chemistry},
	year = {2022},
	month = {2022-12},
	issn = {0236-5731},
	doi = {10.1007/s10967-022-08620-7},
	url = {https://rdcu.be/c2lGj},
	author = {Penchoff, Deborah A. and Peterson, Charles C. and Wrancher, Eleigha M. and George Bosilca and Harrison, Robert J. and Valeev, Edward F. and Benny, Paul D.}
}
@article {,
	title = {Extending MAGMA Portability with OneAPI},
	year = {2022},
	month = {2022-11},
	publisher = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC22), ACM Student Research Competition},
	address = {Dallas, TX},
	url = {https://sc22.supercomputing.org/proceedings/src_poster/poster_files/spostu105s3-file1.pdf},
	author = {Anna Fortenberry and Stanimire Tomov and Kwai Wong}
}
@inproceedings {,
	title = {{Integrating process, control-flow, and data resiliency layers using a hybrid Fenix/Kokkos approach}},
	journal = {2022 IEEE International Conference on Cluster Computing (CLUSTER 2022)},
	year = {2022},
	month = {2022-09},
	address = {Heidelberg, Germany},
	keywords = {checkpointing, Fault tolerance, Fenix, HPC, Kokkos, MPI-ULFM, resilience},
	url = {https://hal.archives-ouvertes.fr/hal-03772536},
	author = {Whitlock, Matthew and Morales, Nicolas and George Bosilca and Bouteiller, Aur{\'e}lien and Nicolae, Bogdan and Teranishi, Keita and Giem, Elisabeth and Sarkar, Vivek}
}
@techreport {,
	title = {PAQR: Pivoting Avoiding QR factorization},
	journal = {ICL Technical Report},
	number = {ICL-UT-22-06},
	year = {2022},
	month = {2022-06},
	abstract = {The solution of linear least-squares problems is at the heart of many scientific and engineering applications. While any method able to minimize the backward error of such problems is considered numerically stable, the theory states that the forward error depends on the condition number of the
matrix in the system of equations. On the one hand, the QR factorization is an efficient method to solve such problems, but the solutions it produces may have large forward errors when the matrix is deficient. On the other hand, QR with column pivoting (QRCP) is able to produce smaller forward errors on deficient matrices, but its cost is prohibitive compared to QR. The aim of this paper is to propose PAQR, an alternative solution method with the same cost (or smaller) as QR and as accurate as QRCP in practical cases, for the solution of rank-deficient linear least-squares problems. After presenting the algorithm and its implementations on different architectures, we compare its accuracy and performance results on a variety of application problems.
},
	author = {Wissam M. Sid-Lakhdar and Sebastien Cayrols and Daniel Bielich and Ahmad Abdelfattah and Piotr Luszczek and Mark Gates and Stanimire Tomov and Hans Johansen and David Williams-Young and Timothy A. Davis and Jack Dongarra}
}
@conference {,
	title = {A Python Library for Matrix Algebra on GPU and Multicore Architectures},
	booktitle = {2022 IEEE 19th International Conference on Mobile Ad Hoc and Smart Systems (MASS)},
	year = {2022},
	month = {2022-12},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Denver, CO},
	doi = {10.1109/MASS56207.2022.00121},
	url = {https://ieeexplore.ieee.org/document/9973474/},
	author = {Nance, Delario and Stanimire Tomov and Wong, Kwai}
}
@article {,
	title = {Resiliency in numerical algorithm design for extreme scale simulations},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {36371337212766180823},
	year = {2022},
	month = {2022-03},
	pages = {251 - 285},
	keywords = {Fault tolerance, Numerical algorithms, parallel computer architecture, resilience},
	issn = {1094-3420},
	doi = {10.1177/10943420211055188},
	url = {http://journals.sagepub.com/doi/10.1177/10943420211055188http://journals.sagepub.com/doi/pdf/10.1177/10943420211055188http://journals.sagepub.com/doi/pdf/10.1177/10943420211055188http://journals.sagepub.com/doi/full-xml/10.1177/10943420211055188},
	author = {Agullo, Emmanuel and Altenbernd, Mirco and Anzt, Hartwig and Bautista-Gomez, Leonardo and Benacchio, Tommaso and Bonaventura, Luca and Bungartz, Hans-Joachim and Chatterjee, Sanjay and Ciorba, Florina M and DeBardeleben, Nathan and Drzisga, Daniel and Eibl, Sebastian and Engelmann, Christian and Gansterer, Wilfried N and Giraud, Luc and G{\"o}ddeke, Dominik and Heisig, Marco and J{\'e}z{\'e}quel, Fabienne and Kohl, Nils and Li, Xiaoye Sherry and Lion, Romain and Mehl, Miriam and Mycek, Paul and Obersteiner, Michael and Quintana-Ort{\'\i}, Enrique S and Rizzi, Francesco and R{\"u}de, Ulrich and Schulz, Martin and Fung, Fred and Speck, Robert and Stals, Linda and Teranishi, Keita and Thibault, Samuel and Th{\"o}nnes, Dominik and Wagner, Andreas and Wohlmuth, Barbara}
}
@article {,
	title = {Efficient exascale discretizations: High-order finite element methods},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2021},
	pages = {10943420211020803},
	abstract = {Efficient exploitation of exascale architectures requires rethinking of the numerical algorithms used in many large-scale applications. These architectures favor algorithms that expose ultra fine-grain parallelism and maximize the ratio of floating point operations to energy intensive data movement. One of the few viable approaches to achieve high efficiency in the area of PDE discretizations on unstructured grids is to use matrix-free/partially assembled high-order finite element methods, since these methods can increase the accuracy and/or lower the computational time due to reduced data motion. In this paper we provide an overview of the research and development activities in the Center for Efficient Exascale Discretizations (CEED), a co-design center in the Exascale Computing Project that is focused on the development of next-generation discretization software and algorithms to enable a wide range of finite element applications to run efficiently on future hardware. CEED is a research partnership involving more than 30 computational scientists from two US national labs and five universities, including members of the Nek5000, MFEM, MAGMA and PETSc projects. We discuss the CEED co-design activities based on targeted benchmarks, miniapps and discretization libraries and our work on performance optimizations for large-scale GPU architectures. We also provide a broad overview of research and development activities in areas such as unstructured adaptive mesh refinement algorithms, matrix-free linear solvers, high-order data visualization, and list examples of collaborations with several ECP and external applications.},
	keywords = {co-design, high-order discretizations, High-performance computing, PDEs, unstructured grids},
	doi = {10.1177/10943420211020803},
	author = {Kolev, Tzanio and Fischer, Paul and Min, Misun and Jack Dongarra and Brown, Jed and Dobrev, Veselin and Warburton, Tim and Stanimire Tomov and Shephard, Mark S and Abdelfattah, Ahmad and others}
}
@inbook {,
	title = {An Introduction to High Performance Computing and Its Intersection with Advances in Modeling Rare Earth Elements and Actinides},
	booktitle = {Rare Earth Elements and Actinides: Progress in Computational Science Applications},
	volume = {1388},
	year = {2021},
	month = {2021-10},
	pages = {3-53},
	publisher = {American Chemical Society},
	organization = {American Chemical Society},
	chapter = {1},
	address = {Washington, DC},
	abstract = {Computationally driven solutions in nuclear and radiochemistry heavily depend on efficient modeling of Rare Earth Elements (REEs) and actinides. Accurate modeling of REEs and actinides faces challenges stemming from limitations from an imbalanced hardware-software ecosystem and its implications on inefficient use of High Performance Computing (HPC). This chapter provides a historical perspective on the evolution of HPC hardware, its intersectionality with domain sciences, the importance of benchmarks for performance, and an overview of challenges and advances in modeling REEs and actinides. This chapter intends to provide an introduction for researchers at the intersection of scientific computing, software development for HPC, and applied computational modeling of REEs and actinides. The chapter is structured in five sections. First, the Introduction includes subsections focusing on the Importance of REEs and Actinides (1.1), Hardware, Software, and the HPC Ecosystem (1.2), and Electronic Structure Modeling of REEs and Actinides (1.3). Second, a section in High Performance Computing focuses on the TOP500 (2.1), HPC Performance (2.2), HPC Benchmarks: Processing, Bandwidth, and Latency (2.3), and HPC Benchmarks and their Relationship to Chemical Modeling (2.4). Third, the Software Challenges and Advances focus on NWChem/NWChemEx (3.1), MADNESS (3.2), and MPQC (3.3). The fourth section provides a short overview of Artificial Intelligence in HPC applications relevant to nuclear and radiochemistry. The fifth section illustrates A Protocol to Evaluate Complexation Preferences in Separations of REEs and Actinides through Computational Modeling.},
	keywords = {actinide, Computational modeling, HPC, REE},
	isbn = {ISBN13: 9780841298255 eISBN: 9780841298248},
	doi = {10.1021/bk-2021-1388.ch001},
	url = {https://pubs.acs.org/doi/10.1021/bk-2021-1388.ch001},
	author = {Deborah A. Penchoff and Edward Valeev and Heike Jagode and Piotr Luszczek and Anthony Danalis and George Bosilca and Robert J. Harrison and Jack Dongarra and Theresa L. Windus}
}
@article {,
	title = {Linear Algebra Prepara.on for Emergent Neural Network Architectures: MAGMA, BLAS, and Batched GPU Computing},
	year = {2021},
	month = {2021-11},
	publisher = {LAPENNA Workshop},
	address = {Virtual},
	author = {Stanimire Tomov and Kwai Wong and Rocco Febbo and Julian Halloy}
}
@book {,
	title = {Rare Earth Elements and Actinides: Progress in Computational Science Applications},
	series = {ACS Symposium Series},
	volume = {1388},
	year = {2021},
	month = {2021-10},
	publisher = {American Chemical Society},
	organization = {American Chemical Society},
	address = {Washington, DC},
	abstract = {Rare earth elements (REEs) and actinides are critical to electronics, communication, military applications, and green energy systems. They also play a large role in nuclear waste challenges with critical national importance. Actinides are still among some of the least studied elements in the periodic table, due to their short half-lives and radioactivity, which demand expert facilities for research. Computational modeling greatly aids in understanding REEs and actinides; however, electronic structure modeling of these elements presents limitations. High Performance Computing (HPC) has had a direct impact not only on technical advances and access to information on a global scale but also on investigations of REEs and actinides. This work discusses recent advances in molecular and data driven modeling that are essential to the study of REEs and actinides, effects of computational science in nuclear and radiochemical applications, and advances and challenges in the exascale era of supercomputing.},
	keywords = {actinides, HPC, REEs},
	isbn = {ISBN13: ‍9780841298255 eISBN: ‍9780841298248},
	doi = {DOI: 10.1021/bk-2021-1388},
	url = {https://pubs.acs.org/doi/book/10.1021/bk-2021-1388},
	author = {Deborah A. Penchoff and Theresa L. Windus and Charles C. Peterson}
}
@inbook {,
	title = {Rare Earth Elements and Critical Materials: Uses and Availability},
	booktitle = {Rare Earth Elements and Actinides: Progress in Computational Science Applications},
	volume = {1388},
	year = {2021},
	month = {2021-10},
	pages = {63-74},
	publisher = {American Chemical Society},
	organization = {American Chemical Society},
	chapter = {3},
	address = {Washington, DC},
	abstract = {Rare Earth Elements (REEs) are essential elements in critical materials for the economy and national security. This chapter explores REEs{\textquoteright} classifications, importance, abundance, resources and challenges, and the need for solutions and alternatives. REEs{\textquoteright} production, reserves, world markets, and net import reliance is discussed. Applications of REEs to clean energy, electronics, and defense are highlighted.},
	keywords = {critical materials, REE},
	isbn = {ISBN13: 9780841298255 eISBN: 9780841298248},
	doi = {10.1021/bk-2021-1388.ch003},
	url = {https://pubs.acs.org/doi/10.1021/bk-2021-1388.ch003},
	author = {Deborah A. Penchoff and Charles B. Sims and Theresa L. Windus}
}
@techreport {,
	title = {CEED ECP Milestone Report: Improve Performance and Capabilities of CEED-Enabled ECP Applications on Summit/Sierra},
	journal = {ECP Milestone Reports},
	year = {2020},
	month = {2020-05},
	publisher = {Zenodo},
	doi = {https://doi.org/10.5281/zenodo.3860804},
	url = {https://doi.org/10.5281/zenodo.3860804},
	author = {Kolev, Tzanio and Fischer, Paul and Abdelfattah, Ahmad and Ananthan, Shreyas and Valeria Barra and Natalie Beams and Bleile, Ryan and Brown, Jed and Carson, Robert and Camier, Jean-Sylvain and Churchfield, Matthew and Dobrev, Veselin and Jack Dongarra and Dudouit, Yohann and Karakus, Ali and Kerkemeier, Stefan and Lan, YuHsiang and Medina, David and Merzari, Elia and Min, Misun and Parker, Scott and Ratnayaka, Thilina and Smith, Cameron and Sprague, Michael and Stitt, Thomas and Thompson, Jeremy and Tomboulides, Ananias and Stanimire Tomov and Tomov, Vladimir and Vargas, Arturo and Warburton, Tim and Weiss, Kenneth}
}
@conference {,
	title = {DeepFreeze: Towards Scalable Asynchronous Checkpointing of Deep Learning Models},
	booktitle = {20th IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing (CCGRID)},
	year = {2020},
	month = {2020-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Melbourne, VIC, Australia},
	abstract = {In the age of big data, deep learning has emerged as a powerful tool to extract insight and exploit its value, both in industry and scientific applications. One common pattern emerging in such applications is frequent checkpointing of the state of the learning model during training, needed in a variety of scenarios: analysis of intermediate states to explain features and correlations with training data, exploration strategies involving alternative models that share a common ancestor, knowledge transfer, resilience, etc. However, with increasing size of the learning models and popularity of distributed data-parallel training approaches, simple checkpointing techniques used so far face several limitations: low serialization performance, blocking I/O, stragglers due to the fact that only a single process is involved in checkpointing. This paper proposes a checkpointing technique specifically designed to address the aforementioned limitations, introducing efficient asynchronous techniques to hide the overhead of serialization and I/O, and distribute the load over all participating processes. Experiments with two deep learning applications (CANDLE and ResNet) on a pre-Exascale HPC platform (Theta) shows significant improvement over state-of-art, both in terms of checkpointing duration and runtime overhead.},
	doi = {https://doi.org/10.1109/CCGrid49817.2020.00-76},
	author = {Bogdan Nicolae and Jiali Li and Justin M. Wozniak and George Bosilca and Matthieu Dorier and Franck Cappello}
}
@conference {,
	title = {FFT-Based Gradient Sparsification for the Distributed Training of Deep Neural Networks},
	booktitle = {9th International Symposium on High-Performance Parallel and Distributed Computing (HPDC 20)},
	year = {2020},
	month = {2020-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Stockholm, Sweden},
	abstract = {The performance and efficiency of distributed training of Deep Neural Networks (DNN) highly depend on the performance of gradient averaging among participating processes, a step bound by communication costs. There are two major approaches to reduce communication overhead: overlap communications with computations (lossless), or reduce communications (lossy). The lossless solution works well for linear neural architectures, e.g. VGG, AlexNet, but more recent networks such as ResNet and Inception limit the opportunity for such overlapping. Therefore, approaches that reduce the amount of data (lossy) become more suitable. In this paper, we present a novel, explainable lossy method that sparsifies gradients in the frequency domain, in addition to a new range-based float point representation to quantize and further compress gradients. These dynamic techniques strike a balance between compression ratio, accuracy, and computational overhead, and are optimized to maximize performance in heterogeneous environments.

Unlike existing works that strive for a higher compression ratio, we stress the robustness of our methods, and provide guidance to recover accuracy from failures. To achieve this, we prove how the FFT sparsification affects the convergence and accuracy, and show that our method is guaranteed to converge using a diminishing θ in training. Reducing θ can also be used to recover accuracy from the failure. Compared to STOA lossy methods, e.g., QSGD, TernGrad, and Top-k sparsification, our approach incurs less approximation error, thereby better in both the wall-time and accuracy. On an 8 GPUs, InfiniBand interconnected cluster, our techniques effectively accelerate AlexNet training up to 2.26x to the baseline of no compression, and 1.31x to QSGD, 1.25x to Terngrad and 1.47x to Top-K sparsification.},
	keywords = {FFT, Gradient Compression, Loosy Gradients, Machine Learning, Neural Networks},
	doi = {https://doi.org/10.1145/3369583.3392681},
	author = {Linnan Wang and Wei Wu and Junyu Zhang and Hang Liu and George Bosilca and Maurice Herlihy and Rodrigo Fonseca}
}
@conference {,
	title = {Flexible Data Redistribution in a Task-Based Runtime System},
	booktitle = {IEEE International Conference on Cluster Computing (Cluster 2020)},
	year = {2020},
	month = {2020-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Kobe, Japan},
	abstract = {Data redistribution aims to reshuffle data to optimize some objective for an algorithm. The objective can be multi-dimensional, such as improving computational load balance or decreasing communication volume or cost, with the ultimate goal to increase the efficiency and therefore decrease the time-to-solution for the algorithm. The classical redistribution problem focuses on optimally scheduling communications when reshuffling data between two regular, usually block-cyclic, data distributions. Recently, task-based runtime systems have gained popularity as a potential candidate to address the programming complexity on the way to exascale. In addition to an increase in portability against complex hardware and software systems, task-based runtime systems have the potential to be able to more easily cope with less-regular data distribution, providing a more balanced computational load during the lifetime of the execution. In this scenario, it becomes paramount to develop a general redistribution algorithm for task-based runtime systems, which could support all types of regular and irregular data distributions. In this paper, we detail a flexible redistribution algorithm, capable of dealing with redistribution problems without constraints of data distribution and data size and implement it in a task-based runtime system, PaRSEC. Performance results show great capability compared to ScaLAPACK, and applications highlight an increased efficiency with little overhead in terms of data distribution and data size.},
	doi = {https://doi.org/10.1109/CLUSTER49012.2020.00032},
	author = {Qinglei Cao and George Bosilca and Wei Wu and Dong Zhong and Aurelien Bouteiller and Jack Dongarra}
}
@conference {,
	title = {HAN: A Hierarchical AutotuNed Collective Communication Framework},
	booktitle = {IEEE Cluster Conference},
	year = {2020},
	month = {2020-09},
	publisher = {Best Paper Award, IEEE Computer Society Press},
	organization = {Best Paper Award, IEEE Computer Society Press},
	address = {Kobe, Japan},
	abstract = {High-performance computing (HPC) systems keep growing in scale and heterogeneity to satisfy the increasing computational need, and this brings new challenges to the design of MPI libraries, especially with regard to collective operations. To address these challenges, we present "HAN," a new hierarchical autotuned collective communication framework in Open MPI, which selects suitable homogeneous collective communication modules as submodules for each hardware level, uses collective operations from the submodules as tasks, and organizes these tasks to perform efficient hierarchical collective operations. With a task-based design, HAN can easily swap out submodules, while keeping tasks intact, to adapt to new hardware. This makes HAN suitable for the current platform and provides a strong and flexible support for future HPC systems. To provide a fast and accurate autotuning mechanism, we present a novel cost model based on benchmarking the tasks instead of a whole collective operation. This method drastically reduces tuning time, as the cost of tasks can be reused across different message sizes, and is more accurate than existing cost models. Our cost analysis suggests the autotuning component can find the optimal configuration in most cases. The evaluation of the HAN framework suggests our design significantly improves the default Open MPI and achieves decent speedups against state-of-the-art MPI implementations on tested applications.},
	author = {Xi Luo and Wei Wu and George Bosilca and Yu Pei and Qinglei Cao and Thananon Patinyasakdikul and Dong Zhong and Jack Dongarra}
}
@article {,
	title = {How to Build Your Own Deep Neural Network},
	year = {2020},
	month = {2020-07},
	publisher = {PEARC20},
	keywords = {AI, Deep Neural Networks, dense linear algebra, HPC, ML},
	author = {Kwai Wong and Stanimire Tomov and Daniel Nichols and Rocco Febbo and Florent Lopez and Julian Halloy and Xianfeng Ma}
}
@article {,
	title = {Integrating Deep Learning in Domain Science at Exascale (MagmaDNN)},
	year = {2020},
	month = {2020-12},
	publisher = {DOD HPCMP seminar},
	address = {virtual},
	abstract = {We will present some of the current challenges in the design and integration of deep learning AI with traditional HPC simulations. We evaluate existing packages for readiness to run efficiently deep learning models and applications on large scale HPC systems, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and up-coming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated in MagmaDNN, an open source HPC deep learning framework. 
Many deep learning frameworks are targeted towards data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how these can be provided, e.g., as in MagmaDNN, through a deep integration with existing HPC libraries such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced and mixed-precision and asynchronous optimization methods. Finally, we present illustrations and potential solutions on enhancing traditional compute and data intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated on materials science, imaging, and climate applications.},
	author = {Stanimire Tomov and Kwai Wong and Jack Dongarra and Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Junqi Yin}
}
@techreport {,
	title = {Integrating Deep Learning in Domain Sciences at Exascale},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-20-10},
	year = {2020},
	month = {2020-08},
	publisher = {University of Tennessee},
	abstract = {This paper presents some of the current challenges in designing deep learning artificial intelligence (AI) and integrating it with traditional high-performance computing (HPC) simulations. We evaluate existing packages for their ability to run deep learning models and applications on large-scale HPC systems e ciently, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and upcoming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated into MagmaDNN, an open-source HPC deep learning framework. Many deep learning frameworks are targeted at data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how those needs can be provided (e.g., as in MagmaDNN) through a deep integration with existing HPC libraries, such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced- and mixed-precision, as well as asynchronous optimization methods. Finally, we present illustrations and potential solutions for enhancing traditional compute- and data-intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated in materials science, imaging, and climate applications.},
	author = {Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Stanimire Tomov and Kwai Wong and Junqi Yin}
}
@conference {,
	title = {Integrating Deep Learning in Domain Sciences at Exascale},
	booktitle = {2020 Smoky Mountains Computational Sciences and Engineering Conference (SMC 2020)},
	year = {2020},
	month = {2020-08},
	abstract = {This paper presents some of the current challenges in designing deep learning artificial intelligence (AI) and integrating it with traditional high-performance computing (HPC) simulations. We evaluate existing packages for their ability to run deep learning models and applications on large-scale HPC systems e ciently, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and upcoming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated into MagmaDNN, an open-source HPC deep learning framework. Many deep learning frameworks are targeted at data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how those needs can be provided (e.g., as in MagmaDNN) through a deep integration with existing HPC libraries, such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced- and mixed-precision, as well as asynchronous optimization methods. Finally, we present illustrations and potential solutions for enhancing traditional compute- and data-intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated in materials science, imaging, and climate applications.},
	author = {Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Stanimire Tomov and Kwai Wong and Junqi Yin}
}
@article {,
	title = {Load-Balancing Sparse Matrix Vector Product Kernels on GPUs},
	journal = {ACM Transactions on Parallel Computing},
	volume = {7},
	year = {2020},
	month = {2020-03},
	abstract = {Efficient processing of Irregular Matrices on Single Instruction, Multiple Data (SIMD)-type architectures is a persistent challenge. Resolving it requires innovations in the development of data formats, computational techniques, and implementations that strike a balance between thread divergence, which is inherent for Irregular Matrices, and padding, which alleviates the performance-detrimental thread divergence but introduces artificial overheads. To this end, in this article, we address the challenge of designing high performance sparse matrix-vector product (SpMV) kernels designed for Nvidia Graphics Processing Units (GPUs). We present a compressed sparse row (CSR) format suitable for unbalanced matrices. We also provide a load-balancing kernel for the coordinate (COO) matrix format and extend it to a hybrid algorithm that stores part of the matrix in SIMD-friendly Ellpack format (ELL) format. The ratio between the ELL- and the COO-part is determined using a theoretical analysis of the nonzeros-per-row distribution. For the over 2,800 test matrices available in the Suite Sparse matrix collection, we compare the performance against SpMV kernels provided by NVIDIA{\textquoteright}s cuSPARSE library and a heavily-tuned sliced ELL (SELL-P) kernel that prevents unnecessary padding by considering the irregular matrices as a combination of matrix blocks stored in ELL format.},
	doi = {https://doi.org/10.1145/3380930},
	author = {Hartwig Anzt and Terry Cojean and Chen Yen-Chen and Jack Dongarra and Goran Flegar and Pratik Nayak and Stanimire Tomov and Yuhsiang M. Tsai and Weichung Wang}
}
@book {,
	title = {Parallel Processing and Applied Mathematics: 13th International Conference, PPAM 2019, Bialystok, Poland, September 8{\textendash}11, 2019, Revised Selected Papers, Part II},
	series = {Lecture Notes in Computer Science},
	number = {12044},
	year = {2020},
	month = {2020-03},
	pages = {503},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	isbn = {978-3-030-43222-5},
	doi = {https://doi.org/10.1007/978-3-030-43222-5},
	author = {Roman Wyrzykowski and Ewa Deelman and Jack Dongarra and Konrad Karczewski}
}
@book {,
	title = {Parallel Processing and Applied Mathematics: 13th International Conference, PPAM 2019, Bialystok, Poland, September 8{\textendash}11, 2019, Revised Selected Papers, Part I},
	series = { Lecture Notes in Computer Science},
	number = {12043},
	year = {2020},
	month = {2020-03},
	pages = {581},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	edition = {1},
	isbn = {978-3-030-43229-4},
	doi = {https://doi.org/10.1007/978-3-030-43229-4},
	author = {Roman Wyrzykowski and Ewa Deelman and Jack Dongarra and Konrad Karczewski}
}
@article {,
	title = {Performance Application Programming Interface for Extreme-Scale Environments (PAPI-EX) (Poster)},
	year = {2020},
	month = {2020-20},
	publisher = {2020 NSF Cyberinfrastructure for Sustained Scientific Innovation (CSSI) Principal Investigator Meeting},
	address = {Seattle, WA},
	author = {Jack Dongarra and Heike Jagode and Anthony Danalis and Daniel Barry and Vince Weaver}
}
@article {1456,
	title = {Project-Based Research and Training in High Performance Data Sciences, Data Analytics, and Machine Learning},
	journal = {The Journal of Computational Science Education},
	volume = {11},
	year = {2020},
	month = {2020-01},
	pages = {36-44},
	doi = {https://doi.org/10.22369/issn.2153-4136/11/1/7},
	url = {http://www.jocse.org/articles/11/1/7/},
	author = {Wong, Kwai and Stanimire Tomov and Jack Dongarra}
}
@techreport {1462,
	title = {Redesigning PAPI{\textquoteright}s High-Level API},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-20-03},
	year = {2020},
	month = {2020-02},
	publisher = {University of Tennessee},
	abstract = {PAPI (Performance Application Programming Interface) provides a portable and efficient API to access the hardware performance counters found on modern microprocessors. With the introduction of Component PAPI or PAPI-C in early 2010 PAPI has extended its reach beyond the CPU and can now monitor system information across a range of components from CPUs to network cards, graphics accelerator cards, parallel file systems and more. To collect performance events, PAPI provides two APIs, the low-level and high-level API. The legacy high-level API was designed for simplicity, but could only handle preset CPU events. To access events from all installed components, the programmer had to use the low-level API. This paper introduces a new high-level API that enables the measurement of both preset and native events. It is intended for programmers who want to perform simple event measurements with minimal code instrumentation.},
	author = {Frank Winkler}
}
@article {1464,
	title = {SLATE Tutorial},
	year = {2020},
	month = {2020-02},
	publisher = {2020 ECP Annual Meeting},
	address = {Houston, TX},
	author = {Mark Gates and Jakub Kurzak and Asim YarKhan and Ali Charara and Jamie Finney and Dalal Sukkari and Mohammed Al Farhan and Ichitaro Yamazaki and Panruo Wu and Jack Dongarra}
}
@conference {,
	title = {Task Bench: A Parameterized Benchmark for Evaluating Parallel Runtime Performance},
	booktitle = {International Conference for High Performance Computing Networking, Storage, and Analysis (SC20)},
	year = {2020},
	month = {2020-11},
	publisher = {ACM},
	organization = {ACM},
	abstract = {We present Task Bench, a parameterized benchmark designed to explore the performance of distributed programming systems under a variety of application scenarios. Task Bench dramatically lowers the barrier to benchmarking and comparing multiple programming systems by making the implementation for a given system orthogonal to the benchmarks themselves: every benchmark constructed with Task Bench runs on every Task Bench implementation. Furthermore, Task Bench{\textquoteright}s parameterization enables a wide variety of benchmark scenarios that distill the key characteristics of larger applications.

To assess the effectiveness and overheads of the tested systems, we introduce a novel metric, minimum effective task granularity (METG). We conduct a comprehensive study with 15 programming systems on up to 256 Haswell nodes of the Cori supercomputer. Running at scale, 100μs-long tasks are the finest granularity that any system runs efficiently with current technologies. We also study each system{\textquoteright}s scalability, ability to hide communication and mitigate load imbalance.},
	url = {https://dl.acm.org/doi/10.5555/3433701.3433783},
	author = {Elliott Slaughter and Wei Wu and Yuankun Fu and Legend Brandenburg and Nicolai Garcia and Wilhem Kautz and Emily Marx and Kaleb S. Morris and Qinglei Cao and George Bosilca and Seema Mirchandaney and Wonchan Lee and Sean Treichler and Patrick McCormick and Alex Aiken}
}
@techreport {1433,
	title = {CEED ECP Milestone Report: Performance Tuning of CEED Software and 1st and 2nd Wave Apps},
	year = {2019},
	month = {2019-10},
	publisher = {Zenodo},
	doi = {https://doi.org/10.5281/zenodo.3477618},
	author = {Stanimire Tomov and Ahmad Abdelfattah and Valeria Barra and Natalie Beams and Jed Brown and Jean-Sylvain Camier and Veselin Dobrev and Jack Dongarra and Yohann Dudouit and Paul Fischer and Ali Karakus and Stefan Kerkemeier and Tzanio Kolev and YuHsiang Lan and Elia Merzari and Misun Min and Aleks Obabko and Scott Parker and Thilina Ratnayaka and Jeremy Thompson and Ananias Tomboulides and Vladimir Tomov and Tim Warburton}
}
@techreport {1434,
	title = {CEED ECP Milestone Report: Public release of CEED 2.0},
	year = {2019},
	month = {2019-04},
	publisher = {Zenodo},
	doi = {10.5281/zenodo.2641316},
	url = {https://doi.org/10.5281/zenodo.2641316},
	author = {Jed Brown and Ahmad Abdelfattah and Valeria Barra and Veselin Dobrev and Yohann Dudouit and Paul Fischer and Tzanio Kolev and David Medina and Misun Min and Thilina Ratnayaka and Cameron Smith and Jeremy Thompson and Stanimire Tomov and Vladimir Tomov and Tim Warburton}
}
@techreport {1398,
	title = {A Collection of Presentations from the BDEC2 Workshop in Kobe, Japan},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-09},
	year = {2019},
	month = {2019-02},
	publisher = {University of Tennessee, Knoxville},
	author = {Rosa M. Badia and Micah Beck and Fran{\c c}ois Bodin and Taisuke Boku and Franck Cappello and Alok Choudhary and Carlos Costa and Ewa Deelman and Nicola Ferrier and Katsuki Fujisawa and Kohei Fujita and Maria Girone and Geoffrey Fox and Shantenu Jha and Yoshinari Kameda and Christian Kniep and William Kramer and James Lin and Kengo Nakajima and Yiwei Qiu and Kishore Ramachandran and Glenn Ricart and Kim Serradell and Dan Stanzione and Lin Gan and Martin Swany and Christine Sweeney and Alex Szalay and Christine Kirkpatrick and Kenton McHenry and Alainna White and Steve Tuecke and Ian Foster and Joe Mambretti and William. M Tang and Michela Taufer and Miguel V{\'a}zquez}
}
@techreport {1399,
	title = {A Collection of White Papers from the BDEC2 Workshop in Poznan, Poland},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-10},
	year = {2019},
	month = {2019-05},
	publisher = {University of Tennessee, Knoxville},
	author = {Gabriel Antoniu and Alexandru Costan and Ovidiu Marcu and Maria S. P{\'e}rez and Nenad Stojanovic and Rosa M. Badia and Miguel V{\'a}zquez and Sergi Girona and Micah Beck and Terry Moore and Piotr Luszczek and Ezra Kissel and Martin Swany and Geoffrey Fox and Vibhatha Abeykoon and Selahattin Akkas and Kannan Govindarajan and Gurhan Gunduz and Supun Kamburugamuve and Niranda Perera and Ahmet Uyar and Pulasthi Wickramasinghe and Chathura Widanage and Maria Girone and Toshihiro Hanawa and Richard Moreno and Ariel Oleksiak and Martin Swany and Ryousei Takano and M.P. van Haarlem and J. van Leeuwen and J.B.R. Oonk and T. Shimwell and L.V.E. Koopmans}
}
@techreport {1408,
	title = {A Collection of White Papers from the BDEC2 Workshop in San Diego, CA},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-13},
	year = {2019},
	month = {2019-10},
	publisher = {University of Tennessee},
	author = {Ilkay Altintas and Kyle Marcus and Volkan Vural and Shweta Purawat and Daniel Crawl and Gabriel Antoniu and Alexandru Costan and Ovidiu Marcu and Prasanna Balaprakash and Rongqiang Cao and Yangang Wang and Franck Cappello and Robert Underwood and Sheng Di and Justin M. Wozniak and Jon C. Calhoun and Cong Xu and Antonio Lain and Paolo Faraboschi and Nic Dube and Dejan Milojicic and Balazs Gerofi and Maria Girone and Viktor Khristenko and Tony Hey and Erza Kissel and Yu Liu and Richard Loft and Pekka Manninen and Sebastian von Alfthan and Takemasa Miyoshi and Bruno Raffin and Olivier Richard and Denis Trystram and Maryam Rahnemoonfar and Robin Murphy and Joel Saltz and Kentaro Sano and Rupak Roy and Kento Sato and Jian Guo and Jen s Domke and Weikuan Yu and Takaki Hatsui and Yasumasa Joti and Alex Szalay and William M. Tang and Michael R. Wyatt II and Michela Taufer and Todd Gamblin and Stephen Herbein and Adam Moody and Dong H. Ahn and Rich Wolski and Chandra Krintz and Fatih Bakir and Wei-tsung Lin and Gareth George}
}
@conference {1325,
	title = {Hands-on Research and Training in High-Performance Data Sciences, Data Analytics, and Machine Learning for Emerging Environments},
	booktitle = {ISC High Performance},
	year = {2019},
	month = {2019-06},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Frankfurt, Germany},
	author = {Kwai Wong and Stanimire Tomov and Jack Dongarra}
}
@article {1366,
	title = {MagmaDNN 0.2 High-Performance Data Analytics for Manycore GPUs and CPUs},
	year = {2019},
	month = {2019-01},
	publisher = {University of Tennessee},
	doi = {10.13140/RG.2.2.14906.64961},
	author = {Lucien Ng and Sihan Chen and Alex Gessinger and Daniel Nichols and Sophia Cheng and Anu Meenasorna and Kwai Wong and Stanimire Tomov and Azzam Haidar and Eduardo D{\textquoteright}Azevedo and Jack Dongarra}
}
@conference {1326,
	title = {MagmaDNN: Accelerated Deep Learning Using MAGMA},
	booktitle = {Practice and Experience in Advanced Research Computing (PEARC {\textquoteright}19)},
	year = {2019},
	month = {2019-07},
	publisher = {ACM},
	organization = {ACM},
	address = {Chicago, IL},
	author = {Daniel Nichols and Kwai Wong and Stanimire Tomov and Lucien Ng and Sihan Chen and Alex Gessinger}
}
@conference {1324,
	title = {MagmaDNN: Towards High-Performance Data Analytics and Machine Learning for Data-Driven Scientific Computing},
	booktitle = {ISC High Performance},
	year = {2019},
	month = {2019-06},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Frankfurt, Germany},
	abstract = {In this paper, we present work towards the development of a new data analytics and machine learning (ML) framework, called MagmaDNN. Our main goal is to provide scalable, high-performance data analytics and ML solutions for scientific applications running on current and upcoming heterogeneous many-core GPU-accelerated architectures. To this end, since many of the functionalities needed are based on standard linear algebra (LA) routines, we designed MagmaDNN to derive its performance power from the MAGMA library. The close integration provides the fundamental (scalable high-performance) LA routines available in MAGMA as a backend to MagmaDNN. We present some design issues for performance and scalability that are specific to ML using Deep Neural Networks (DNN), as well as the MagmaDNN designs towards overcoming them. In particular, MagmaDNN uses well established HPC techniques from the area of dense LA, including task-based parallelization, DAG representations, scheduling, mixed-precision algorithms, asynchronous solvers, and autotuned hyperparameter optimization. We illustrate these techniques and their incorporation and use to outperform other frameworks, currently available.},
	doi = {https://doi.org/10.1007/978-3-030-34356-9_37},
	author = {Daniel Nichols and Natalie-Sofia Tomov and Frank Betancourt and Stanimire Tomov and Kwai Wong and Jack Dongarra}
}
@conference {1327,
	title = {OpenDIEL: A Parallel Workflow Engine and DataAnalytics Framework},
	booktitle = {Practice and Experience in Advanced Research Computing (PEARC {\textquoteright}19)},
	year = {2019},
	month = {2019-07},
	publisher = {ACM},
	organization = {ACM},
	address = {Chicago, IL},
	author = {Frank Betancourt and Kwai Wong and Efosa Asemota and Quindell Marshall and Daniel Nichols and Stanimire Tomov}
}
@article {1270,
	title = {PLASMA: Parallel Linear Algebra Software for Multicore Using OpenMP},
	journal = {ACM Transactions on Mathematical Software},
	volume = {45},
	year = {2019},
	month = {2019-06},
	doi = {https://doi.org/10.1145/3264491},
	author = {Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Maksims Abalenkovs and Negin Bagherpour and Sven Hammarling and Jakub Sistek}
}
@conference {1318,
	title = {Towards Continuous Benchmarking},
	booktitle = {Platform for Advanced Scientific Computing Conference (PASC 2019)},
	year = {2019},
	month = {2019-06},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Zurich, Switzerland},
	abstract = {We present an automated performance evaluation framework that enables an automated workflow for testing and performance evaluation of software libraries. Integrating this component into an ecosystem enables sustainable software development, as a community effort, via a web application for interactively evaluating the performance of individual software components. The performance evaluation tool is based exclusively on web technologies, which removes the burden of downloading performance data or installing additional software. We employ this framework for the Ginkgo software ecosystem, but the framework can be used with essentially any software project, including the comparison between different software libraries. The Continuous Integration (CI) framework of Ginkgo is also extended to automatically run a benchmark suite on predetermined HPC systems, store the state of the machine and the environment along with the compiled binaries, and collect results in a publicly accessible performance data repository based on Git. The Ginkgo performance explorer (GPE) can be used to retrieve the performance data from the repository, and visualizes it in a web browser. GPE also implements an interface that allows users to write scripts, archived in a Git repository, to extract particular data, compute particular metrics, and visualize them in many different formats (as specified by the script). The combination of these approaches creates a workflow which enables performance reproducibility and software sustainability of scientific software. In this paper, we present example scripts that extract and visualize performance data for Ginkgo{\textquoteright}s SpMV kernels that allow users to identify the optimal kernel for specific problem characteristics.},
	isbn = {9781450367707},
	doi = {https://doi.org/10.1145/3324989.3325719},
	author = {Hartwig Anzt and Yen Chen Chen and Terry Cojean and Jack Dongarra and Goran Flegar and Pratik Nayak and Enrique S. Quintana-Orti and Yuhsiang M. Tsai and Weichung Wang}
}
@conference {,
	title = {Understanding Scalability and Fine-Grain Parallelism of Synchronous Data Parallel Training},
	booktitle = {2019 IEEE/ACM Workshop on Machine Learning in High Performance Computing Environments (MLHPC)},
	year = {2019},
	month = {2019-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Denver, CO},
	abstract = {In the age of big data, deep learning has emerged as a powerful tool to extract insight and exploit its value, both in industry and scientific applications. With increasing complexity of learning models and amounts of training data, data-parallel approaches based on frequent all-reduce synchronization steps are increasingly popular. Despite the fact that high-performance computing (HPC) technologies have been designed to address such patterns efficiently, the behavior of data-parallel approaches on HPC platforms is not well understood. To address this issue, in this paper we study the behavior of Horovod, a popular data-parallel approach that relies on MPI, on Theta, a pre-Exascale machine at Argonne National Laboratory. Using two representative applications, we explore two aspects: (1) how performance and scalability is affected by important parameters such as number of nodes, number of workers, threads per node, batch size; (2) how computational phases are interleaved withall-reduce communication phases at fine granularity and what consequences this interleaving has in terms of potential bottlenecks. Our findings show that pipelining of back-propagation, gradient reduction and weight updates mitigate the effects of stragglers during all-reduce only partially. Furthermore, there can be significant delays between weights update, which can be leveraged to mask the overhead of additional background operations that are coupled with the training.},
	doi = {https://doi.org/10.1109/MLHPC49564.2019.00006},
	author = {Jiali Li and Bogdan Nicolae and Justin M. Wozniak and George Bosilca}
}
@article {1272,
	title = {The 30th Anniversary of the Supercomputing Conference: Bringing the Future Closer{\textemdash}Supercomputing History and the Immortality of Now},
	journal = {Computer},
	volume = {51},
	year = {2018},
	month = {2018-11},
	pages = {74{\textendash}85},
	abstract = {A panel of experts{\textemdash}including Gordon Bell, Jack Dongarra, William E. (Bill) Johnston, Horst Simon, Erich Strohmaier, and Mateo Valero{\textemdash}discuss historical reflections on the past 30 years of the Supercomputing (SC) conference, its leading role for the professional community and some exciting future challenges.},
	keywords = {High-performance computing, history of computing, SC, Scientific computing, supercomputing, Virtual Roundtable},
	doi = {10.1109/MC.2018.3971352},
	author = {Jack Dongarra and Vladimir Getov and Kevin Walsh}
}
@article {1331,
	title = {Accelerating 2D FFT: Exploit GPU Tensor Cores through Mixed-Precision},
	year = {2018},
	month = {2018-11},
	publisher = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC18), ACM Student Research Poster},
	address = {Dallas, TX},
	author = {Xaiohe Cheng and Anumeena Soma and Eduardo D{\textquoteright}Azevedo and Kwai Wong and Stanimire Tomov}
}
@conference {1205,
	title = {ADAPT: An Event-Based Adaptive Collective Communication Framework},
	booktitle = {The 27th International Symposium on High-Performance Parallel and Distributed Computing (HPDC {\textquoteright}18)},
	year = {2018},
	month = {2018-06},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Tempe, Arizona},
	abstract = {The increase in scale and heterogeneity of high-performance computing (HPC) systems predispose the performance of Message Passing Interface (MPI) collective communications to be susceptible to noise, and to adapt to a complex mix of hardware capabilities. The designs of state of the art MPI collectives heavily rely on synchronizations; these designs magnify noise across the participating processes, resulting in significant performance slowdown. Therefore, such design philosophy must be reconsidered to efficiently and robustly run on the large-scale heterogeneous platforms. In this paper, we present ADAPT, a new collective communication framework in Open MPI, using event-driven techniques to morph collective algorithms to heterogeneous environments. The core concept of ADAPT is to relax synchronizations, while mamtaining the minimal data dependencies of MPI collectives. To fully exploit the different bandwidths of data movement lanes in heterogeneous systems, we extend the ADAPT collective framework with a topology-aware communication tree. This removes the boundaries of different hardware topologies while maximizing the speed of data movements. We evaluate our framework with two popular collective operations: broadcast and reduce on both CPU and GPU clusters. Our results demonstrate drastic performance improvements and a strong resistance against noise compared to other state of the art MPI libraries. In particular, we demonstrate at least 1.3X and 1.5X speedup for CPU data and 2X and 10X speedup for GPU data using ADAPT event-based broadcast and reduce operations.},
	isbn = {9781450357852},
	doi = {10.1145/3208040.3208054},
	author = {Xi Luo and Wei Wu and George Bosilca and Thananon Patinyasakdikul and Linnan Wang and Jack Dongarra}
}
@article {1211,
	title = {Big Data and Extreme-Scale Computing: Pathways to Convergence - Toward a Shaping Strategy for a Future Software and Data Ecosystem for Scientific Inquiry},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	year = {2018},
	month = {2018-07},
	pages = {435{\textendash}479},
	abstract = {Over the past four years, the Big Data and Exascale Computing (BDEC) project organized a series of five international workshops that aimed to explore the ways in which the new forms of data-centric discovery introduced by the ongoing revolution in high-end data analysis (HDA) might be integrated with the established, simulation-centric paradigm of the high-performance computing (HPC) community. Based on those meetings, we argue that the rapid proliferation of digital data generators, the unprecedented growth in the volume and diversity of the data they generate, and the intense evolution of the methods for analyzing and using that data are radically reshaping the landscape of scientific computing. The most critical problems involve the logistics of wide-area, multistage workflows that will move back and forth across the computing continuum, between the multitude of distributed sensors, instruments and other devices at the networks edge, and the centralized resources of commercial clouds and HPC centers. We suggest that the prospects for the future integration of technological infrastructures and research ecosystems need to be considered at three different levels. First, we discuss the convergence of research applications and workflows that establish a research paradigm that combines both HPC and HDA, where ongoing progress is already motivating efforts at the other two levels. Second, we offer an account of some of the problems involved with creating a converged infrastructure for peripheral environments, that is, a shared infrastructure that can be deployed throughout the network in a scalable manner to meet the highly diverse requirements for processing, communication, and buffering/storage of massive data workflows of many different scientific domains. Third, we focus on some opportunities for software ecosystem convergence in big, logically centralized facilities that execute large-scale simulations and models and/or perform large-scale data analytics. We close by offering some conclusions and recommendations for future investment and policy review.},
	doi = {https://doi.org/10.1177/1094342018778123},
	author = {Mark Asch and Terry Moore and Rosa M. Badia and Micah Beck and Pete Beckman and Thierry Bidot and Fran{\c c}ois Bodin and Franck Cappello and Alok Choudhary and Bronis R. de Supinski and Ewa Deelman and Jack Dongarra and Anshu Dubey and Geoffrey Fox and Haohuan Fu and Sergi Girona and Michael Heroux and Yutaka Ishikawa and Kate Keahey and David Keyes and William T. Kramer and Jean-Fran{\c c}ois Lavignon and Yutong Lu and Satoshi Matsuoka and Bernd Mohr and St{\'e}phane Requena and Joel Saltz and Thomas Schulthess and Rick Stevens and Martin Swany and Alexander Szalay and William Tang and Ga{\"e}l Varoquaux and Jean-Pierre Vilotte and Robert W. Wisniewski and Zhiwei Xu and Igor Zacharov}
}
@techreport {1397,
	title = {A Collection of White Papers from the BDEC2 Workshop in Bloomington, IN},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-15},
	year = {2018},
	month = {2018-11},
	publisher = {University of Tennessee, Knoxville},
	author = {James Ahrens and Christopher M. Biwer and Alexandru Costan and Gabriel Antoniu and Maria S. P{\'e}rez and Nenad Stojanovic and Rosa Badia and Oliver Beckstein and Geoffrey Fox and Shantenu Jha and Micah Beck and Terry Moore and Sunita Chandrasekaran and Carlos Costa and Thierry Deutsch and Luigi Genovese and Tarek El-Ghazawi and Ian Foster and Dennis Gannon and Toshihiro Hanawa and Tevfik Kosar and William Kramer and Madhav V. Marathe and Christopher L. Barrett and Takemasa Miyoshi and Alex Pothen and Ariful Azad and Judy Qiu and Bo Peng and Ravi Teja and Sahil Tyagi and Chathura Widanage and Jon Koskey and Maryam Rahnemoonfar and Umakishore Ramachandran and Miles Deegan and William Tang and Osamu Tatebe and Michela Taufer and Michel Cuende and Ewa Deelman and Trilce Estrada and Rafael Ferreira Da Silva and Harrel Weinstein and Rodrigo Vargas and Miwako Tsuji and Kevin G. Yager and Wanling Gao and Jianfeng Zhan and Lei Wang and Chunjie Luo and Daoyi Zheng and Xu Wen and Rui Ren and Chen Zheng and Xiwen He and Hainan Ye and Haoning Tang and Zheng Cao and Shujie Zhang and Jiahui Dai}
}
@inproceedings {1259,
	title = {The Design of Fast and Energy-Efficient Linear Solvers: On the Potential of Half-Precision Arithmetic and Iterative Refinement Techniques},
	journal = {International Conference on Computational Science (ICCS 2018)},
	volume = {10860},
	year = {2018},
	month = {2018-06},
	pages = {586{\textendash}600},
	publisher = {Springer},
	address = {Wuxi, China},
	abstract = {As parallel computers approach exascale, power efficiency in high-performance computing (HPC) systems is of increasing concern. Exploiting both the hardware features and algorithms is an effective solution to achieve power efficiency, and to address the energy constraints in modern and future HPC systems. In this work, we present a novel design and implementation of an energy-efficient solution for dense linear systems of equations, which are at the heart of large-scale HPC applications. The proposed energy-efficient linear system solvers are based on two main components: (1) iterative refinement techniques, and (2) reduced-precision computing features in modern accelerators and coprocessors. While most of the energy efficiency approaches aim to reduce the consumption with a minimal performance penalty, our method improves both the performance and the energy efficiency. Compared to highly-optimized linear system solvers, our kernels deliver the same accuracy solution up to   2{\texttimes}  faster and reduce the energy consumption up to half on Intel Knights Landing (KNL) architectures. By efficiently using the Tensor Cores available in the NVIDIA V100 PCIe GPUs, the speedups can be up to   4{\texttimes} , with more than 80\% reduction in the energy consumption.},
	doi = {https://doi.org/10.1007/978-3-319-93698-7_45},
	url = {https://rdcu.be/bcKSC},
	author = {Azzam Haidar and Ahmad Abdelfattah and Mawussi Zounon and Panruo Wu and Srikara Pranesh and Stanimire Tomov and Jack Dongarra}
}
@article {1219,
	title = {Optimization and Performance Evaluation of the IDR Iterative Krylov Solver on GPUs},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	number = {2},
	year = {2018},
	month = {2018-03},
	pages = {220{\textendash}230},
	abstract = {In this paper, we present an optimized GPU implementation for the induced dimension reduction algorithm. We improve data locality, combine it with an efficient sparse matrix vector kernel, and investigate the potential of overlapping computation with communication as well as the possibility of concurrent kernel execution. A comprehensive performance evaluation is conducted using a suitable performance model. The analysis reveals efficiency of up to 90\%, which indicates that the implementation achieves performance close to the theoretically attainable bound.},
	keywords = {co-design, gpu, Induced dimension reduction (IDR), kernel fusion, kernel overlap, roofline performance model},
	doi = {https://doi.org/10.1177/1094342016646844},
	author = {Hartwig Anzt and Moritz Kreutzer and Eduardo Ponce and Gregory D. Peterson and Gerhard Wellein and Jack Dongarra}
}
@techreport {1191,
	title = {Parallel BLAS Performance Report},
	journal = {SLATE Working Notes},
	number = {05, ICL-UT-18-01},
	year = {2018},
	month = {2018-04},
	publisher = {University of Tennessee},
	author = {Jakub Kurzak and Mark Gates and Asim YarKhan and Ichitaro Yamazaki and Panruo Wu and Piotr Luszczek and Jamie Finney and Jack Dongarra}
}
@article {1189,
	title = {Symmetric Indefinite Linear Solver using OpenMP Task on Multicore Architectures},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {29},
	year = {2018},
	month = {2018-08},
	pages = {1879{\textendash}1892},
	abstract = {Recently, the Open Multi-Processing (OpenMP) standard has incorporated task-based programming, where a function call with input and output data is treated as a task. At run time, OpenMP{\textquoteright}s superscalar scheduler tracks the data dependencies among the tasks and executes the tasks as their dependencies are resolved. On a shared-memory architecture with multiple cores, the independent tasks are executed on different cores in parallel, thereby enabling parallel execution of a seemingly sequential code. With the emergence of many-core architectures, this type of programming paradigm is gaining attention-not only because of its simplicity, but also because it breaks the artificial synchronization points of the program and improves its thread-level parallelization. In this paper, we use these new OpenMP features to develop a portable high-performance implementation of a dense symmetric indefinite linear solver. Obtaining high performance from this kind of solver is a challenge because the symmetric pivoting, which is required to maintain numerical stability, leads to data dependencies that prevent us from using some common performance-improving techniques. To fully utilize a large number of cores through tasking, while conforming to the OpenMP standard, we describe several techniques. Our performance results on current many-core architectures-including Intel{\textquoteright}s Broadwell, Intel{\textquoteright}s Knights Landing, IBM{\textquoteright}s Power8, and Arm{\textquoteright}s ARMv8-demonstrate the portable and superior performance of our implementation compared with the Linear Algebra PACKage (LAPACK). The resulting solver is now available as a part of the PLASMA software package.},
	keywords = {linear algebra, multithreading, runtime, symmetric indefinite matrices},
	doi = {10.1109/TPDS.2018.2808964},
	author = {Ichitaro Yamazaki and Jakub Kurzak and Panruo Wu and Mawussi Zounon and Jack Dongarra}
}
@techreport {1175,
	title = {C++ API for Batch BLAS},
	journal = {SLATE Working Notes},
	number = {04, ICL-UT-17-12},
	year = {2017},
	month = {2017-12},
	publisher = {University of Tennessee},
	author = {Ahmad Abdelfattah and Konstantin Arturov and Cris Cecka and Jack Dongarra and Chip Freitag and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Panruo Wu}
}
@techreport {1138,
	title = {The Case for Directive Programming for Accelerator Autotuner Optimization},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-07},
	year = {2017},
	month = {2017-10},
	publisher = {University of Tennessee},
	abstract = {In this work, we present the use of compiler pragma directives for parallelizing autotuning of specialized compute kernels for hardware accelerators. A set of constructs, that include prallelizing a source code that prune a generated search space with a large number of constraints for an autotunning infrastructure. For a better performance we studied optimization aimed at minimization of the run time.We also studied the behavior of the parallel load balance and the speedup on four different machines: x86, Xeon Phi, ARMv8, and POWER8.},
	author = {Diana Fayad and Jakub Kurzak and Piotr Luszczek and Panruo Wu and Jack Dongarra}
}
@techreport {1133,
	title = {Designing SLATE: Software for Linear Algebra Targeting Exascale},
	journal = {SLATE Working Notes},
	number = {03, ICL-UT-17-06},
	year = {2017},
	month = {2017-10},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	type = {SLATE Working Notes},
	author = {Jakub Kurzak and Panruo Wu and Mark Gates and Ichitaro Yamazaki and Piotr Luszczek and Gerald Ragghianti and Jack Dongarra}
}
@conference {1100,
	title = {Efficient Communications in Training Large Scale Neural Networks},
	booktitle = {ACM MultiMedia Workshop 2017},
	year = {2017},
	month = {2017-10},
	publisher = {ACM},
	organization = {ACM},
	address = {Mountain View, CA},
	abstract = {We consider the problem of how to reduce the cost of communication that is required for the parallel training of a neural network. The state-of-the-art method, Bulk Synchronous Parallel Stochastic Gradient Descent (BSP-SGD), requires many collective communication operations, like broadcasts of parameters or reductions for sub-gradient aggregations, which for large messages quickly dominates overall execution time and limits parallel scalability. To address this problem, we develop a new technique for collective operations, referred to as Linear Pipelining (LP). It is tuned to the message sizes that arise in BSP-SGD, and works effectively on multi-GPU systems. Theoretically, the cost of LP is invariant to P, where P is the number of GPUs, while the cost of more conventional Minimum Spanning Tree (MST) scales like O(logP). LP also demonstrate up to 2x faster bandwidth than Bidirectional Exchange (BE) techniques that are widely adopted by current MPI implementations. We apply these collectives to BSP-SGD, showing that the proposed implementations reduce communication bottlenecks in practice while preserving the attractive convergence properties of BSP-SGD.},
	author = {Yiyang Zhao and Linnan Wan and Wei Wu and George Bosilca and Richard Vuduc and Jinmian Ye and Wenqi Tang and Zenglin Xu}
}
@conference {1140,
	title = {Investigating Half Precision Arithmetic to Accelerate Dense Linear System Solvers},
	booktitle = {ScalA17: 8th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2017},
	month = {11/2017},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	abstract = {The use of low-precision arithmetic in mixed-precision computing methods has been a powerful tool to accelerate numerous scientific computing applications. Artificial intelligence (AI) in particular has pushed this to current extremes, making use of half-precision floating-point arithmetic (FP16) in approaches based on neural networks. The appeal of FP16 is in the high performance that can be achieved using it on today{\textquoteright}s powerful manycore GPU accelerators, e.g., like the NVIDIA V100, that can provide 120 TeraFLOPS alone in FP16. We present an investigation showing that other HPC applications can harness this power too, and in particular, the general HPC problem of solving Ax = b, where A is a large dense matrix, and the solution is needed in FP32 or FP64 accuracy. Our approach is based on the mixed-precision iterative refinement technique {\textendash} we generalize and extend prior advances into a framework, for which we develop architecture-specific algorithms and highly-tuned implementations that resolve the main computational challenges of efficiently parallelizing, scaling, and using FP16 arithmetic in the approach on high-end GPUs. Subsequently, we show for a first time how the use of FP16 arithmetic can significantly accelerate, as well as make more energy efficient, FP32 or FP64-precision Ax = b solvers. Our results are reproducible and the developments will be made available through the MAGMA library. We quantify in practice the performance, and limitations of the approach.},
	author = {Azzam Haidar and Panruo Wu and Stanimire Tomov and Jack Dongarra}
}
@article {1166,
	title = {A Look Back on 30 Years of the Gordon Bell Prize},
	journal = {International Journal of High Performance Computing and Networking},
	volume = {31},
	year = {2017},
	pages = {469{\textendash}484},
	abstract = {The Gordon Bell Prize is awarded each year by the Association for Computing Machinery to recognize outstanding achievement in high-performance computing (HPC). The purpose of the award is to track the progress of parallel computing with particular emphasis on rewarding innovation in applying HPC to applications in science, engineering, and large-scale data analytics. Prizes may be awarded for peak performance or special achievements in scalability and time-to-solution on important science and engineering problems. Financial support for the US$10,000 award is provided through an endowment by Gordon Bell, a pioneer in high-performance and parallel computing. This article examines the evolution of the Gordon Bell Prize and the impact it has had on the field.},
	keywords = {benchmarks, Computational Science, Gordon Bell Prize, High Performance Computing, HPC Cost-Performance, HPC Progress, HPC Recognition, HPC special hardware, HPPC Award. HPC Prize, Technical Computing},
	url = {http://journals.sagepub.com/doi/10.1177/1094342017738610},
	author = {Gordon Bell and David Bailey and Alan H. Karp and Jack Dongarra and Kevin Walsh}
}
@article {1337,
	title = {MagmaDNN {\textendash} High-Performance Data Analytics for Manycore GPUs and CPUs},
	year = {2017},
	month = {2017-12},
	publisher = {2017 Summer Research Experiences for Undergraduate (REU), Presentation},
	address = {Knoxville, TN},
	author = {Lucien Ng and Kwai Wong and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@techreport {1130,
	title = {MAGMA-sparse Interface Design Whitepaper},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-05},
	year = {2017},
	month = {2017-09},
	type = {Technical Report},
	abstract = {In this report we describe the logic and interface we develop for the MAGMA-sparse library
to allow for easy integration as third-party library into a top-level software ecosystem. The
design choices are based on extensive consultation with other software library developers, in
particular the Trilinos software development team. The interface documentation is at this point
not exhaustive, but a first proposal for setting a standard. Although the interface description
targets the MAGMA-sparse software module, we hope that the design choices carry beyond this
specific library, and are attractive for adoption in other packages.
This report is not intended as static document, but will be updated over time to reflect the agile
software development in the ECP 1.3.3.11 STMS11-PEEKS project.},
	author = {Hartwig Anzt and Erik Boman and Jack Dongarra and Goran Flegar and Mark Gates and Mike Heroux and Mark Hoemmen and Jakub Kurzak and Piotr Luszczek and Sivasankaran Rajamanickam and Stanimire Tomov and Stephen Wood and Ichitaro Yamazaki}
}
@techreport {1173,
	title = {PLASMA 17 Performance Report},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-11},
	year = {2017},
	month = {2017-06},
	publisher = {University of Tennessee},
	abstract = {PLASMA (Parallel Linear Algebra for Multicore Architectures) is a dense linear algebra package at the forefront of multicore computing. PLASMA is designed to deliver the highest possible performance from a system with multiple sockets of multicore processors. PLASMA achieves this objective by combining state of the art solutions in parallel algorithms, scheduling, and software engineering. PLASMA currently offers a collection of routines for solving linear systems of equations and least square problems.},
	author = {Maksims Abalenkovs and Negin Bagherpour and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Samuel Relton and Jakub Sistek and David Stevens and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Mawussi Zounon}
}
@techreport {1172,
	title = {PLASMA 17.1 Functionality Report},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-10},
	year = {2017},
	month = {2017-06},
	publisher = {University of Tennessee},
	abstract = {PLASMA (Parallel Linear Algebra for Multicore Architectures) is a dense linear algebra package at the forefront of multicore computing. PLASMA is designed to deliver the highest possible performance from a system with multiple sockets of multicore processors. PLASMA achieves this objective by combining state of the art solutions in parallel algorithms, scheduling, and software engineering. PLASMA currently offers a collection of routines for solving linear systems of equations and least square problems.},
	author = {Maksims Abalenkovs and Negin Bagherpour and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Samuel Relton and Jakub Sistek and David Stevens and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Mawussi Zounon}
}
@article {1067,
	title = {Preconditioned Krylov Solvers on GPUs},
	journal = {Parallel Computing},
	year = {2017},
	month = {2017-06},
	abstract = {In this paper, we study the effect of enhancing GPU-accelerated Krylov solvers with preconditioners. We consider the BiCGSTAB, CGS, QMR, and IDR(s) Krylov solvers. For a large set of test matrices, we assess the impact of Jacobi and incomplete factorization preconditioning on the solvers{\textquoteright} numerical stability and time-to-solution performance. We also analyze how the use of a preconditioner impacts the choice of the fastest solver.},
	keywords = {gpu, ILU, Jacobi, Krylov solvers, Preconditioning},
	issn = {01678191},
	doi = {10.1016/j.parco.2017.05.006},
	url = {http://www.sciencedirect.com/science/article/pii/S0167819117300777},
	author = {Hartwig Anzt and Mark Gates and Jack Dongarra and Moritz Kreutzer and Gerhard Wellein and Martin Kohler}
}
@techreport {1080,
	title = {Roadmap for the Development of a Linear Algebra Library for Exascale Computing: SLATE: Software for Linear Algebra Targeting Exascale},
	journal = {SLATE Working Notes},
	number = {01, ICL-UT-17-02},
	year = {2017},
	month = {2017-06},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	type = {SLATE Working Notes},
	author = {Ahmad Abdelfattah and Hartwig Anzt and Aurelien Bouteiller and Anthony Danalis and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Stephen Wood and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan}
}
@article {994,
	title = {Accelerating the Conjugate Gradient Algorithm with GPU in CFD Simulations},
	journal = {VECPAR},
	year = {2016},
	abstract = {This paper illustrates how GPU computing can be used to accelerate computational fluid dynamics (CFD) simulations. For sparse linear systems arising from finite volume discretization, we evaluate and optimize the performance of Conjugate Gradient (CG) routines designed for manycore accelerators and compare against an industrial CPU-based implementation. We also investigate how the recent advances in preconditioning, such as iterative Incomplete Cholesky (IC, as symmetric case of ILU) preconditioning, match the requirements for solving real world problems.},
	url = {http://hgpu.org/?p=16264},
	author = {Hartwig Anzt and Marc Baboulin and Jack Dongarra and Yvan Fournier and Frank Hulsemann and Amal Khabou and Yushan Wang}
}
@inbook {883,
	title = {Dense Symmetric Indefinite Factorization on GPU Accelerated Architectures},
	booktitle = {Lecture Notes in Computer Science},
	series = {11th International Conference, PPAM 2015, Krakow, Poland, September 6-9, 2015. Revised Selected Papers, Part I},
	volume = {9573},
	year = {2016},
	month = {2015-09},
	pages = {86-95},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	chapter = {Parallel Processing and Applied Mathematics},
	abstract = {We study the performance of dense symmetric indefinite factorizations (Bunch-Kaufman and Aasen{\textquoteright}s algorithms) on multicore CPUs with a Graphics Processing Unit (GPU). Though such algorithms are needed in many scientific and engineering simulations, obtaining high performance of the factorization on the GPU is difficult because the pivoting that is required to ensure the numerical stability of the factorization leads to frequent synchronizations and irregular data accesses. As a result, until recently, there has not been any implementation of these algorithms on hybrid CPU/GPU architectures. To improve their performance on the hybrid architecture, we explore different techniques to reduce the expensive communication and synchronization between the CPU and GPU, or on the GPU. We also study the performance of an LDL^T factorization with no pivoting combined with the preprocessing technique based on Random Butterfly Transformations. Though such transformations only have probabilistic results on the numerical stability, they avoid the pivoting and obtain a great performance on the GPU.
},
	keywords = {Communication-avoiding, Dense symmetric indefinite factorization, gpu computation, randomization},
	isbn = {978-3-319-32149-3},
	doi = {10.1007/978-3-319-32149-3_9},
	author = {Marc Baboulin and Jack Dongarra and Adrien Remy and Stanimire Tomov and Ichitaro Yamazaki},
	editor = {Roman Wyrzykowski and Ewa Deelman and Konrad Karczewski and Jacek Kitowski and Kazimierz Wiatr}
}
@inproceedings {992,
	title = {Efficiency of General Krylov Methods on GPUs {\textendash} An Experimental Study},
	journal = {2016 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2016},
	month = {2016-05},
	pages = {683-691},
	abstract = {This paper compares different Krylov methods based on short recurrences with respect to their efficiency whenimplemented on GPUs. The comparison includes BiCGSTAB, CGS, QMR, and IDR using different shadow space dimensions. These methods are known for their good convergencecharacteristics. For a large set of test matrices taken from theUniversity of Florida Matrix Collection, we evaluate the methods{\textquoteright}performance against different target metrics: convergence, number of sparse matrix-vector multiplications, and executiontime. We also analyze whether the methods are "orthogonal"in terms of problem suitability. We propose best practicesfor choosing methods in a "black box" scenario, where noinformation about the optimal solver is available.},
	keywords = {algorithmic bombardment, BiCGSTAB, CGS, Convergence, Electric breakdown, gpu, graphics processing units, Hardware, IDR(s), Krylov solver, Libraries, linear systems, QMR, Sparse matrices},
	doi = {10.1109/IPDPSW.2016.45},
	author = {Hartwig Anzt and Jack Dongarra and Moritz Kreutzer and Gerhard Wellein and Martin Kohler}
}
@conference {937,
	title = {Efficiency of General Krylov Methods on GPUs {\textendash} An Experimental Study},
	booktitle = {The Sixth International Workshop on Accelerators and Hybrid Exascale Systems (AsHES)},
	year = {2016},
	month = {2016-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {This paper compares different Krylov methods based on short recurrences with respect to their efficiency when implemented on GPUs. The comparison includes BiCGSTAB, CGS, QMR, and IDR using different shadow space dimensions. These methods are known for their good convergence characteristics. For a large set of test matrices taken from the University of Florida Matrix Collection, we evaluate the methods{\textquoteright}
performance against different target metrics: convergence, number of sparse matrix-vector multiplications, and execution time. We also analyze whether the methods are {\textquotedblleft}orthogonal{\textquotedblright} in terms of problem suitability. We propose best practices for choosing methods in a {\textquotedblleft}black box{\textquotedblright} scenario, where no information about the optimal solver is available.},
	keywords = {algorithmic bombardment, BiCGSTAB, CGS, gpu, IDR(s), Krylov solver, QMR},
	doi = {10.1109/IPDPSW.2016.45},
	author = {Hartwig Anzt and Jack Dongarra and Moritz Kreutzer and Gerhard Wellein and Martin Kohler}
}
@conference {941,
	title = {GPU-Aware Non-contiguous Data Movement In Open MPI},
	booktitle = {25th International Symposium on High-Performance Parallel and Distributed Computing (HPDC{\textquoteright}16)},
	year = {2016},
	month = {2016-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Kyoto, Japan},
	abstract = {<p>Due to better parallel density and power efficiency, GPUs have become more popular for use in scientific applications. Many of these applications are based on the ubiquitous Message Passing Interface (MPI) programming paradigm, and take advantage of non-contiguous memory layouts to exchange data between processes. However, support for efficient non-contiguous data movements for GPU-resident data is still in its infancy, imposing a negative impact on the overall application performance.</p>

<p>To address this shortcoming, we present a solution where we take advantage of the inherent parallelism in the datatype packing and unpacking operations. We developed a close integration between Open MPI{\textquoteright}s stack-based datatype engine, NVIDIA{\textquoteright}s Unied Memory Architecture and GPUDirect capabilities. In this design the datatype packing and unpacking operations are offloaded onto the GPU and handled by specialized GPU kernels, while the CPU remains the driver for data movements between nodes. By incorporating our design into the Open MPI library we have shown significantly better performance for non-contiguous GPU-resident data transfers on both shared and distributed memory machines.</p>},
	keywords = {datatype, gpu, hybrid architecture, MPI, non-contiguous data},
	doi = {http://dx.doi.org/10.1145/2907294.2907317},
	author = {Wei Wu and George Bosilca and Rolf vandeVaart and Sylvain Jeaugey and Jack Dongarra}
}
@conference {939,
	title = {Heterogeneous Streaming},
	booktitle = {The Sixth International Workshop on Accelerators and Hybrid Exascale Systems (AsHES), IPDPS 2016},
	year = {2016},
	month = {2016-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {This paper introduces a new heterogeneous streaming library called hetero Streams (hStreams). We show how a simple FIFO streaming model can be applied to heterogeneous systems that include manycore coprocessors and multicore CPUs. This model supports concurrency across nodes, among tasks within a node, and between data transfers and computation. We give examples for different approaches, show how the implementation can be layered, analyze overheads among layers, and apply those models to parallelize applications using simple, intuitive interfaces. We compare the features and versatility of hStreams, OpenMP, CUDA Streams1 and OmpSs. We show how the use of hStreams makes it easier for scientists to identify tasks and easily expose concurrency among them, and how it enables tuning experts and runtime systems to tailor execution for different heterogeneous targets. Practical application examples are taken from the field of numerical linear algebra, commercial structural simulation software, and a seismic processing application.},
	keywords = {plasma},
	author = {Chris J. Newburn and Gaurav Bansal and Michael Wood and Luis Crivelli and Judit Planas and Alejandro Duran and Paulo Souza and Leonardo Borges and Piotr Luszczek and Stanimire Tomov and Jack Dongarra and Hartwig Anzt and Mark Gates and Azzam Haidar and Yulu Jia and Khairul Kabir and Ichitaro Yamazaki and Jesus Labarta}
}
@conference {841,
	title = {Hierarchical DAG scheduling for Hybrid Distributed Systems},
	booktitle = {29th IEEE International Parallel \& Distributed Processing Symposium (IPDPS)},
	year = {2015},
	month = {2015-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Hyderabad, India},
	abstract = {Accelerator-enhanced computing platforms have drawn a lot of attention due to their massive peak com-putational capacity. Despite significant advances in the pro-gramming interfaces to such hybrid architectures, traditional programming paradigms struggle mapping the resulting multi-dimensional heterogeneity and the expression of algorithm parallelism, resulting in sub-optimal effective performance. Task-based programming paradigms have the capability to alleviate some of the programming challenges on distributed hybrid many-core architectures. In this paper we take this concept a step further by showing that the potential of task-based programming paradigms can be greatly increased with minimal modification of the underlying runtime combined with the right algorithmic changes. We propose two novel recursive algorithmic variants for one-sided factorizations and describe the changes to the PaRSEC task-scheduling runtime to build a framework where the task granularity is dynamically adjusted to adapt the degree of available parallelism and kernel effi-ciency according to runtime conditions. Based on an extensive set of results we show that, with one-sided factorizations, i.e. Cholesky and QR, a carefully written algorithm, supported by an adaptive tasks-based runtime, is capable of reaching a degree of performance and scalability never achieved before in distributed hybrid environments. },
	keywords = {dense linear algebra, gpu, heterogeneous architecture, PaRSEC runtime},
	author = {Wei Wu and Aurelien Bouteiller and George Bosilca and Mathieu Faverge and Jack Dongarra}
}
@conference {768,
	title = {Utilizing Dataflow-based Execution for Coupled Cluster Methods},
	booktitle = {2014 IEEE International Conference on Cluster Computing},
	number = {ICL-UT-14-02},
	year = {2014},
	month = {2014-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Madrid, Spain},
	abstract = {Computational chemistry comprises one of the driving forces of High Performance Computing. In particular, many-body methods, such as Coupled Cluster (CC) methods of the quantum chemistry package NWCHEM, are of particular interest for the applied chemistry community.

Harnessing large fractions of the processing power of modern large scale computing platforms has become increasingly difficult. With the increase in scale, complexity, and heterogeneity of modern platforms, traditional programming models fail to deliver the expected performance scalability. On our way to Exascale and with these extremely hybrid platforms, dataflow-based programming models may be the only viable way for achieving and maintaining computation at scale.

In this paper, we discuss a dataflow-based programming model and its applicability to NWCHEM{\textquoteright}s CC methods. Our dataflow version of the CC kernels breaks down the algorithm into fine-grained tasks with explicitly defined data dependencies. As a result, many of the traditional synchronization points can be eliminated, allowing for a dynamic reshaping of the execution based on the ongoing availability of computational resources. We build this experiment using PARSEC {\textendash} a task-based dataflow-driven execution engine {\textendash} that enables efficient task scheduling on distributed systems, providing a desirable portability layer for application developers.},
	author = {Heike McCraw and Anthony Danalis and George Bosilca and Jack Dongarra and Karol Kowalski and Theresa Windus}
}
@article {icl:698,
	title = {Dense Linear Algebra on Distributed Heterogeneous Hardware with a Symbolic DAG Approach},
	journal = {Scalable Computing and Communications: Theory and Practice},
	year = {2013},
	month = {2013-03},
	pages = {699-735},
	publisher = {John Wiley \& Sons},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Piotr Luszczek and Jack Dongarra},
	editor = {Samee Khan and Lin-Wang Wang and Albert Zomaya}
}
@inbook {762,
	title = {Keeneland: Computational Science Using Heterogeneous GPU Computing},
	booktitle = {Contemporary High Performance Computing: From Petascale Toward Exascale},
	series = {CRC Computational Science Series},
	year = {2013},
	publisher = {Taylor and Francis},
	organization = {Taylor and Francis},
	chapter = {7},
	address = {Boca Raton, FL},
	abstract = {The Keeneland Project is a five year Track 2D grant awarded by the National Science Foundation (NSF) under solicitation NSF 08-573 in August 2009 for the development and deployment of an innovative high performance computing system. The Keeneland project is led by the Georgia Institute of Technology (Georgia Tech) in collaboration with the University of Tennessee at Knoxville, National Institute of Computational Sciences, and Oak Ridge National Laboratory.},
	author = {Jeffrey Vetter and Richard Glassbrook and Karsten Schwan and Sudha Yalamanchili and Mitch Horton and Ada Gavrilovska and Magda Slawinska and Jack Dongarra and Jeremy Meredith and Philip Roth and Kyle Spafford and Stanimire Tomov and John Wynkoop}
}
@inbook {747,
	title = {LAPACK},
	booktitle = {Handbook of Linear Algebra},
	year = {2013},
	publisher = {CRC Press},
	organization = {CRC Press},
	edition = {Second},
	address = {Boca Raton, FL},
	abstract = {With a substantial amount of new material, the Handbook of Linear Algebra, Second Edition provides comprehensive coverage of linear algebra concepts, applications, and computational software packages in an easy-to-use format. It guides you from the very elementary aspects of the subject to the frontiers of current research. Along with revisions and updates throughout, the second edition of this bestseller includes 20 new chapters.},
	isbn = {9781466507289},
	author = {Zhaojun Bai and James Demmel and Jack Dongarra and Julien Langou and Jenny Wang}
}
@article {756,
	title = {Level-3 Cholesky Factorization Routines Improve Performance of Many Cholesky Algorithms},
	journal = {ACM Transactions on Mathematical Software (TOMS)},
	volume = {39},
	year = {2013},
	month = {2013-02},
	abstract = {Four routines called DPOTF3i, i = a,b,c,d, are presented. DPOTF3i are a novel type of level-3 BLAS for use by BPF (Blocked Packed Format) Cholesky factorization and LAPACK routine DPOTRF. Performance of routines DPOTF3i are still increasing when the performance of Level-2 routine DPOTF2 of LAPACK starts decreasing. This is our main result and it implies, due to the use of larger block size nb, that DGEMM, DSYRK, and DTRSM performance also increases! The four DPOTF3i routines use simple register blocking. Different platforms have different numbers of registers. Thus, our four routines have different register blocking sizes.

BPF is introduced. LAPACK routines for POTRF and PPTRF using BPF instead of full and packed format are shown to be trivial modifications of LAPACK POTRF source codes. We call these codes BPTRF. There are two variants of BPF: lower and upper. Upper BPF is {\textquotedblleft}identical{\textquotedblright} to Square Block Packed Format (SBPF). {\textquotedblleft}LAPACK{\textquotedblright} implementations on multicore processors use SBPF. Lower BPF is less efficient than upper BPF. Vector inplace transposition converts lower BPF to upper BPF very efficiently. Corroborating performance results for DPOTF3i versus DPOTF2 on a variety of common platforms are given for n ≈ nb as well as results for large n comparing DBPTRF versus DPOTRF.},
	doi = {10.1145/2427023.2427026},
	author = {Fred G. Gustavson and Jerzy Wasniewski and Jack Dongarra and Jos{\'e} Herrero and Julien Langou}
}
@conference {1381,
	title = {Non-Determinism and Overcount on Modern Hardware Performance Counter Implementations},
	booktitle = {2013 IEEE International Symposium on Performance Analysis of Systems and Software},
	year = {2013},
	month = {2013-04},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Austin, TX},
	author = {Vincent Weaver and Dan Terpstra and Shirley Moore}
}
@article {1382,
	title = {PAPI 5: Measuring Power, Energy, and the Cloud},
	year = {2013},
	month = {2013-04},
	publisher = {2013 IEEE International Symposium on Performance Analysis of Systems and Software},
	address = {Austin, TX},
	author = {Vincent Weaver and Dan Terpstra and Heike McCraw and Matt Johnson and Kiran Kasichayanula and James Ralph and John Nelson and Phil Mucci and Tushar Mohan and Shirley Moore}
}
@conference {763,
	title = {A Parallel Solver for Incompressible Fluid Flows},
	booktitle = {International Conference on Computational Science (ICCS 2013)},
	year = {2013},
	month = {2013-06},
	publisher = {Elsevier B.V.},
	organization = {Elsevier B.V.},
	address = {Barcelona, Spain},
	abstract = {The Navier-Stokes equations describe a large class of fluid flows but are difficult to solve analytically because of their nonlin- earity. We present in this paper a parallel solver for the 3-D Navier-Stokes equations of incompressible unsteady flows with constant coefficients, discretized by the finite difference method. We apply the prediction-projection method which transforms the Navier-Stokes equations into three Helmholtz equations and one Poisson equation. For each Helmholtz system, we apply the Alternating Direction Implicit (ADI) method resulting in three tridiagonal systems. The Poisson equation is solved using partial diagonalization which transforms the Laplacian operator into a tridiagonal one. We describe an implementation based on MPI where the computations are performed on each subdomain and information is exchanged on the interfaces, and where the tridiagonal system solutions are accelerated using vectorization techniques. We present performance results on a current multicore system.},
	keywords = {ADI, Navier-Stokes equations, Parallel computing, Partial diagonalization, Prediction-projection, SIMD},
	doi = {DOI: 10.1016/j.procs.2013.05.207},
	author = {Yushan Wang and Marc Baboulin and Jo{\"e}l Falcou and Yann Fraigneau and Olivier Le Ma{\^\i}tre}
}
@conference {761,
	title = {Standards for Graph Algorithm Primitives},
	booktitle = {17th IEEE High Performance Extreme Computing Conference (HPEC {\textquoteright}13)},
	year = {2013},
	month = {2013-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {It is our view that the state of the art in constructing a large collection of graph algorithms in terms of linear algebraic operations is mature enough to support the emergence of a standard set of primitive building blocks. This paper is a position paper defining the problem and announcing our intention to launch an open effort to define this standard.},
	keywords = {algorithms, graphs, linear algebra, software standards},
	doi = {10.1109/HPEC.2013.6670338},
	author = {Tim Mattson and David Bader and Jon Berry and Aydin Buluc and Jack Dongarra and Christos Faloutsos and John Feo and John Gilbert and Joseph Gonzalez and Bruce Hendrickson and Jeremy Kepner and Charles Lieserson and Andrew Lumsdaine and David Padua and Steve W. Poole and Steve Reinhardt and Mike Stonebraker and Steve Wallach and Andrew Yoo}
}
@inproceedings {icl:711,
	title = {Energy Footprint of Advanced Dense Numerical Linear Algebra using Tile Algorithms on Multicore Architecture},
	journal = {The 2nd International Conference on Cloud and Green Computing (submitted)},
	year = {2012},
	month = {2012-11},
	address = {Xiangtan, Hunan, China},
	author = {Jack Dongarra and Hatem Ltaeif and Piotr Luszczek and Vincent M Weaver}
}
@article {icl:725,
	title = {From CUDA to OpenCL: Towards a Performance-portable Solution for Multi-platform GPU Programming},
	journal = {Parallel Computing},
	volume = {38},
	number = {8},
	year = {2012},
	month = {2012-08},
	pages = {391-407},
	author = {Peng Du and Rick Weber and Piotr Luszczek and Stanimire Tomov and Gregory D. Peterson and Jack Dongarra}
}
@inproceedings {icl:689,
	title = {Measuring Energy and Power with PAPI},
	journal = {International Workshop on Power-Aware Systems and Architectures},
	year = {2012},
	month = {2012-09},
	address = {Pittsburgh, PA},
	abstract = {Energy and power consumption are becoming critical metrics in the design and usage of high performance systems. We have extended the Performance API (PAPI) analysis library to measure and report energy and power values. These values are reported using the existing PAPI API, allowing code previously instrumented for performance counters to also measure power and energy. Higher level tools that build on PAPI will automatically gain support for power and energy readings when used with the newest version of PAPI. We describe in detail the types of energy and power readings available through PAPI. We support external power meters, as well as values provided internally by recent CPUs and GPUs. Measurements are provided directly to the instrumented process, allowing immediate code analysis in real time. We provide examples showing results that can be obtained with our infrastructure.},
	keywords = {papi},
	doi = {10.1109/ICPPW.2012.39},
	author = {Vincent M Weaver and Matt Johnson and Kiran Kasichayanula and James Ralph and Piotr Luszczek and Dan Terpstra and Shirley Moore}
}
@article {icl:688,
	title = {PAPI-V: Performance Monitoring for Virtual Machines},
	journal = {CloudTech-HPC 2012},
	year = {2012},
	month = {2012-09},
	address = {Pittsburgh, PA},
	abstract = {This paper describes extensions to the PAPI hardware counter library for virtual environments, called PAPI-V. The extensions support timing routines, I/O measurements, and processor counters. The PAPI-V extensions will allow application and tool developers to use a familiar interface to obtain relevant hardware performance monitoring information in virtual environments.},
	keywords = {papi},
	doi = {10.1109/ICPPW.2012.29},
	author = {Matt Johnson and Heike McCraw and Shirley Moore and Phil Mucci and John Nelson and Dan Terpstra and Vincent M Weaver and Tushar Mohan}
}
@article {icl:727,
	title = {Parallel Processing and Applied Mathematics, 9th International Conference, PPAM 2011},
	journal = {Lecture Notes in Computer Science},
	volume = {7203},
	year = {2012},
	month = {2012-00},
	address = {Torun, Poland},
	editor = {Roman Wyrzykowski and Jack Dongarra and Konrad Karczewski and Jerzy Wasniewski}
}
@article {icl:722,
	title = {Reducing the Amount of Pivoting in Symmetric Indefinite Systems},
	journal = {Parallel Processing and Applied Mathematics, Lecture Notes in Computer Science (PPAM 2011)},
	volume = {7203},
	year = {2012},
	month = {2012-00},
	pages = {133-142},
	publisher = {Springer-Verlag Berlin Heidelberg},
	author = {Dulceneia Becker and Marc Baboulin and Jack Dongarra},
	editor = {Roman Wyrzykowski and Jack Dongarra and Konrad Karczewski and Jerzy Wasniewski}
}
@inproceedings {icl:736,
	title = {User Level Failure Mitigation in MPI},
	journal = {Euro-Par 2012: Parallel Processing Workshops},
	volume = {7640},
	year = {2012},
	month = {2012-08},
	pages = {499-504},
	publisher = {Springer Berlin Heidelberg},
	address = {Rhodes Island, Greece},
	keywords = {ftmpi},
	author = {Wesley Bland},
	editor = {Ioannis Caragiannis and Michael Alexander and Rosa M. Badia and Mario Cannataro and Alexandru Costan and Marco Danelutto and Frederic Desprez and Bettina Krammer and Sahuquillo, J. and Stephen L. Scott and J. Weidendorfer}
}
@article {icl:623,
	title = {Energy and performance characteristics of different parallel implementations of scientific applications on multicore systems},
	journal = {International Journal of High Performance Computing Applications},
	volume = {25},
	number = {3},
	year = {2011},
	month = {2011-00},
	pages = {342-350},
	keywords = {mumi},
	author = {Charles Lively and Xingfu Wu and Valerie Taylor and Shirley Moore and Hung-Ching Chang and Kirk Cameron}
}
@inproceedings {icl:616,
	title = {Evaluation of the HPC Challenge Benchmarks in Virtualized Environments},
	journal = {6th Workshop on Virtualization in High-Performance Cloud Computing},
	year = {2011},
	month = {2011-08},
	address = {Bordeaux, France},
	keywords = {hpcc},
	author = {Piotr Luszczek and Eric Meek and Shirley Moore and Dan Terpstra and Vincent M Weaver and Jack Dongarra}
}
@article {,
	title = {High-Performance High-Resolution Semi-Lagrangian Tracer Transport on a Sphere},
	journal = {Journal of Computational Physics},
	volume = {230},
	year = {2011},
	month = {2011-07},
	pages = {6778-6799},
	abstract = {Current climate models have a limited ability to increase spatial resolution because numerical stability requires the time step to decrease. We describe a semi-Lagrangian method for tracer transport that is stable for arbitrary Courant numbers, and we test a parallel implementation discretized on the cubed sphere. The method includes a fixer that conserves mass and constrains tracers to a physical range of values. The method shows third-order convergence and maintains nonlinear tracer correlations to second order. It shows optimal accuracy at Courant numbers of 10{\textendash}20, more than an order of magnitude higher than explicit methods. We present parallel performance in terms of strong scaling, weak scaling, and spatial scaling (where the time step stays constant while the resolution increases). For a 0.2{\textdegree} test with 100 tracers, the implementation scales efficiently to 10,000 MPI tasks.},
	keywords = {cubed sphere, high resolution, High-performance computing, semi-Lagrangian, spherical geometry, tracer transport},
	doi = {https://doi.org/10.1016/j.jcp.2011.05.008},
	author = {James B. White and Jack Dongarra}
}
@article {icl:643,
	title = {The International Exascale Software Project Roadmap},
	journal = {International Journal of High Performance Computing},
	volume = {25},
	number = {1},
	year = {2011},
	month = {2011-01},
	pages = {3-60},
	abstract = {Over the last 20 years, the open-source community has provided more and more software on which the world{\textquoteright}s high-performance computing systems depend for performance and productivity. The community has invested millions of dollars and years of effort to build key components. However, although the investments in these separate software elements have been tremendously valuable, a great deal of productivity has also been lost because of the lack of planning, coordination, and key integration of technologies necessary to make them work together smoothly and efficiently, both within individual petascale systems and between different systems. It seems clear that this completely uncoordinated development model will not provide the software needed to support the unprecedented parallelism required for peta/ exascale computation on millions of cores, or the flexibility required to exploit new hardware models and features, such as transactional memory, speculative execution, and graphics processing units. This report describes the work of the community to prepare for the challenges of exascale computing, ultimately combing their efforts in a coordinated International Exascale Software Project.},
	doi = {https://doi.org/10.1177/1094342010391989},
	author = {Jack Dongarra and Pete Beckman and Terry Moore and Patrick Aerts and Giovanni Aloisio and Jean-Claude Andre and David Barkai and Jean-Yves Berthou and Taisuke Boku and Bertrand Braunschweig and Franck Cappello and Barbara Chapman and Xuebin Chi and Alok Choudhary and Sudip Dosanjh and Thom Dunning and Sandro Fiore and Al Geist and Bill Gropp and Robert Harrison and Mark Hereld and Michael Heroux and Adolfy Hoisie and Koh Hotta and Zhong Jin and Yutaka Ishikawa and Fred Johnson and Sanjay Kale and Richard Kenway and David Keyes and Bill Kramer and Jesus Labarta and Alain Lichnewsky and Thomas Lippert and Bob Lucas and Barney MacCabe and Satoshi Matsuoka and Paul Messina and Peter Michielse and Bernd Mohr and Matthias S. Mueller and Wolfgang E. Nagel and Hiroshi Nakashima and Michael E. Papka and Dan Reed and Mitsuhisa Sato and Ed Seidel and John Shalf and David Skinner and Marc Snir and Thomas Sterling and Rick Stevens and Fred Streitz and Bob Sugar and Shinji Sumimoto and William Tang and John Taylor and Rajeev Thakur and Anne Trefethen and Mateo Valero and Aad van der Steen and Jeffrey Vetter and Peg Williams and Robert Wisniewski and Kathy Yelick}
}
@inproceedings {icl:590,
	title = {Overlapping Computation and Communication for Advection on a Hybrid Parallel Computer},
	journal = {IEEE International Parallel and Distributed Processing Symposium (submitted)},
	year = {2011},
	month = {2011-05},
	address = {Anchorage, AK},
	author = {James B White and Jack Dongarra}
}
@inproceedings {icl:619,
	title = {Power-Aware Prediction Models of Hybrid (MPI/OpenMP) Scientific Applications},
	journal = {International Conference on Energy-Aware High Performance Computing (EnA-HPC 2011)},
	year = {2011},
	month = {2011-09},
	address = {Hamburg, Germany},
	keywords = {mumi},
	author = {Charles Lively and Xingfu Wu and Valerie Taylor and Shirley Moore and Hung-Ching Chang and Chun-Yi Su and Kirk Cameron}
}
@inproceedings {icl:580,
	title = {8th International Conference on Parallel Processing and Applied Mathematics, Lecture Notes in Computer Science (LNCS)},
	journal = {PPAM 2009 Proceedings},
	volume = {6067},
	year = {2010},
	month = {2010-09},
	publisher = {Springer},
	address = {Wroclaw, Poland},
	editor = {Roman Wyrzykowski and Jack Dongarra and Konrad Karczewski and Jerzy Wasniewski}
}
@inproceedings {icl:598,
	title = {Can Hardware Performance Counters Produce Expected, Deterministic Results?},
	journal = {3rd Workshop on Functionality of Hardware Performance Monitoring},
	year = {2010},
	month = {2010-12},
	address = {Atlanta, GA},
	keywords = {papi},
	author = {Vincent M Weaver and Jack Dongarra}
}
@article {icl:586,
	title = {Empirical Performance Tuning of Dense Linear Algebra Software},
	journal = {in Performance Tuning of Scientific Applications (to appear)},
	year = {2010},
	month = {2010-00},
	author = {Jack Dongarra and Shirley Moore},
	editor = {David Bailey and Robert Lucas and Sam Williams}
}
@article {icl:573,
	title = {Level-3 Cholesky Kernel Subroutine of a Fully Portable High Performance Minimal Storage Hybrid Format Cholesky Algorithm},
	journal = {ACM TOMS (submitted), also LAPACK Working Note (LAWN) 211},
	year = {2010},
	month = {2010-00},
	author = {Fred G. Gustavson and Jerzy Wasniewski and Jack Dongarra}
}
@article {icl:551,
	title = {Rectangular Full Packed Format for Cholesky{\textquoteright}s Algorithm: Factorization, Solution, and Inversion},
	journal = {ACM Transactions on Mathematical Software (TOMS)},
	volume = {37},
	number = {2},
	year = {2010},
	month = {2010-04},
	address = {Atlanta, GA},
	author = {Fred G. Gustavson and Jerzy Wasniewski and Jack Dongarra and Julien Langou}
}
@article {icl:570,
	title = {Rectangular Full Packed Format for Cholesky{\textquoteright}s Algorithm: Factorization, Solution and Inversion},
	journal = {ACM Transactions on Mathematical Software (TOMS)},
	volume = {37},
	number = {2},
	year = {2010},
	month = {2010-04},
	author = {Fred G. Gustavson and Jerzy Wasniewski and Jack Dongarra and Julien Langou}
}
@article {icl:568,
	title = {Accelerating Time-To-Solution for Computational Science and Engineering},
	journal = {SciDAC Review},
	year = {2009},
	month = {2009-00},
	author = {James Demmel and Jack Dongarra and Armando Fox and Sam Williams and Vasily Volkov and Katherine Yelick}
}
@article {icl:500,
	title = {Paravirtualization Effect on Single- and Multi-threaded Memory-Intensive Linear Algebra Software},
	journal = {Cluster Computing Journal: Special Issue on High Performance Distributed Computing},
	volume = {12},
	number = {2},
	year = {2009},
	month = {2009-00},
	pages = {101-122},
	publisher = {Springer Netherlands},
	author = {Lamia Youseff and Keith Seymour and Haihang You and Dmitrii Zagorodnov and Jack Dongarra and Rich Wolski}
}
@article {icl:511,
	title = {Rectangular Full Packed Format for Cholesky{\textquoteright}s Algorithm: Factorization, Solution and Inversion},
	journal = {ACM TOMS (to appear)},
	year = {2009},
	month = {2009-00},
	author = {Fred G. Gustavson and Jerzy Wasniewski and Jack Dongarra and Julien Langou}
}
@article {icl:506,
	title = {Towards Efficient MapReduce Using MPI},
	journal = {Lecture Notes in Computer Science, Recent Advances in Parallel Virtual Machine and Message Passing Interface - 16th European PVM/MPI Users{\textquoteright} Group Meeting},
	volume = {5759},
	year = {2009},
	month = {2009-00},
	pages = {240-249},
	publisher = {Springer Berlin / Heidelberg},
	address = {Espoo, Finland},
	author = {Torsten Hoefler and Yuan-Shun Dai and Jack Dongarra},
	editor = {M. Ropo and J Westerholm and Jack Dongarra}
}
@inproceedings {icl:519,
	title = {VGrADS: Enabling e-Science Workflows on Grids and Clouds with Fault Tolerance},
	journal = {SC{\textquoteright}09 The International Conference for High Performance Computing, Networking, Storage and Analysis (to appear)},
	year = {2009},
	month = {2009-00},
	address = {Portland, OR},
	keywords = {grads},
	author = {Lavanya Ramakrishan and Daniel Nurmi and Anirban Mandal and Charles Koelbel and Dennis Gannon and Mark Huang and Yang-Suk Kee and Graziano Obertelli and Kiran Thyagaraja and Rich Wolski and Asim YarKhan and Dmitrii Zagorodnov}
}
@inproceedings {icl:452,
	journal = {7th International parallel Processing and Applied Mathematics Conference, Lecture Notes in Comptuer Science},
	volume = {4967},
	year = {2008},
	month = {2008-01},
	publisher = {Springer Berlin},
	address = {Gdansk, Poland},
	editor = {Roman Wyrzykowski and Jack Dongarra and Konrad Karczewski and Jerzy Wasniewski}
}
@article {icl:450,
	title = {How Elegant Code Evolves With Hardware: The Case Of Gaussian Elimination},
	journal = {in Beautiful Code Leading Programmers Explain How They Think (Chapter 14)},
	year = {2008},
	month = {2008-01},
	pages = {243-282},
	author = {Jack Dongarra and Piotr Luszczek},
	editor = {Andy Oram and G. Wilson}
}
@inproceedings {icl:410,
	title = {The Impact of Paravirtualized Memory Hierarchy on Linear Algebra Computational Kernels and Software},
	journal = {ACM/IEEE International Symposium on High Performance Distributed Computing},
	year = {2008},
	month = {2008-06},
	address = {Boston, MA.},
	keywords = {gco, netsolve},
	author = {Lamia Youseff and Keith Seymour and Haihang You and Jack Dongarra and Rich Wolski}
}
@inproceedings {icl:416,
	title = {Interior State Computation of Nano Structures},
	journal = {PARA 2008, 9th International Workshop on State-of-the-Art in Scientific and Parallel Computing},
	year = {2008},
	month = {2008-05},
	address = {Trondheim, Norway},
	author = {Andrew Canning and Jack Dongarra and Julien Langou and Osni Marques and Stanimire Tomov and Christof Voemel and Lin-Wang Wang}
}
@article {icl:417,
	title = {Performance Instrumentation and Compiler Optimizations for MPI/OpenMP Applications},
	journal = {Lecture Notes in Computer Science, OpenMP Shared Memory Parallel Programming},
	volume = {4315},
	year = {2008},
	month = {2008-00},
	publisher = {Springer Berlin / Heidelberg},
	author = {Oscar Hernandez and Fengguang Song and Barbara Chapman and Jack Dongarra and Bernd Mohr and Shirley Moore and Felix Wolf}
}
@article {icl:462,
	title = {PERI Auto-tuning},
	journal = {Proc. SciDAC 2008},
	volume = {125},
	year = {2008},
	month = {2008-01},
	publisher = {Journal of Physics},
	address = {Seatlle, Washington},
	keywords = {gco},
	author = {David Bailey and Jacqueline Chame and Chun Chen and Jack Dongarra and Mary Hall and Jeffrey K. Hollingsworth and Paul D. Hovland and Shirley Moore and Keith Seymour and Jaewook Shin and Ananta Tiwari and Sam Williams and Haihang You}
}
@techreport {icl:422,
	title = {Rectangular Full Packed Format for Cholesky{\textquoteright}s Algorithm: Factorization, Solution and Inversion},
	journal = {University of Tennessee Computer Science Technical Report, UT-CS-08-614 (also LAPACK Working Note 199)},
	year = {2008},
	month = {2008-04},
	author = {Fred G. Gustavson and Jerzy Wasniewski and Jack Dongarra}
}
@article {icl:447,
	title = {State-of-the-Art Eigensolvers for Electronic Structure Calculations of Large Scale Nano-Systems},
	journal = {Journal of Computational Physics},
	volume = {227},
	number = {15},
	year = {2008},
	month = {2008-01},
	pages = {7113-7124},
	author = {Christof Voemel and Stanimire Tomov and Osni Marques and Andrew Canning and Lin-Wang Wang and Jack Dongarra}
}
@inproceedings {icl:412,
	title = {Usage of the Scalasca Toolset for Scalable Performance Analysis of Large-scale Parallel Applications},
	journal = {Proceedings of the 2nd International Workshop on Tools for High Performance Computing},
	year = {2008},
	month = {2008-01},
	pages = {157-167},
	publisher = {Springer},
	address = {Stuttgart, Germany},
	keywords = {point},
	author = {Felix Wolf and Brian Wylie and Erika Abraham and Wolfgang Frings and Karl F{\"u}rlinger and Markus Geimer and Marc-Andre Hermanns and Bernd Mohr and Shirley Moore and Matthias Pfeifer},
	editor = {Michael Resch and Rainer Keller and Valentin Himmler and Bettina Krammer and A Schulz}
}
@article {icl:400,
	title = {Automatic Analysis of Inefficiency Patterns in Parallel Applications},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {19},
	number = {11},
	year = {2007},
	month = {2007-08},
	pages = {1481-1496},
	author = {Felix Wolf and Bernd Mohr and Jack Dongarra and Shirley Moore}
}
@article {icl:365,
	title = {How Elegant Code Evolves With Hardware: The Case Of Gaussian Elimination},
	journal = {in Beautiful Code Leading Programmers Explain How They Think},
	year = {2007},
	month = {2007-06},
	publisher = {O{\textquoteright}Reilly Media, Inc.},
	author = {Jack Dongarra and Piotr Luszczek},
	editor = {Andy Oram and Greg Wilson}
}
@inproceedings {icl:387,
	title = {Results of the PERI survey of SciDAC applications},
	journal = {Journal of Physics: Conference Series, SciDAC 2007},
	volume = {78},
	number = {2007},
	year = {2007},
	month = {2007-01},
	author = {Bronis R. de Supinski and Jeffrey K. Hollingsworth and Shirley Moore and Patrick H. Worley}
}
@article {icl:401,
	title = {The Use of Bulk States to Accelerate the Band Edge State Calculation of a Semiconductor Quantum Dot},
	journal = {Journal of Computational Physics},
	volume = {223},
	year = {2007},
	month = {2007-00},
	pages = {774-782},
	author = {Christof Voemel and Stanimire Tomov and Lin-Wang Wang and Osni Marques and Jack Dongarra}
}
@article {icl:402,
	title = {Conjugate-Gradient Eigenvalue Solvers in Computing Electronic Properties of Nanostructure Architectures},
	journal = {International Journal of Computational Science and Engineering},
	volume = {2},
	number = {3/4},
	year = {2006},
	month = {2006-00},
	pages = {205-212},
	author = {Stanimire Tomov and Julien Langou and Jack Dongarra and Andrew Canning and Lin-Wang Wang}
}
@inproceedings {icl:309,
	title = {Large Event Traces in Parallel Performance Analysis},
	journal = {8th Workshop {\textquoteright}Parallel Systems and Algorithms{\textquoteright} (PASA), Lecture Notes in Informatics},
	number = {ICL-UT-06-08},
	year = {2006},
	month = {2006-03},
	publisher = {Gesellschaft f{\"u}r Informatik},
	address = {Frankfurt/Main, Germany},
	keywords = {kojak},
	author = {Felix Wolf and Felix Freitag and Bernd Mohr and Shirley Moore and Brian Wylie}
}
@inproceedings {icl:325,
	title = {Performance evaluation of eigensolvers in nano-structure computations},
	journal = {IEEE/ACM Proceedings of HPCNano SC06 (to appear)},
	year = {2006},
	month = {2006-01},
	keywords = {doe-nano},
	author = {Andrew Canning and Jack Dongarra and Julien Langou and Osni Marques and Stanimire Tomov and Christof Voemel and Lin-Wang Wang}
}
@inproceedings {icl:319,
	title = {Performance Instrumentation and Compiler Optimizations for MPI/OpenMP Applications},
	journal = {Second International Workshop on OpenMP},
	year = {2006},
	month = {2006-01},
	address = {Reims, France},
	keywords = {kojak},
	author = {Oscar Hernandez and Fengguang Song and Barbara Chapman and Jack Dongarra and Bernd Mohr and Shirley Moore and Felix Wolf}
}
@article {icl:327,
	title = {Predicting the electronic properties of 3D, million-atom semiconductor nanostructure architectures},
	journal = {J. Phys.: Conf. Ser. 46},
	volume = {:101088/1742-6596/46/1/040},
	year = {2006},
	month = {2006-01},
	pages = {292-298},
	keywords = {DOE_NANO},
	author = {Alex Zunger and Alberto Franceschetti and Gabriel Bester and Wesley B. Jones and Kwiseon Kim and Peter A. Graf and Lin-Wang Wang and Andrew Canning and Osni Marques and Christof Voemel and Jack Dongarra and Julien Langou and Stanimire Tomov}
}
@inproceedings {icl:307,
	title = {A Systematic Multi-step Methodology for Performance Analysis of Communication Traces of Distributed Applications based on Hierarchical Clustering},
	journal = {Proc. of the 5th International Workshop on Performance Modeling, Evaluation, and Organization of Parallel and Distributed Systems (PMEO-PDS 2006)},
	number = {ICL-UT-05-06},
	year = {2006},
	month = {2006-04},
	publisher = {IEEE Computer Society},
	address = {Rhodes Island, Greece},
	keywords = {kojak},
	author = {Gabriela Aguilera and Patricia J. Teller and Michela Taufer and Felix Wolf}
}
@inproceedings {icl:324,
	title = {Towards bulk based preconditioning for quantum dot computations},
	journal = {IEEE/ACM Proceedings of HPCNano SC06 (to appear)},
	year = {2006},
	month = {2006-01},
	keywords = {doe-nano},
	author = {Andrew Canning and Jack Dongarra and Julien Langou and Osni Marques and Stanimire Tomov and Christof Voemel and Lin-Wang Wang}
}
@article {icl:326,
	title = {The use of bulk states to accelerate the band edge state calculation of a semiconductor quantum dot},
	journal = {Journal of Computational Physics (submitted)},
	year = {2006},
	month = {2006-01},
	keywords = {doe-nano},
	author = {Christof Voemel and Stanimire Tomov and Lin-Wang Wang and Osni Marques and Jack Dongarra}
}
@article {icl:271,
	title = {Automatic analysis of inefficiency patterns in parallel applications},
	journal = {Concurrency and Computation: Practice and Experience, Special issue "Automatic Performance Analysis" (submitted)},
	year = {2005},
	month = {2005-00},
	keywords = {kojak},
	author = {Felix Wolf and Bernd Mohr and Jack Dongarra and Shirley Moore}
}
@inproceedings {icl:288,
	title = {Automatic Experimental Analysis of Communication Patterns in Virtual Topologies},
	journal = {In Proceedings of the International Conference on Parallel Processing},
	year = {2005},
	month = {2005-06},
	publisher = {IEEE Computer Society},
	address = {Oslo, Norway},
	keywords = {kojak},
	author = {Nikhil Bhatia and Fengguang Song and Felix Wolf and Jack Dongarra and Bernd Mohr and Shirley Moore}
}
@inproceedings {icl:284,
	title = {Comparison of Nonlinear Conjugate-Gradient methods for computing the Electronic Properties of Nanostructure Architectures},
	journal = {Proceedings of 5th International Conference on Computational Science (ICCS)},
	year = {2005},
	month = {2005-01},
	pages = {317-325},
	publisher = {Springer{\textquoteright}s Lecture Notes in Computer Science},
	address = {Atlanta, GA, USA},
	keywords = {doe-nano},
	author = {Stanimire Tomov and Julien Langou and Andrew Canning and Lin-Wang Wang and Jack Dongarra},
	editor = {V. S. Sunderman and Geert Dick van Albada and Peter M. Sloot and Jack Dongarra}
}
@article {icl:292,
	title = {Conjugate-Gradient Eigenvalue Solvers in Computing Electronic Properties of Nanostructure Architectures},
	journal = {International Journal of Computational Science and Engineering (to appear)},
	year = {2005},
	month = {2005-01},
	author = {Stanimire Tomov and Julien Langou and Andrew Canning and Lin-Wang Wang and Jack Dongarra}
}
@inproceedings {icl:272,
	title = {Event-based Measurement and Analysis of One-sided Communication},
	journal = {In Proceedings of the European Conference on Parallel Computing (Euro-Par)},
	year = {2005},
	month = {2005-08},
	publisher = {Springer},
	address = {Lisbon, Portugal},
	keywords = {kojak},
	author = {Marc-Andre Hermanns and Bernd Mohr and Felix Wolf}
}
@inproceedings {icl:248,
	title = {Improving Time to Solution with Automated Performance Analysis},
	journal = {Second Workshop on Productivity and Performance in High-End Computing (P-PHEC) at 11th International Symposium on High Performance Computer Architecture (HPCA-2005)},
	year = {2005},
	month = {2005-02},
	address = {San Francisco},
	keywords = {kojak},
	author = {Shirley Moore and Felix Wolf and Jack Dongarra and Bernd Mohr}
}
@article {,
	title = {NanoPSE: A Nanoscience Problem Solving Environment for Atomistic Electronic Structure of Semiconductor Nanostructures},
	journal = {Journal of Physics: Conference Series},
	year = {2005},
	month = {2005-06},
	pages = {277-282},
	abstract = {Researchers at the National Renewable Energy Laboratory and their collaborators have developed over the past ~10 years a set of algorithms for an atomistic description of the electronic structure of nanostructures, based on plane-wave pseudopotentials and configuration interaction. The present contribution describes the first step in assembling these various codes into a single, portable, integrated set of software packages. This package is part of an ongoing research project in the development stage. Components of NanoPSE include codes for atomistic nanostructure generation and passivation, valence force field model for atomic relaxation, code for potential field generation, empirical pseudopotential method solver, strained linear combination of bulk bands method solver, configuration interaction solver for excited states, selection of linear algebra methods, and several inverse band structure solvers. Although not available for general distribution at this time as it is being developed and tested, the design goal of the NanoPSE software is to provide a software context for collaboration. The software package is enabled by fcdev, an integrated collection of best practice GNU software for open source development and distribution augmented to better support FORTRAN.},
	doi = {https://doi.org/10.1088/1742-6596/16/1/038},
	url = {https://iopscience.iop.org/article/10.1088/1742-6596/16/1/038/meta},
	author = {Wesley B. Jones and Gabriel Bester and Andrew Canning and Alberto Franceschetti and Peter A. Graf and Kwiseon Kim and Julien Langou and Lin-Wang Wang and Jack Dongarra and Alex Zunger}
}
@inproceedings {icl:274,
	title = {A Pattern-Based Approach to Automated Application Performance Analysis},
	journal = {Workshop on Patterns in High Performance Computing},
	year = {2005},
	month = {2005-05},
	address = {University of Illinois at Urbana-Champaign},
	keywords = {kojak},
	author = {Nikhil Bhatia and Shirley Moore and Felix Wolf and Jack Dongarra and Bernd Mohr}
}
@inproceedings {icl:287,
	title = {Performance Analysis of GYRO: A Tool Evaluation},
	journal = {In Proceedings of the 2005 SciDAC Conference},
	year = {2005},
	month = {2005-06},
	address = {San Francisco, CA},
	keywords = {kojak},
	author = {Patrick H. Worley and Jeff Candy and Laura Carrington and Kevin Huck and Timothy Kaiser and Kumar Mahinthakumar and Allen D. Malony and Shirley Moore and Dan Reed and Philip C. Roth and H. Shan and Sameer Shende and Allan Snavely and S. Sreepathi and Felix Wolf and Y. Zhang}
}
@inproceedings {icl:295,
	title = {Performance Analysis of One-sided Communication Mechanisms},
	journal = {Mini-Symposium "Tools Support for Parallel Programming", Proceedings of Parallel Computing (ParCo)},
	number = {ICL-UT-06-07},
	year = {2005},
	month = {2005-09},
	address = {Malaga, Spain},
	keywords = {kojak},
	author = {Bernd Mohr and Andrej K{\"u}hnal and Marc-Andre Hermanns and Felix Wolf}
}
@conference {icl:298,
	title = {Performance Profiling and Analysis of DoD Applications using PAPI and TAU},
	booktitle = {Proceedings of DoD HPCMP UGC 2005},
	year = {2005},
	month = {2005-06},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Nashville, TN},
	keywords = {papi},
	author = {Shirley Moore and David Cronk and Felix Wolf and Avi Purkayastha and Patricia J. Teller and Robert Araiza and Gabriela Aguilera and Jamie Nava}
}
@inproceedings {icl:269,
	title = {Performance Profiling Overhead Compensation for MPI Programs},
	journal = {In Proc. of the 12th European Parallel Virtual Machine and Message Passing Interface Conference},
	year = {2005},
	month = {2005-09},
	publisher = {Springer LNCS},
	keywords = {kojak},
	author = {Sameer Shende and Allen D. Malony and Alan Morris and Felix Wolf}
}
@inproceedings {icl:270,
	title = {A Scalable Approach to MPI Application Performance Analysis},
	journal = {In Proc. of the 12th European Parallel Virtual Machine and Message Passing Interface Conference},
	year = {2005},
	month = {2005-09},
	publisher = {Springer LNCS},
	keywords = {kojak},
	author = {Shirley Moore and Felix Wolf and Jack Dongarra and Sameer Shende and Allen D. Malony and Bernd Mohr}
}
@inproceedings {icl:268,
	title = {Trace-Based Parallel Performance Overhead Compensation},
	journal = {In Proc. of the International Conference on High Performance Computing and Communications (HPCC)},
	year = {2005},
	month = {2005-09},
	address = {Sorrento (Naples), Italy},
	keywords = {kojak},
	author = {Felix Wolf and Allen D. Malony and Sameer Shende and Alan Morris}
}
@inproceedings {icl:233,
	title = {An Algebra for Cross-Experiment Performance Analysis},
	journal = {2004 International Conference on Parallel Processing (ICCP-04)},
	year = {2004},
	month = {2004-08},
	address = {Montreal, Quebec, Canada},
	keywords = {kojak},
	author = {Fengguang Song and Felix Wolf and Nikhil Bhatia and Jack Dongarra and Shirley Moore}
}
@conference {icl:239,
	title = {Automating the Large-Scale Collection and Analysis of Performance},
	booktitle = {5th LCI International Conference on Linux Clusters: The HPC Revolution},
	year = {2004},
	month = {2004-05},
	address = {Austin, Texas},
	keywords = {kojak, papi},
	author = {Phil Mucci and Jack Dongarra and Rick Kufrin and Shirley Moore and Fengguang Song and Felix Wolf}
}
@article {icl:236,
	title = {Cray X1 Evaluation Status Report},
	journal = {Oak Ridge National Laboratory Report},
	volume = {/-2004/13},
	year = {2004},
	month = {2004-01},
	author = {Pratul Agarwal and R. A. Alexander and E. Apra and Satish Balay and Arthur S. Bland and James Colgan and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Tom Dunigan and Mark Fahey and Al Geist and M. Gordon and Robert Harrison and Dinesh Kaushik and M. Krishnakumar and Piotr Luszczek and Tony Mezzacapa and Jeff Nichols and Jarek Nieplocha and Leonid Oliker and T. Packwood and M. Pindzola and Thomas C. Schulthess and Jeffrey Vetter and James B White and T. Windus and Patrick H. Worley and Thomas Zacharia}
}
@techreport {icl:196,
	title = {CUBE User Manual},
	journal = {ICL Technical Report},
	number = {ICL-UT-04-01},
	year = {2004},
	month = {2004-02},
	keywords = {kojak},
	author = {Fengguang Song and Felix Wolf}
}
@techreport {icl:243,
	title = {EARL - API Documentation},
	journal = {ICL Technical Report},
	number = {ICL-UT-04-03},
	year = {2004},
	month = {2004-10},
	keywords = {kojak},
	author = {Felix Wolf}
}
@inproceedings {icl:232,
	title = {Efficient Pattern Search in Large Traces through Successive Refinement},
	journal = {Proceedings of Euro-Par 2004},
	year = {2004},
	month = {2004-08},
	publisher = {Springer-Verlag},
	address = {Pisa, Italy},
	keywords = {kojak},
	author = {Felix Wolf and Bernd Mohr and Jack Dongarra and Shirley Moore}
}
@inproceedings {icl:238,
	title = {Self Adapting Linear Algebra Algorithms and Software},
	journal = {IEEE Proceedings (to appear)},
	year = {2004},
	month = {2004-00},
	keywords = {salsa, sans},
	author = {James Demmel and Jack Dongarra and Victor Eijkhout and Erika Fuentes and Antoine Petitet and Rich Vuduc and Clint Whaley and Katherine Yelick}
}
@article {icl:202,
	title = {The Virtual Instrument: Support for Grid-enabled Scientific Simulations},
	journal = {International Journal of High Performance Computing Applications},
	volume = {18},
	number = {1},
	year = {2004},
	month = {2004-01},
	pages = {3-17},
	author = {Henri Casanova and Thomas Bartol and Francine Berman and Adam Birnbaum and Jack Dongarra and Mark Ellisman and Marcio Faerman and Erhan Gockay and Michelle Miller and Graziano Obertelli and Stuart Pomerantz and Terry Sejnowski and Joel Stiles and Rich Wolski}
}
@article {icl:195,
	title = {Automatic performance analysis of hybrid MPI/OpenMP applications},
	journal = {Journal of Systems Architecture, Special Issue {\textquoteright}Evolutions in parallel distributed and network-based processing{\textquoteright}},
	volume = {49(10-11)},
	year = {2003},
	month = {2003-11},
	pages = {421-439},
	publisher = {Elsevier},
	keywords = {kojak},
	author = {Felix Wolf and Bernd Mohr},
	editor = {Andrea Clematis and Daniele D{\textquoteright}Agostino}
}
@article {icl:192,
	title = {Hardware-Counter Based Automatic Performance Analysis of Parallel Programs},
	journal = {Advances in Parallel Computing},
	volume = {13},
	year = {2003},
	month = {2004-01},
	pages = {753-760},
	publisher = {Elsevier},
	address = {Dresden, Germany},
	abstract = {The KOJAK performance-analysis environment identifies a large number of performance problems on parallel computers with SMP nodes. The current version concentrates on parallelism-related performance problems that arise from an inefficient usage of the parallel programming interfaces MPI and OpenMP, while ignoring individual CPU performance. This chapter describes an extended design of KOJAK capable of diagnosing low individual-CPU performance based on hardware-counter information and of integrating the results with those of the parallelism-centered analysis. The performance of parallel applications is determined by a variety of different factors. Performance of single components frequently influences the overall behavior in unexpected ways. Application programmers on current parallel machines have to deal with numerous performance-critical aspects: different modes of parallel execution, such as message passing, multi-threading or even a combination of the two, and performance on individual CPU that is determined by the interaction of different functional units. The KOJAK analysis process is composed of two parts: a semi-automatic instrumentation of the user application followed by an automatic analysis of the generated performance data. KOJAK{\textquoteright}s instrumentation software runs on most major UNlX platforms and works on multiple levels, including source-code, compiler, and linker.},
	keywords = {kojak, papi},
	doi = {https://doi.org/10.1016/S0927-5452(04)80092-3},
	author = {Felix Wolf and Bernd Mohr}
}
@inproceedings {icl:193,
	title = {KOJAK - A Tool Set for Automatic Performance Analysis of Parallel Applications},
	journal = {Proc. of the European Conference on Parallel Computing (EuroPar)},
	volume = {2790},
	year = {2003},
	month = {2003-08},
	pages = {1301-1304},
	publisher = {Springer-Verlag},
	address = {Klagenfurt, Austria},
	keywords = {kojak},
	author = {Bernd Mohr and Felix Wolf}
}
@inproceedings {icl:171,
	title = {Optimizing Performance and Reliability in Distributed Computing Systems Through Wide Spectrum Storage},
	journal = {Proceedings of the IPDPS 2003, NGS Workshop},
	year = {2003},
	month = {2003-01},
	pages = {209},
	address = {Nice, France},
	author = {James Plank and Micah Beck and Jack Dongarra and Rich Wolski and Henri Casanova}
}
@article {icl:205,
	title = {A Parallel Implementation of the Nonsymmetric QR Algorithm for Distributed Memory Architectures},
	journal = {SIAM Journal on Scientific Computing},
	volume = {24},
	number = {1},
	year = {2003},
	month = {2003-01},
	pages = {284-311},
	author = {Greg Henry and David Watkins and Jack Dongarra}
}
@inproceedings {icl:122,
	title = {The Internet BackPlane Protocol: A Study in Resource Sharing},
	journal = {Proceedings of the second IEEE/ACM International Symposium on Cluster Computing and the Grid (CCGRID 2002)},
	year = {2002},
	month = {2002-10},
	address = {Berlin, Germany},
	keywords = {ftmpi},
	author = {Alessandro Bassi and Micah Beck and Graham Fagg and Terry Moore and James Plank and Martin Swany and Rich Wolski}
}
@article {icl:101,
	title = {Middleware for the Use of Storage in Communication},
	journal = {Parallel Computing},
	volume = {28},
	number = {12},
	year = {2002},
	month = {2002-08},
	pages = {1773-1788},
	keywords = {netsolve},
	author = {Micah Beck and Dorian Arnold and Alessandro Bassi and Francine Berman and Henri Casanova and Jack Dongarra and Terry Moore and Graziano Obertelli and James Plank and Martin Swany and Sathish Vadhiyar and Rich Wolski}
}
@article {icl:114,
	title = {A Parallel Implementation of the Nonsymmetric QR Algorithm for Disitributed Memory Architectures},
	journal = {SIAM Journal on Scientific Computing},
	volume = {16},
	number = {2},
	year = {2002},
	month = {2002-10},
	pages = {284-311},
	author = {Greg Henry and David Watkins and Jack Dongarra}
}
@article {icl:125,
	title = {An Updated Set of Basic Linear Algebra Subprograms (BLAS)},
	journal = {ACM Transactions on Mathematical Software},
	volume = {28},
	number = {2},
	year = {2002},
	month = {2002-12},
	pages = {135-151},
	doi = {10.1145/567806.567807},
	author = {Susan Blackford and James Demmel and Jack Dongarra and Iain Duff and Sven Hammarling and Greg Henry and Michael Heroux and Linda Kaufman and Andrew Lumsdaine and Antoine Petitet and Roldan Pozo and Karin Remington and Clint Whaley}
}
@article {icl:95,
	title = {The Virtual Instrument: Support for Grid-enabled Scientific Simulations},
	journal = {Journal of Parallel and Distributed Computing (submitted)},
	year = {2002},
	month = {2002-10},
	author = {Henri Casanova and Thomas Bartol and Francine Berman and Adam Birnbaum and Jack Dongarra and Mark Ellisman and Marcio Faerman and Erhan Gockay and Michelle Miller and Graziano Obertelli and Stuart Pomerantz and Terry Sejnowski and Joel Stiles and Rich Wolski}
}
@article {icl:85,
	title = {Automated Empirical Optimization of Software and the ATLAS Project},
	journal = {Parallel Computing},
	volume = {27},
	number = {1-2},
	year = {2001},
	month = {2001-01},
	pages = {3-25},
	keywords = {atlas},
	author = {Clint Whaley and Antoine Petitet and Jack Dongarra}
}
@article {icl:6,
	title = {Basic Linear Algebra Subprograms (BLAS)},
	journal = {(an update), submitted to ACM TOMS},
	year = {2001},
	month = {2001-02},
	author = {Susan Blackford and James Demmel and Jack Dongarra and Iain Duff and Sven Hammarling and Greg Henry and Michael Heroux and Linda Kaufman and Andrew Lumsdaine and Antoine Petitet and Roldan Pozo and Karin Remington and Clint Whaley}
}
@article {icl:90,
	title = {The GrADS Project: Software Support for High-Level Grid Application Development},
	journal = {International Journal of High Performance Applications and Supercomputing},
	volume = {15},
	number = {4},
	year = {2001},
	month = {2001-01},
	pages = {327-344},
	keywords = {grads},
	author = {Francine Berman and Andrew Chien and Keith Cooper and Jack Dongarra and Ian Foster and Dennis Gannon and Lennart Johnsson and Ken Kennedy and Carl Kesselman and John Mellor-Crummey and Dan Reed and Linda Torczon and Rich Wolski}
}
@techreport {icl:2,
	title = {Internet Backplane Protocol: API 1.0},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-CS-01-464},
	year = {2001},
	month = {2001-01},
	author = {Alessandro Bassi and Micah Beck and James Plank and Rich Wolski}
}
@article {icl:4,
	title = {Logistical Computing and Internetworking: Middleware for the Use of Storage in Communication},
	journal = {submitted to SC2001},
	year = {2001},
	month = {2001-11},
	address = {Denver, Colorado},
	keywords = {netsolve},
	author = {Micah Beck and Dorian Arnold and Alessandro Bassi and Francine Berman and Henri Casanova and Jack Dongarra and Terry Moore and Graziano Obertelli and James Plank and Martin Swany and Sathish Vadhiyar and Rich Wolski}
}
@article {icl:12,
	title = {The Quest for Petascale Computing},
	journal = {Computing in Science and Engineering},
	volume = {3},
	number = {3},
	year = {2001},
	month = {2001-05},
	pages = {32-39},
	author = {Jack Dongarra and David W. Walker}
}
@techreport {icl:61,
	title = {Repository in a Box Toolkit for Software and Resource Sharing},
	journal = {University of Tennessee Computer Science Department Technical Report},
	number = {ICL-UT-05-05},
	year = {2001},
	month = {2001-00},
	keywords = {rib},
	author = {Shirley Browne and Paul McMahan and Scott Wells}
}
@techreport {icl:49,
	title = {Automated Empirical Optimizations of Software and the ATLAS Project (LAPACK Working Note 147)},
	journal = {University of Tennessee Computer Science Department Technical Report,},
	number = {UT-CS-00-448},
	year = {2000},
	month = {2000-09},
	keywords = {atlas},
	author = {Clint Whaley and Antoine Petitet and Jack Dongarra}
}
@techreport {icl:30,
	title = {The GrADS Project: Software Support for High-Level Grid Application Development},
	journal = {Technical Report},
	year = {2000},
	month = {2000-02},
	keywords = {grads},
	author = {Francine Berman and Andrew Chien and Keith Cooper and Jack Dongarra and Ian Foster and Dennis Gannon and Lennart Johnsson and Ken Kennedy and Carl Kesselman and Dan Reed and Linda Torczon and Rich Wolski}
}
@article {icl:118,
	title = {Message Passing Software Systems},
	journal = {Encyclopedia of Electrical and Engineering, Supplement 1},
	year = {2000},
	month = {2000-00},
	publisher = {John Wiley \& Sons, Inc.},
	keywords = {ftmpi},
	author = {Jack Dongarra and Graham Fagg and Rolf Hempel and David W. Walker},
	editor = {J. Webster}
}
@article {icl:227,
	title = {Providing Infrastructure and Interface to High Performance Applications in a Distributed Setting},
	journal = {ASTC-HPC 2000},
	year = {2000},
	month = {2000-04},
	address = {Washington, DC},
	author = {Dorian Arnold and Wonsuck Lee and Jack Dongarra and Mary Wheeler}
}
@article {icl:53,
	title = {Logistical Quality of Service in NetSolve},
	journal = {Computer Communications},
	volume = {22},
	number = {11},
	year = {1999},
	month = {1999-01},
	pages = {1034-1044},
	keywords = {netsolve},
	author = {Micah Beck and Henri Casanova and Jack Dongarra and Terry Moore and James Plank and Francine Berman and Rich Wolski}
}
@article {icl:229,
	title = {A Numerical Linear Algebra Problem Solving Environment Designer{\textquoteright}s Perspective (LAPACK Working Note 139)},
	journal = {SIAM Annual Meeting},
	year = {1999},
	month = {1999-05},
	address = {Atlanta, GA},
	author = {Antoine Petitet and Henri Casanova and Clint Whaley and Jack Dongarra and Yves Robert}
}
@article {icl:75,
	title = {Parallel and Distributed Scientific Computing: A Numerical Linear Algebra Problem Solving Environment Designer{\textquoteright}s Perspective},
	journal = {Handbook on Parallel and Distributed Processing},
	year = {1999},
	month = {1999-01},
	author = {Antoine Petitet and Henri Casanova and Jack Dongarra and Yves Robert and Clint Whaley}
}
@techreport {icl:64,
	title = {Towards An Efficient, Scalable Replication Mechanism for the I2-DSI Project},
	journal = {University of North Carolina School of Library and Information Science Technical Report},
	number = {TR-1999-01},
	year = {1999},
	month = {1999-01},
	author = {Bert Dempsey and Debra Weiss}
}
@conference {967,
	title = {Automatically Tuned Linear Algebra Software},
	booktitle = {1998 ACM/IEEE conference on Supercomputing (SC {\textquoteright}98)},
	year = {1998},
	month = {1998-11},
	publisher = {IEEE Computer Society},
	organization = {IEEE Computer Society},
	address = {Orlando, FL},
	abstract = {This paper describes an approach for the automatic generation and optimization of numerical software for processors with deep memory hierarchies and pipelined functional units. The production of such software for machines ranging from desktop workstations to embedded processors can be a tedious and time consuming process. The work described here can help in automating much of this process. We will concentrate our efforts on the widely used linear algebra kernels called the Basic Linear Algebra Subroutines (BLAS). In particular, the work presented here is for general matrix multiply, DGEMM. However much of the technology and approach developed here can be applied to the other Level 3 BLAS and the general strategy can have an impact on basic linear algebra operations in general and may be extended to other important kernel operations.},
	keywords = {BLAS, code generation, high performance, linear algebra, optimization, Tuning},
	isbn = {0-89791-984-X},
	author = {Clint Whaley and Jack Dongarra}
}
@book {1469,
	title = {MPI - The Complete Reference, Volume 1: The MPI Core},
	year = {1998},
	month = {1998-08},
	pages = {426},
	publisher = {MIT Press},
	organization = {MIT Press},
	edition = {Second},
	address = {Cambridge, MA, USA},
	abstract = {Since its release in summer 1994, the Message Passing Interface (MPI) specification has become a standard for message-passing libraries for parallel computations. There exist more than a dozen implementations on a variety of computing platforms, from the IBM SP-2 supercomputer to PCs running Windows NT. The initial MPI Standard, known as MPI-1, has been modified over the last two years. This volume, the definitive reference manual for the latest version of MPI-1, contains a complete specification of the MPI Standard. It is annotated with comments that clarify complicated issues, including why certain design choices were made, how users are intended to use the interface, and how they should construct their version of MPI. The volume also provides many detailed, illustrative programming examples.},
	isbn = {978-0-262-69215-1},
	author = {Marc Snir and Steve Otto and Steven Huss-Lederman and David Walker and Jack Dongarra}
}
@article {icl:256,
	title = {National HPCC Software Exchange (NHSE): Uniting the High Performance Computing and Communications Community},
	journal = {D-Lib Magazine},
	year = {1998},
	month = {1998-01},
	keywords = {rib},
	author = {Shirley Browne and Jack Dongarra and Jeff Horner and Paul McMahan and Scott Wells}
}
@article {1467,
	title = {ScaLAPACK: A Portable Linear Algebra Library for Distributed Memory Computers - Design Issues and Performance},
	journal = {Computer Physics Communications},
	volume = {97},
	year = {1996},
	month = {1996-08},
	pages = {1-15},
	abstract = {This paper outlines the content and performance of ScaLAPACK, a collection of mathematical software for linear algebra computations on distributed memory computers. The importance of developing standards for computational and message passing interfaces is discussed. We present the different components and building blocks of ScaLAPACK. This paper outlines the difficulties inherent in producing correct codes for networks of heterogeneous processors. We define a theoretical model of parallel computers dedicated to linear algebra applications: the Distributed Linear Algebra Machine (DLAM). This model provides a convenient framework for developing parallel algorithms and investigating their scalability, performance and programmability. Extensive performance results on various platforms are presented and analyzed with the help of the DLAM. Finally, this paper briefly describes future directions for the ScaLAPACK library and concludes by suggesting alternative approaches to mathematical libraries, explaining how ScaLAPACK could be integrated into efficient and user-friendly distributed systems.},
	doi = {https://doi.org/10.1016/0010-4655(96)00017-3},
	author = {Jaeyoung Choi and Jim Demmel and Inderjit Dhillon and Jack Dongarra and Susan Ostrouchov and Antoine Petitet and Kendall Stanley and David Walker and Clint Whaley}
}
