@presentation {rex2023ML4SEISMIClsp,
	title = {Large-scale parametric PDE approximations with model-parallel Fourier neural operators},
	year = {2023},
	month = {11},
	abstract = {Solving PDEs to simulate two-phase flow is expensive since it involves the inversion of large ill-conditioned matrices. FNOs represent a special type of neural network capable of approximating solutions to two-phase flow equations. In order to speed up this computation, we develop a high-level software abstraction tool to exploit the linearly separable property of Fourier Transforms via Kronecker products. We perform a series of all-to-all operations where we partition the data to apply the operations in a distributed fashion. We apply these FNOs to predict the evolution of CO2-plumes in subsurface environments. Our model takes an input permeability model, and outputs time-varying CO2 saturations in a quick and cost-effective manner. Additionally, our research involves developing a distributed matrix-free abstraction library that can be used to represent any generic linear and nonlinear operator. This library is scalable and auto-differentiable thanks to the hand-written customized AD rules, allowing us to represent and train any network. We provide an evaluation on the accuracy of FNO simulations compared to traditional PDE simulations in solving various classes of PDEs.},
	keywords = {CCS, deep learning, Fourier neural operators, HPC, ML4SEISMIC, SLIM},
	url = {https://slim.gatech.edu/Publications/Public/Conferences/ML4SEISMIC/2023/rex2023ML4SEISMIClsp},
	author = {Richard Rex and Thomas J. Grady II and Rishi Khan and Ziyi Yin and Felix J. Herrmann}
}
@article {louboutin2023lmi,
	title = {Learned multiphysics inversion with differentiable programming and machine learning},
	journal = {The Leading Edge},
	volume = {42},
	year = {2023},
	note = {(The Leading Edge)},
	month = {07},
	pages = {452-516},
	abstract = {We present the Seismic Laboratory for Imaging and Modeling/Monitoring (SLIM) open-source software framework for computational geophysics and, more generally, inverse problems involving the wave-equation (e.g., seismic and medical ultrasound), regularization with learned priors, and learned neural surrogates for multiphase flow simulations. By integrating multiple layers of abstraction, our software is designed to be both readable and scalable. This allows researchers to easily formulate their problems in an abstract fashion while exploiting the latest developments in high-performance computing. We illustrate and demonstrate our design principles and their benefits by means of building a scalable prototype for permeability inversion from time-lapse crosswell seismic data, which aside from coupling of wave physics and multiphase flow, involves machine learning.},
	keywords = {CCS, deep learning, Fourier neural operators, GCS, Imaging, inversion, invertible networks, machine learning, monitoring, multiphysics, Normalizing flows, SLIM, software, time-lapse},
	url = {https://library.seg.org/doi/10.1190/tle42070474.1},
	presentation = {https://slim.gatech.edu/Publications/Public/Journals/TheLeadingEdge/2023/louboutin2023lmi/le_software.pdf},
	url2 = {https://slim.gatech.edu/Publications/Public/Journals/TheLeadingEdge/2023/louboutin2023lmi/le_software.html},
	author = {Mathias Louboutin and Ziyi Yin and Rafael Orozco and Thomas J. Grady II and Ali Siahkoohi and Gabrio Rizzuti and Philipp A. Witte and Olav M{\o}yner and Gerard J. Gorman and Felix J. Herrmann}
}
@article {grady2022SCtll,
	title = {Model-Parallel Fourier Neural Operators as Learned Surrogates for Large-Scale Parametric PDEs},
	journal = {Computers \& Geosciences},
	volume = {178},
	year = {2023},
	note = {(Computers \& Geosciences)},
	month = {07},
	pages = {105402},
	abstract = {Fourier neural operators (FNOs) are a recently introduced neural network architecture for learning solution operators of partial differential equations (PDEs), which have been shown to perform significantly better than comparable deep learning approaches. Once trained, FNOs can achieve speed-ups of multiple orders of magnitude over conventional numerical PDE solvers. However, due to the high dimensionality of their input data and network weights, FNOs have so far only been applied to two-dimensional or small three-dimensional problems. To remove this limited problem-size barrier, we propose a model-parallel version of FNOs based on domain-decomposition of both the input data and network weights. We demonstrate that our model-parallel FNO is able to predict time-varying PDE solutions of over 2.6 billion variables on Perlmutter using up to 512 A100 GPUs and show an example of training a distributed FNO on the Azure cloud for simulating multiphase CO2 dynamics in the Earth{\textquoteright}s subsurface.},
	keywords = {CCS, Computers and Geosciences, deep learning, Fourier neural operators, HPC, large-scale, Model Parallelism, Multiphase Flow, Operator Learning},
	doi = {10.1016/j.cageo.2023.105402},
	url = {https://slim.gatech.edu/Publications/Public/Journals/ComputersAndGeosciences/2023/grady2022SCtll/grady2022SCtll.pdf},
	software = {https://github.com/slimgroup/dfno},
	author = {Thomas J. Grady II and Rishi Khan and Mathias Louboutin and Ziyi Yin and Philipp A. Witte and Ranveer Chandra and Russell J. Hewett and Felix J. Herrmann}
}
@conference {louboutin2022RHPCafa,
	title = {Abstractions for at-scale seismic inversion},
	booktitle = {Rice Oil and Gas High Performance Computing Conference 2022},
	year = {2022},
	note = {Rice Oil and Gas High Performance Computing Conference 2022},
	month = {03},
	pages = {Thursday Workshop: Devito Training and Hackathon},
	abstract = {We present the SLIM open-source software framework for computational geophysics, and more generally, inverse problems based on the wave-equation (e.g., medical ultrasound). We developed a software environment aimed at scalable research and development by designing multiple layers of abstractions. This environment allows the researchers to easily formulate their problem in an abstract fashion, while still being able to exploit the latest developments in high-performance computing. We illustrate and demonstrate the benefits of our software design on many geophysical applications, including seismic inversion and physics-informed machine learning for geophysics (e.g., loop unrolled imaging, uncertainty quantification), all while facilitating the integration of external software.},
	keywords = {CCS, devito, FWI, HPC, inversion, JUDI, machine learning, RHPC, software, Uncertainty quantification},
	url = {https://slim.gatech.edu/Publications/Public/Conferences/RHPC/2022/louboutin2022RHPCafa/RiceHPC22.pdf},
	url2 = {https://youtu.be/scRTbP8w6Wk?t=4542},
	author = {Mathias Louboutin and Ali Siahkoohi and Ziyi Yin and Rafael Orozco and Thomas J. Grady II and Yijun Zhang and Philipp A. Witte and Gabrio Rizzuti and Felix J. Herrmann}
}
@conference {herrmann2022EAGEcvm,
	title = {Capturing velocity-model uncertainty and two-phase flow with Fourier Neural Operators},
	booktitle = {EAGE Annual Conference Proceedings},
	year = {2022},
	note = {(EAGE, Madrid)},
	month = {06},
	pages = {AI in Geoscience and Geophysics: Current Trends and Future Prospects (Dedicated Session)},
	keywords = {CCS, EAGE, Fourier neural operators, JRM, seismic imaging, Uncertainty quantification},
	presentation = {https://slim.gatech.edu/Publications/Public/Conferences/EAGE/2022/herrmann2022EAGEcvm/index.html},
	author = {Ali Siahkoohi and Thomas J. Grady II and Abhinav Prakash Gahlot and Huseyin Tuna Erdinc and Felix J. Herrmann}
}
@presentation {grady2022ML4SEISMICesn,
	title = {Effective scaling of numerical surrogates via domain-decomposed Fourier neural operators},
	year = {2022},
	month = {11},
	abstract = {Numerical surrogates are models which learn to mimic a complex physical process (such as the solution to a PDE produced by a solver) from a set of input/output pairs. Fourier neural operators (FNOs) are a specific type of numerical surrogate which use a learned matched filter to quickly approximate solutions to relatively smooth complex physical processes. In the case of carbon capture sequestration (CCS) technology, FNOs have been shown to well-approximate solutions to the two-phase flow equations, with speedups of 1,000 to 10,000 times at inference time versus a tradtitional solver. This speed combined with the fact that FNOs are differentiable with respect to their input parameters allows for inverse and uncertainty quantification problems to theoretically be solved on real 3D data, a previously intractible task. However, due to the size of the input data, network weights, and optimizer state, FNOs have thus far been limited to small to medium 2D and 3D problems, well below the size of an industry standard such as the Sleipner benchmark. Here we alleviate this problem by proposing a model-parallel FNO which makes use of domain decomposition of the input data and network weights, and exploits architectural features of FNOs to also include a natural form of asynchronous pipeline parallelism. Our network can scale to arbitrary problem sizes on CPU and GPU systems.},
	keywords = {CCS, deep learning, Fourier neural operators, HPC, ML4SEISMIC, SLIM},
	url = {https://slim.gatech.edu/Publications/Public/Conferences/ML4SEISMIC/2022/grady2022ML4SEISMICesn/index.html},
	author = {Thomas J. Grady II and Rishi Khan and Mathias Louboutin and Ziyi Yin and Philipp A. Witte and Ranveer Chandra and Russell J. Hewett and Felix J. Herrmann}
}
@presentation {grady2021ML4SEISMICdfn,
	title = {Distributed Fourier Neural Operators},
	year = {2021},
	month = {11},
	abstract = {Fourier Neural Operators (FNOs) are a class of neural operator, which use weightings on the Fourier transform of their inputs to approximate infinite-dimensional mappings between function spaces. They are particularly useful in approximating the solutions of smooth parametric (e.g. by permeability) partial differential equations (PDEs), and once trained are capable of producing a nearly identical output to a traditional numerical solver roughly three orders of magnitude faster, making them very useful in a wide variety of engineering applications that require repeated PDE solves (e.g. during uncertainty quantification). Until now, FNOs have been limited to small problems, as their memory intensive design makes them difficult to scale in a traditional machine learning setting on a single computer or GPU. In this work, a decomposition scheme is described and implemented in PyTorch using the DistDL distributed deep learning framework, which is capable of scaling FNOs to arbitrary dimension and input size running on many nodes in a distributed memory system. This parallel implementations allows FNOs to be trained and run on problems of a practical scale on both CPU and GPU clusters.},
	keywords = {CCS, deep learning, Fourier neural operators, ML4SEISMIC, SLIM},
	url = {https://slim.gatech.edu/Publications/Public/Conferences/ML4SEISMIC/2021/grady2021ML4SEISMICdfn/Tue-12-10-Grady.pdf},
	author = {Thomas J. Grady II and Rishi Khan and Felix J. Herrmann}
}
