@techreport {louboutin2021NIPSmcte,
	title = {Low-memory stochastic backpropagation with multi-channel randomized trace estimation},
	number = {TR-CSE-2021-1},
	year = {2021},
	month = {06},
	abstract = {Thanks to the combination of state-of-the-art accelerators and highly optimized open software frameworks, there has been tremendous progress in the performance of deep neural networks. While these developments have been responsible for many breakthroughs, progress towards solving large-scale problems, such as video encoding and semantic segmentation in 3D, is hampered because access to on-premise memory is often limited. Instead of relying on (optimal) checkpointing or invertibility of the network layers{\textendash}-to recover the activations during backpropagation{\textendash}-we propose to approximate the gradient of convolutional layers in neural networks with a multi-channel randomized trace estimation technique. Compared to other methods, this approach is simple, amenable to analyses, and leads to a greatly reduced memory footprint. Even though the randomized trace estimation introduces stochasticity during training, we argue that this is of little consequence as long as the induced errors are of the same order as errors in the gradient due to the use of stochastic gradient descent. We discuss the performance of networks trained with stochastic backpropagation and how the error can be controlled while maximizing memory usage and minimizing computational overhead.},
	keywords = {Convolutions, HPC, Low Memory, machine learning, randomized linear algebra},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2021/louboutin2021NIPSmcte/louboutin2021NIPSmcte.pdf},
	software = {https://github.com/slimgroup/XConv},
	author = {Mathias Louboutin and Ali Siahkoohi and Rongrong Wang and Felix J. Herrmann}
}
@article {yang2020lrpo,
	title = {Low-rank representation of omnidirectional subsurface extended image volumes},
	journal = {Geophysics},
	volume = {86},
	number = {3},
	year = {2021},
	note = {(Geophysics)},
	month = {1},
	pages = {1MJ-WA152},
	abstract = {Subsurface-offset gathers play an increasingly important role in seismic imaging. These gathers are used during velocity model building and inversion of rock properties from amplitude variations. While powerful, these gathers come with high computational and storage demands to form and manipulate these high dimensional objects. This explains why only limited numbers of image gathers are computed over a limited offset range. We avoid these high costs by working with highly compressed low-rank factorizations. We arrive at these factorizations via a combination of probings with the double two-way wave equation and randomized singular value decompositions. In turn, the resulting factorizations give us access to all subsurface offsets without having to form the full extended image volumes. The latter is computationally prohibitive because extended image volumes are quadratic in image size. As a result, we can easily handle situations where conventional horizontal offset gathers are no longer focused. More importantly, the factorization also provides a mechanism to use the invariance relation of extended image volumes for velocity continuation. With this technique, extended image volumes for one background velocity model can directly be mapped to those of another background velocity model. Our low-rank factorization inherits this invariance property so we incur factorization costs only once when examining different imaging scenarios. Because all imaging experiments only involve the factors, they are computationally cheap with costs that scale with the rank of the factorization. We validate our methodology on 2D synthetics including a challenging imaging example with salt. Our experiments show that our low-rank factorization parameterizes extended image volumes naturally. Instead of brute force explicit cross-correlations between shifted source and receiver wavefields, our approach relies on the underlying linear-algebra structure that enables us to work with these objects without incurring unfeasible demands on computation and storage.},
	keywords = {extended image volumes, invariance relationship, low rank, power schemes, randomized linear algebra},
	doi = {10.1190/geo2020-0152.1},
	url = {https://slim.gatech.edu/Publications/Public/Journals/Geophysics/2021/yang2020lrpo/Paper_final.html},
	author = {Mengmeng Yang and Marie Graff and Rajiv Kumar and Felix J. Herrmann}
}
@conference {herrmann2021Microsoftrla,
	title = {ML@scale using randomized linear algebra},
	booktitle = {Microsoft},
	year = {2021},
	note = {Talk at Microsoft},
	month = {03},
	abstract = {Deep Learning for large-scale applications such as video encoding or seismic segmentation are challenged by the excessive amounts of memory that is required for training networks via backpropagation. In this talk, I will discuss how techniques from randomized linear algebra can be used to address these bottle necks and bring down the memory footprint of training CNNs by up to a factor of O(N) (where N is number of pixels) without increasing computational cost. Additionally, I will illustrate how the seemingly disparate technologies of deep learning and large-scale PDE-constrained optimization share important similarities that can be taken advantage of in the development of next-generation deep learning technologies, with possible applications in scientific computing and sustainability.},
	keywords = {deep learning, randomized linear algebra, Uncertainty quantification},
	presentation = {https://slim.gatech.edu/Publications/Public/Conferences/Microsoft/2021/herrmann2021Microsoftrla/herrmann2021Microsoftrla_pres.pdf},
	author = {Felix J. Herrmann and Mathias Louboutin and Ali Siahkoohi}
}
@presentation {louboutin2021ML4SEISMICrla,
	title = {Randomized linear algebra for inversion},
	year = {2021},
	month = {11},
	abstract = {Inverse problems in exploration geophysics or machine learning heavily relies on linear algebra and large matrices manipulations. To tackle the growing cost of storing these matrices, randomized algorithms have been developed to obtain information from these matrices via randomized sketching. Inspired by previous work on extended image volumes, we will first show in this talk how the seismic imaging condition can be expressed in a randomized linear algebra framework leading to drastic memory savings. In a second part, we will extend this idea to convolutional neural networks to reduce the memory cost of training by orders of magnitude. We will demonstrate the practicality of these methods on representative examples.},
	keywords = {FWI, HPC, inversion, ML4SEISMIC, randomized linear algebra, SLIM, software},
	url = {https://slim.gatech.edu/Publications/Public/Conferences/ML4SEISMIC/2021/louboutin2021ML4SEISMICrla/Tue-10-20-Louboutin.pdf},
	author = {Mathias Louboutin and Felix J. Herrmann}
}
@conference {louboutin2021SEGulm,
	title = {Ultra-low memory seismic inversion with randomized trace estimation},
	booktitle = {SEG Technical Program Expanded Abstracts},
	year = {2021},
	note = {(IMAGE, Denver)},
	month = {09},
	pages = {787-791},
	abstract = {Inspired by recent work on extended image volumes that lays the ground for randomized probing of extremely large seismic wavefield matrices, we present a memory frugal and computationally efficient inversion methodology that uses techniques from randomized linear algebra. By means of a carefully selected realistic synthetic example, we demonstrate that we are capable of achieving competitive inversion results at a fraction of the memory cost of conventional full-waveform inversion with limited computational overhead. By exchanging memory for negligible computational overhead, we open with the presented technology the door towards the use of low-memory accelerators such as GPUs.},
	keywords = {FWI, HPC, inversion, randomized linear algebra, SEG},
	doi = {10.1190/segam2021-3584072.1},
	url = {https://slim.gatech.edu/Publications/Public/Conferences/SEG/2021/louboutin2021SEGulm/louboutinp.html},
	url2 = {https://slim.gatech.edu/Publications/Public/Conferences/SEG/2021/louboutin2021SEGulm/louboutin2021SEGulm.mp4},
	software = {https://github.com/slimgroup/TimeProbeSeismic.jl},
	author = {Mathias Louboutin and Felix J. Herrmann}
}
@article {kumar2018toi,
	title = {Target-oriented imaging using extended image volumes{\textemdash}a low-rank factorization approach},
	journal = {Geophysical Prospecting},
	volume = {67},
	number = {5},
	year = {2019},
	note = {(Geophysical Prospecting)},
	pages = {1312-1328},
	abstract = {Imaging in geological challenging environments has led to new developments, including the idea of generating reflection responses by means of interferometric redatuming at a given target datum in the subsurface, when the target datum lies beneath a complex overburden. One way to perform this redatuming is via conventional model-based wave-equation techniques. But those techniques can be computationally expensive for large-scale seismic problems since the number of wave-equation solves is equal to two-times the number of sources involved during seismic data acquisition. Also conventional shot-profile techniques require lots of memory to save full subsurface extended image volumes. Therefore, they only form subsurface image volumes in either horizontal or vertical directions. We now present a randomized singular value decomposition based approach built upon the matrix probing scheme, which takes advantage of the algebraic structure of the extended imaging system. This low-rank representation enables us to overcome both the computational cost associated with the number of wave-equation solutions and memory usage due to explicit storage of full subsurface extended image volumes employed by conventional migration methods. Experimental results on complex geological models demonstrate the efficacy of the proposed methodology and allow practical reflection-based extended imaging for large-scale 5D seismic data.},
	keywords = {extended image volumes, randomized linear algebra, target-oriented imaging},
	doi = {10.1111/1365-2478.12779},
	url = {https://slim.gatech.edu/Publications/Public/Journals/GeophysicalProspecting/2019/kumar2018toi/kumar2018toi.html},
	author = {Rajiv Kumar and Marie Graff-Kray and Ivan Vasconcelos and Felix J. Herrmann}
}
@conference {peters2015PRECONasl,
	title = {An algorithm for solving least-squares problems with a Helmholtz block and multiple right-hand-sides},
	booktitle = {International Conference On Preconditioning Techniques For Scientific And Industrial Applications},
	year = {2015},
	note = {(PRECON, The Netherlands)},
	month = {06},
	keywords = {CGMN, Helmholtz, least-squares, matrix-free, randomized linear algebra},
	url = {https://slim.gatech.edu/Publications/Public/Conferences/PRECON/2015/peters2015PRECONasl/peters2015PRECONasl_pres.pdf},
	author = {Bas Peters and Chen Greif and Felix J. Herrmann}
}
