@techreport {louboutin2021NIPSmcte,
	title = {Low-memory stochastic backpropagation with multi-channel randomized trace estimation},
	number = {TR-CSE-2021-1},
	year = {2021},
	month = {06},
	abstract = {Thanks to the combination of state-of-the-art accelerators and highly optimized open software frameworks, there has been tremendous progress in the performance of deep neural networks. While these developments have been responsible for many breakthroughs, progress towards solving large-scale problems, such as video encoding and semantic segmentation in 3D, is hampered because access to on-premise memory is often limited. Instead of relying on (optimal) checkpointing or invertibility of the network layers{\textendash}-to recover the activations during backpropagation{\textendash}-we propose to approximate the gradient of convolutional layers in neural networks with a multi-channel randomized trace estimation technique. Compared to other methods, this approach is simple, amenable to analyses, and leads to a greatly reduced memory footprint. Even though the randomized trace estimation introduces stochasticity during training, we argue that this is of little consequence as long as the induced errors are of the same order as errors in the gradient due to the use of stochastic gradient descent. We discuss the performance of networks trained with stochastic backpropagation and how the error can be controlled while maximizing memory usage and minimizing computational overhead.},
	keywords = {Convolutions, HPC, Low Memory, machine learning, randomized linear algebra},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2021/louboutin2021NIPSmcte/louboutin2021NIPSmcte.pdf},
	software = {https://github.com/slimgroup/XConv},
	author = {Mathias Louboutin and Ali Siahkoohi and Rongrong Wang and Felix J. Herrmann}
}
@conference {herrmann2021Microsoftrla,
	title = {ML@scale using randomized linear algebra},
	booktitle = {Microsoft},
	year = {2021},
	note = {Talk at Microsoft},
	month = {03},
	abstract = {Deep Learning for large-scale applications such as video encoding or seismic segmentation are challenged by the excessive amounts of memory that is required for training networks via backpropagation. In this talk, I will discuss how techniques from randomized linear algebra can be used to address these bottle necks and bring down the memory footprint of training CNNs by up to a factor of O(N) (where N is number of pixels) without increasing computational cost. Additionally, I will illustrate how the seemingly disparate technologies of deep learning and large-scale PDE-constrained optimization share important similarities that can be taken advantage of in the development of next-generation deep learning technologies, with possible applications in scientific computing and sustainability.},
	keywords = {deep learning, randomized linear algebra, Uncertainty quantification},
	presentation = {https://slim.gatech.edu/Publications/Public/Conferences/Microsoft/2021/herrmann2021Microsoftrla/herrmann2021Microsoftrla_pres.pdf},
	author = {Felix J. Herrmann and Mathias Louboutin and Ali Siahkoohi}
}
