@techreport {louboutin2021NIPSmcte,
	title = {Low-memory stochastic backpropagation with multi-channel randomized trace estimation},
	number = {TR-CSE-2021-1},
	year = {2021},
	month = {06},
	abstract = {Thanks to the combination of state-of-the-art accelerators and highly optimized open software frameworks, there has been tremendous progress in the performance of deep neural networks. While these developments have been responsible for many breakthroughs, progress towards solving large-scale problems, such as video encoding and semantic segmentation in 3D, is hampered because access to on-premise memory is often limited. Instead of relying on (optimal) checkpointing or invertibility of the network layers{\textendash}-to recover the activations during backpropagation{\textendash}-we propose to approximate the gradient of convolutional layers in neural networks with a multi-channel randomized trace estimation technique. Compared to other methods, this approach is simple, amenable to analyses, and leads to a greatly reduced memory footprint. Even though the randomized trace estimation introduces stochasticity during training, we argue that this is of little consequence as long as the induced errors are of the same order as errors in the gradient due to the use of stochastic gradient descent. We discuss the performance of networks trained with stochastic backpropagation and how the error can be controlled while maximizing memory usage and minimizing computational overhead.},
	keywords = {Convolutions, HPC, Low Memory, machine learning, randomized linear algebra},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2021/louboutin2021NIPSmcte/louboutin2021NIPSmcte.pdf},
	software = {https://github.com/slimgroup/XConv},
	author = {Mathias Louboutin and Ali Siahkoohi and Rongrong Wang and Felix J. Herrmann}
}
@techreport {siahkoohi2020TRfuqf,
	title = {Faster Uncertainty Quantification for Inverse Problems with Conditional Normalizing Flows},
	number = {TR-CSE-2020-2},
	year = {2020},
	month = {07},
	institution = {Georgia Institute of Technology},
	abstract = {In inverse problems, we often have access to data consisting of paired samples $(x,y)\sim p_X,Y(x,y)$ where $y$ are partial observations of a physical system, and $x$ represents the unknowns of the problem. Under these circumstances, we can employ supervised training to learn a solution $x$ and its uncertainty from the observations $y$. We refer to this problem as the "supervised" case. However, the data $y\sim p_Y(y)$ collected at one point could be distributed differently than observations $y{\textquoteright}\sim p_Y{\textquoteright}(y{\textquoteright})$, relevant for a current set of problems. In the context of Bayesian inference, we propose a two-step scheme, which makes use of normalizing flows and joint data to train a conditional generator $q_θ(x|y)$ to approximate the target posterior density $p_X|Y(x|y)$. Additionally, this preliminary phase provides a density function $q_θ(x|y)$, which can be recast as a prior for the "unsupervised" problem, e.g. when only the observations $y{\textquoteright}\sim p_Y{\textquoteright}(y{\textquoteright})$, a likelihood model $y{\textquoteright}|x$, and a prior on $x{\textquoteright}$ are known. We then train another invertible generator with output density $q{\textquoteright}_φ(x|y{\textquoteright})$ specifically for $y{\textquoteright}$, allowing us to sample from the posterior $p_X|Y{\textquoteright}(x|y{\textquoteright})$. We present some synthetic results that demonstrate considerable training speedup when reusing the pretrained network $q_θ(x|y{\textquoteright})$ as a warm start or preconditioning for approximating $p_X|Y{\textquoteright}(x|y{\textquoteright})$, instead of learning from scratch. This training modality can be interpreted as an instance of transfer learning. This result is particularly relevant for large-scale inverse problems that employ expensive numerical simulations.},
	keywords = {deep learning, invertible networks, Uncertainty quantification},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2020/siahkoohi2020TRfuqf/siahkoohi2020TRfuqf.html},
	author = {Ali Siahkoohi and Gabrio Rizzuti and Philipp A. Witte and Felix J. Herrmann}
}
@techreport {sharan2020lsh,
	title = {Large scale high-frequency wavefield reconstruction with recursively weighted matrix factorizations},
	number = {TR-CSE-2020-4},
	year = {2020},
	month = {10},
	abstract = {Acquiring seismic data on a regular periodic fine grid is challenging. By exploiting the low-rank approximation property of fully sampled seismic data in some transform domain, low-rank matrix completion offers a scalable way to reconstruct seismic data on a regular periodic fine grid from coarsely randomly sampled data acquired in the field. While wavefield reconstruction have been applied successfully at the lower end of the spectrum, its performance deteriorates at the higher frequencies where the low-rank assumption no longer holds rendering this type of wavefield reconstruction ineffective in situations where high resolution images are desired. We overcome this shortcoming by exploiting similarities between adjacent frequency slices explicitly. During low-rank matrix factorization, these similarities translate to alignment of subspaces of the factors, a notion we propose to employ as we reconstruct monochromatic frequency slices recursively starting at the low frequencies. While this idea is relatively simple in its core, to turn this recent insight into a successful scalable wavefield reconstruction scheme for 3D seismic requires a number of important steps. First, we need to move the weighting matrices, which encapsulate the prior information from adjacent frequency slices, from the objective to the data misfit constraint. This move considerably improves the performance of the weighted low-rank matrix factorization on which our wavefield reconstructions is based. Secondly, we introduce approximations that allow us to decouple computations on a row-by-row and column-by-column basis, which in turn allow to parallelize the alternating optimization on which our low-rank factorization relies. The combination of weighting and decoupling leads to a computationally feasible full-azimuth wavefield reconstruction scheme that scales to industry-scale problem sizes. We demonstrate the performance of the proposed parallel algorithm on a 2D field data and on a 3D synthetic dataset. In both cases our approach produces high-fidelity broadband wavefield reconstructions from severely subsampled data.},
	keywords = {5D reconstruction, compressed sensing, frequency-domain, parallel, signal processing},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2020/sharan2020lsh/sharan2020lsh.html},
	author = {Shashin Sharan and Yijun Zhang and Oscar Lopez and Felix J. Herrmann}
}
@techreport {siahkoohi2019TRnna,
	title = {Neural network augmented wave-equation simulation},
	number = {TR-CSE-2019-1},
	year = {2019},
	month = {09},
	institution = {Georgia Institute of Technology},
	abstract = {Accurate forward modeling is important for solving inverse problems. An inaccurate wave-equation simulation, as a forward operator, will offset the results obtained via inversion. In this work, we consider the case where we deal with incomplete physics. One proxy of incomplete physics is an inaccurate discretization of Laplacian in simulation of wave equation via finite-difference method. We exploit intrinsic one-to-one similarities between timestepping algorithm with Convolutional Neural Networks (CNNs), and propose to intersperse CNNs between low-fidelity timesteps. Augmenting neural networks with low-fidelity timestepping algorithms may allow us to take large timesteps while limiting the numerical dispersion artifacts. While simulating the wave-equation with low-fidelity timestepping algorithm, by correcting the wavefield several time during propagation, we hope to limit the numerical dispersion artifact introduced by a poor discretization of the Laplacian. As a proof of concept, we demonstrate this principle by correcting for numerical dispersion by keeping the velocity model fixed, and varying the source locations to generate training and testing pairs for our supervised learning algorithm.},
	keywords = {deep learning, finite difference, wave equation},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2019/siahkoohi2019TRnna/siahkoohi2019TRnna.html},
	url2 = {https://arxiv.org/pdf/1910.00925.pdf},
	author = {Ali Siahkoohi and Mathias Louboutin and Felix J. Herrmann}
}
@techreport {peters2015EAGErwi,
	title = {Regularizing waveform inversion by projection onto intersections of convex sets},
	number = {TR-EOAS-2015-4},
	year = {2015},
	month = {01},
	institution = {UBC},
	abstract = {A framework is proposed for regularizing the waveform inversion problem by projections onto intersections of convex sets. Multiple pieces of prior information about the geology are represented by multiple convex sets, for example limits on the velocity or minimum smoothness conditions on the model. The data-misfit is then minimized, such that the estimated model is always in the intersection of the convex sets. Therefore, it is clear what properties the estimated model will have at each iteration. This approach does not require any quadratic penalties to be used and thus avoids the known problems and limitations of those types of penalties. It is shown that by formulating waveform inversion as a constrained problem, regularization ideas such as Tikhonov regularization and gradient filtering can be incorporated into one framework. The algorithm is generally applicable, in the sense that it works with any (differentiable) objective function, several gradient and quasi-Newton based solvers and does not require significant additional computation. The method is demonstrated on the inversion of very noisy synthetic data and vertical seismic profiling field data.},
	keywords = {convex sets, regularization, waveform inversion},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2015/peters2015EAGErwi/peters2015EAGErwi.html},
	author = {Bas Peters and Brendan R. Smithyman and Felix J. Herrmann}
}
@techreport {peters2015SEGrwi,
	title = {Regularizing waveform inversion by projections onto convex sets {\textendash}- application to the 2D Chevron 2014 synthetic blind-test dataset},
	number = {TR-EOAS-2015-7},
	year = {2015},
	month = {06},
	institution = {UBC},
	abstract = {A framework is proposed for regularizing the waveform inversion problem by projections onto intersections of convex sets. Multiple pieces of prior information about the geology are represented by multiple convex sets, for example limits on the velocity or minimum smoothness conditions on the model. The data-misfit is then minimized, such that the estimated model is always in the intersection of the convex sets. Therefore, it is clear what properties the estimated model will have at each iteration. This approach does not require any quadratic penalties to be used and thus avoids the known problems and limitations of those types of penalties. It is shown that by formulating waveform inversion as a constrained problem, regularization ideas such as Tikhonov regularization and gradient filtering can be incorporated into one framework. The algorithm is generally applicable, in the sense that it works with any (differentiable) objective function and does not require significant additional computation. The method is demonstrated on the inversion of the 2D marine isotropic elastic synthetic seismic benchmark by Chevron using an acoustic modeling code. To highlight the effect of the projections, we apply no data pre-processing.},
	keywords = {blind-test, projection, regularization, SEG, Wavefield Reconstruction Inversion, waveform inversion},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2015/peters2015SEGrwi/peters2015SEGrwi.html},
	author = {Bas Peters and Zhilong Fang and Brendan R. Smithyman and Felix J. Herrmann}
}
@techreport {smithyman2014SEGjfw,
	title = {Joint full-waveform inversion of on-land surface and VSP data from the Permian Basin},
	number = {TR-EOAS-2014-4},
	year = {2014},
	month = {04},
	institution = {UBC},
	abstract = {Full-waveform Inversion is applied to generate a high-resolution model of P-wave velocity for a site in the Permian Basin, Texas, USA. This investigation jointly inverts seismic waveforms from a surface 3-D vibroseis surface seismic survey and a co-located 3-D Vertical Seismic Profiling (VSP) survey, which shared common source Vibration Points (VPs). The resulting velocity model captures features that were not resolvable by conventional migration velocity analysis.},
	keywords = {downhole receivers, Full-waveform inversion, land, seismic, vibroseis},
	url = {https://slim.gatech.edu/Publications/Public/TechReport/2014/smithyman2014SEGjfw/smithyman2014SEGjfw.html},
	author = {Brendan R. Smithyman and Bas Peters and Bryan DeVault and Felix J. Herrmann}
}
@techreport {tang09TRdtr,
	title = {Design of two-dimensional randomized sampling schemes for curvelet-based sparsity-promoting seismic data recovery},
	number = {TR-EOAS-2009-3},
	year = {2009},
	institution = {UBC Earth and Ocean Sciences Department},
	abstract = {The tasks of sampling, compression and reconstruction are very common and often necessary in seismic data processing due to the large size of seismic data. Curvelet-based Recovery by Sparsity-promoting Inversion, motivated by the newly developed theory of compressive sensing, is among the best recovery strategies for seismic data. The incomplete data input to this curvelet-based recovery is determined by randomized sampling of the original complete data. Unlike usual regular undersampling, randomized sampling can convert aliases to easy-to-eliminate noise, thus facilitating the process of reconstruction of the complete data from the incomplete data. Randomized sampling methods such as jittered sampling have been developed in the past that are suitable for curvelet-based recovery, however most have only been applied to sampling in one dimension. Considering that seismic datasets are usually higher dimensional and extremely large, in the present paper, we extend the 1D version of jittered sampling to two dimensions, both with underlying Cartesian and hexagonal grids. We also study separable and non-separable two dimensional jittered sampling, the former referring to the Kronecker product of two one-dimensional jittered samplings. These different categories of jittered sampling are compared against one another in terms of signal-to-noise ratio and visual quality, from which we find that jittered hexagonal sampling is better than jittered Cartesian sampling, while fully non-separable jittered sampling is better than separable sampling. Because in the image processing and computer graphics literature, sampling patterns with blue-noise spectra are found to be ideal to avoid aliasing, we also introduce two other randomized sampling methods, possessing sampling spectra with beneficial blue noise characteristics, Poisson Disk sampling and Farthest Point sampling. We compare these methods, and apply the introduced sampling methodologies to higher dimensional curvelet-based reconstruction. These sampling schemes are shown to lead to better results from CRSI compared to the other more traditional sampling protocols, e.g. regular subsampling.},
	url = {https://slim.gatech.edu/Publications/Public/Journals/2009/tang09TRdtr/tang09TRdtr.pdf},
	author = {Gang Tang and Reza Shahidi and Jianwei Ma}
}
@techreport {vandenberg08gsv,
	title = {Group sparsity via linear-time projection},
	number = {TR-2008-09},
	year = {2008},
	month = {06},
	institution = {UBC - Department of Computer Science},
	abstract = {We present an efficient spectral projected-gradient algorithm for optimization subject to a group one-norm constraint. Our approach is based on a novel linear-time algorithm for Euclidean projection onto the one- and group one-norm constraints. Numerical experiments on large data sets suggest that the proposed method is substantially more efficient and scalable than existing methods.},
	keywords = {Optimization, SLIM},
	url = {http://www.optimization-online.org/DB_FILE/2008/07/2056.pdf},
	author = {Ewout van den Berg and Mark Schmidt and Michael P. Friedlander and K. Murphy}
}
