%0 Generic %1 zhang2019cyclical %A Zhang, Ruqi %A Li, Chunyuan %A Zhang, Jianyi %A Chen, Changyou %A Wilson, Andrew Gordon %D 2019 %K bayesian cyclical_stochastic_gradient deep_learning machine_learning %T Cyclical Stochastic Gradient MCMC for Bayesian Deep Learning %U http://arxiv.org/abs/1902.03932 %X The posteriors over neural network weights are high dimensional and multimodal. Each mode typically characterizes a meaningfully different representation of the data. We develop Cyclical Stochastic Gradient MCMC (SG-MCMC) to automatically explore such distributions. In particular, we propose a cyclical stepsize schedule, where larger steps discover new modes, and smaller steps characterize each mode. We prove that our proposed learning rate schedule provides faster convergence to samples from a stationary distribution than SG-MCMC with standard decaying schedules. Moreover, we provide extensive experimental results to demonstrate the effectiveness of cyclical SG-MCMC in learning complex multimodal distributions, especially for fully Bayesian inference with modern deep neural networks.

@misc{zhang2019cyclical, abstract = {The posteriors over neural network weights are high dimensional and multimodal. Each mode typically characterizes a meaningfully different representation of the data. We develop Cyclical Stochastic Gradient MCMC (SG-MCMC) to automatically explore such distributions. In particular, we propose a cyclical stepsize schedule, where larger steps discover new modes, and smaller steps characterize each mode. We prove that our proposed learning rate schedule provides faster convergence to samples from a stationary distribution than SG-MCMC with standard decaying schedules. Moreover, we provide extensive experimental results to demonstrate the effectiveness of cyclical SG-MCMC in learning complex multimodal distributions, especially for fully Bayesian inference with modern deep neural networks.}, added-at = {2019-02-14T15:18:41.000+0100}, author = {Zhang, Ruqi and Li, Chunyuan and Zhang, Jianyi and Chen, Changyou and Wilson, Andrew Gordon}, biburl = {https://www.bibsonomy.org/bibtex/2c73d9fa7093fb03aa3651ba4de01d85d/jpvaldes}, interhash = {08be32949e3d48d946d4b6cc90c88b5a}, intrahash = {c73d9fa7093fb03aa3651ba4de01d85d}, keywords = {bayesian cyclical_stochastic_gradient deep_learning machine_learning}, note = {cite arxiv:1902.03932}, timestamp = {2019-02-14T15:18:41.000+0100}, title = {Cyclical Stochastic Gradient MCMC for Bayesian Deep Learning}, url = {http://arxiv.org/abs/1902.03932}, year = 2019 }