Recent work in unsupervised feature learning and deep learning has shown that being able to train large models can dramatically improve performance. In this paper, we consider the problem of training a deep network with billions of parameters using tens of thousands of CPU cores. We have developed a software framework called DistBelief that can utilize computing clusters with thousands of machines to train large models. Within this framework, we have developed two algorithms for large-scale distributed training: (i) Downpour SGD, an asynchronous stochastic gradient descent procedure supporting a large number of model replicas, and (ii) Sandblaster, a framework that supports a variety of distributed batch optimization procedures, including a distributed implementation of L-BFGS. Downpour SGD and Sandblaster L-BFGS both increase the scale and speed of deep network training. We have successfully used our system to train a deep network 30x larger than previously reported in the literature, and achieves state-of-the-art performance on ImageNet, a visual object recognition task with 16 million images and 21k categories. We show that these same techniques dramatically accelerate the training of a more modestly- sized deep network for a commercial speech recognition service. Although we focus on and report performance of these methods as applied to training large neural networks, the underlying algorithms are applicable to any gradient-based machine learning algorithm.
%0 Conference Paper
%1 Dean:2012:LSD:2999134.2999271
%A Dean, Jeffrey
%A Corrado, Greg S.
%A Monga, Rajat
%A Chen, Kai
%A Devin, Matthieu
%A Le, Quoc V.
%A Mao, Mark Z.
%A Ranzato, Marc'Aurelio
%A Senior, Andrew
%A Tucker, Paul
%A Yang, Ke
%A Ng, Andrew Y.
%B Proceedings of the 25th International Conference on Neural Information Processing Systems - Volume 1
%C USA
%D 2012
%I Curran Associates Inc.
%K deep-learning distributed-computing dl4j scalable-neural-netwrok
%P 1223--1231
%T Large Scale Distributed Deep Networks
%U http://dl.acm.org/citation.cfm?id=2999134.2999271
%X Recent work in unsupervised feature learning and deep learning has shown that being able to train large models can dramatically improve performance. In this paper, we consider the problem of training a deep network with billions of parameters using tens of thousands of CPU cores. We have developed a software framework called DistBelief that can utilize computing clusters with thousands of machines to train large models. Within this framework, we have developed two algorithms for large-scale distributed training: (i) Downpour SGD, an asynchronous stochastic gradient descent procedure supporting a large number of model replicas, and (ii) Sandblaster, a framework that supports a variety of distributed batch optimization procedures, including a distributed implementation of L-BFGS. Downpour SGD and Sandblaster L-BFGS both increase the scale and speed of deep network training. We have successfully used our system to train a deep network 30x larger than previously reported in the literature, and achieves state-of-the-art performance on ImageNet, a visual object recognition task with 16 million images and 21k categories. We show that these same techniques dramatically accelerate the training of a more modestly- sized deep network for a commercial speech recognition service. Although we focus on and report performance of these methods as applied to training large neural networks, the underlying algorithms are applicable to any gradient-based machine learning algorithm.
@inproceedings{Dean:2012:LSD:2999134.2999271,
abstract = {Recent work in unsupervised feature learning and deep learning has shown that being able to train large models can dramatically improve performance. In this paper, we consider the problem of training a deep network with billions of parameters using tens of thousands of CPU cores. We have developed a software framework called DistBelief that can utilize computing clusters with thousands of machines to train large models. Within this framework, we have developed two algorithms for large-scale distributed training: (i) Downpour SGD, an asynchronous stochastic gradient descent procedure supporting a large number of model replicas, and (ii) Sandblaster, a framework that supports a variety of distributed batch optimization procedures, including a distributed implementation of L-BFGS. Downpour SGD and Sandblaster L-BFGS both increase the scale and speed of deep network training. We have successfully used our system to train a deep network 30x larger than previously reported in the literature, and achieves state-of-the-art performance on ImageNet, a visual object recognition task with 16 million images and 21k categories. We show that these same techniques dramatically accelerate the training of a more modestly- sized deep network for a commercial speech recognition service. Although we focus on and report performance of these methods as applied to training large neural networks, the underlying algorithms are applicable to any gradient-based machine learning algorithm.},
acmid = {2999271},
added-at = {2018-02-10T17:45:26.000+0100},
address = {USA},
author = {Dean, Jeffrey and Corrado, Greg S. and Monga, Rajat and Chen, Kai and Devin, Matthieu and Le, Quoc V. and Mao, Mark Z. and Ranzato, Marc'Aurelio and Senior, Andrew and Tucker, Paul and Yang, Ke and Ng, Andrew Y.},
biburl = {https://www.bibsonomy.org/bibtex/22e2f1089c083ce7e238e1423a4be139f/ven7u},
booktitle = {Proceedings of the 25th International Conference on Neural Information Processing Systems - Volume 1},
description = {Large scale distributed deep networks},
interhash = {f32bac5178c09cbc0021852a4766c4cf},
intrahash = {2e2f1089c083ce7e238e1423a4be139f},
keywords = {deep-learning distributed-computing dl4j scalable-neural-netwrok},
location = {Lake Tahoe, Nevada},
numpages = {9},
pages = {1223--1231},
publisher = {Curran Associates Inc.},
series = {NIPS'12},
timestamp = {2018-02-10T17:45:26.000+0100},
title = {Large Scale Distributed Deep Networks},
url = {http://dl.acm.org/citation.cfm?id=2999134.2999271},
year = 2012
}