A longstanding goal in deep learning research has been to precisely
characterize training and generalization. However, the often complex loss
landscapes of neural networks have made a theory of learning dynamics elusive.
In this work, we show that for wide neural networks the learning dynamics
simplify considerably and that, in the infinite width limit, they are governed
by a linear model obtained from the first-order Taylor expansion of the network
around its initial parameters. Furthermore, mirroring the correspondence
between wide Bayesian neural networks and Gaussian processes, gradient-based
training of wide neural networks with a squared loss produces test set
predictions drawn from a Gaussian process with a particular compositional
kernel. While these theoretical results are only exact in the infinite width
limit, we nevertheless find excellent empirical agreement between the
predictions of the original network and those of the linearized version even
for finite practically-sized networks. This agreement is robust across
different architectures, optimization methods, and loss functions.
Description
[1902.06720] Wide Neural Networks of Any Depth Evolve as Linear Models Under Gradient Descent
%0 Journal Article
%1 lee2019neural
%A Lee, Jaehoon
%A Xiao, Lechao
%A Schoenholz, Samuel S.
%A Bahri, Yasaman
%A Novak, Roman
%A Sohl-Dickstein, Jascha
%A Pennington, Jeffrey
%D 2019
%K deep-learning generalization kernels optimization readings theory
%T Wide Neural Networks of Any Depth Evolve as Linear Models Under Gradient
Descent
%U http://arxiv.org/abs/1902.06720
%X A longstanding goal in deep learning research has been to precisely
characterize training and generalization. However, the often complex loss
landscapes of neural networks have made a theory of learning dynamics elusive.
In this work, we show that for wide neural networks the learning dynamics
simplify considerably and that, in the infinite width limit, they are governed
by a linear model obtained from the first-order Taylor expansion of the network
around its initial parameters. Furthermore, mirroring the correspondence
between wide Bayesian neural networks and Gaussian processes, gradient-based
training of wide neural networks with a squared loss produces test set
predictions drawn from a Gaussian process with a particular compositional
kernel. While these theoretical results are only exact in the infinite width
limit, we nevertheless find excellent empirical agreement between the
predictions of the original network and those of the linearized version even
for finite practically-sized networks. This agreement is robust across
different architectures, optimization methods, and loss functions.
@article{lee2019neural,
abstract = {A longstanding goal in deep learning research has been to precisely
characterize training and generalization. However, the often complex loss
landscapes of neural networks have made a theory of learning dynamics elusive.
In this work, we show that for wide neural networks the learning dynamics
simplify considerably and that, in the infinite width limit, they are governed
by a linear model obtained from the first-order Taylor expansion of the network
around its initial parameters. Furthermore, mirroring the correspondence
between wide Bayesian neural networks and Gaussian processes, gradient-based
training of wide neural networks with a squared loss produces test set
predictions drawn from a Gaussian process with a particular compositional
kernel. While these theoretical results are only exact in the infinite width
limit, we nevertheless find excellent empirical agreement between the
predictions of the original network and those of the linearized version even
for finite practically-sized networks. This agreement is robust across
different architectures, optimization methods, and loss functions.},
added-at = {2019-09-25T05:57:51.000+0200},
author = {Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel S. and Bahri, Yasaman and Novak, Roman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
biburl = {https://www.bibsonomy.org/bibtex/21ac3cd2d8cc07d0326969c468e36fbe3/kirk86},
description = {[1902.06720] Wide Neural Networks of Any Depth Evolve as Linear Models Under Gradient Descent},
interhash = {cfe4ba10654a3f5d3d38bdbdc013c46b},
intrahash = {1ac3cd2d8cc07d0326969c468e36fbe3},
keywords = {deep-learning generalization kernels optimization readings theory},
note = {cite arxiv:1902.06720Comment: 11+17 pages, open-source code available at https://github.com/google/neural-tangents},
timestamp = {2019-09-25T05:58:19.000+0200},
title = {Wide Neural Networks of Any Depth Evolve as Linear Models Under Gradient
Descent},
url = {http://arxiv.org/abs/1902.06720},
year = 2019
}