We consider learning two layer neural networks using stochastic gradient
descent. The mean-field description of this learning dynamics approximates the
evolution of the network weights by an evolution in the space of probability
distributions in $R^D$ (where $D$ is the number of parameters associated to
each neuron). This evolution can be defined through a partial differential
equation or, equivalently, as the gradient flow in the Wasserstein space of
probability distributions. Earlier work shows that (under some regularity
assumptions), the mean field description is accurate as soon as the number of
hidden units is much larger than the dimension $D$. In this paper we establish
stronger and more general approximation guarantees. First of all, we show that
the number of hidden units only needs to be larger than a quantity dependent on
the regularity properties of the data, and independent of the dimensions. Next,
we generalize this analysis to the case of unbounded activation functions,
which was not covered by earlier bounds. We extend our results to noisy
stochastic gradient descent.
Finally, we show that kernel ridge regression can be recovered as a special
limit of the mean field analysis.
Описание
[1902.06015] Mean-field theory of two-layers neural networks: dimension-free bounds and kernel limit
%0 Journal Article
%1 mei2019meanfield
%A Mei, Song
%A Misiakiewicz, Theodor
%A Montanari, Andrea
%D 2019
%K approximate dynamic generalization optimization probability readings
%T Mean-field theory of two-layers neural networks: dimension-free bounds
and kernel limit
%U http://arxiv.org/abs/1902.06015
%X We consider learning two layer neural networks using stochastic gradient
descent. The mean-field description of this learning dynamics approximates the
evolution of the network weights by an evolution in the space of probability
distributions in $R^D$ (where $D$ is the number of parameters associated to
each neuron). This evolution can be defined through a partial differential
equation or, equivalently, as the gradient flow in the Wasserstein space of
probability distributions. Earlier work shows that (under some regularity
assumptions), the mean field description is accurate as soon as the number of
hidden units is much larger than the dimension $D$. In this paper we establish
stronger and more general approximation guarantees. First of all, we show that
the number of hidden units only needs to be larger than a quantity dependent on
the regularity properties of the data, and independent of the dimensions. Next,
we generalize this analysis to the case of unbounded activation functions,
which was not covered by earlier bounds. We extend our results to noisy
stochastic gradient descent.
Finally, we show that kernel ridge regression can be recovered as a special
limit of the mean field analysis.
@article{mei2019meanfield,
abstract = {We consider learning two layer neural networks using stochastic gradient
descent. The mean-field description of this learning dynamics approximates the
evolution of the network weights by an evolution in the space of probability
distributions in $R^D$ (where $D$ is the number of parameters associated to
each neuron). This evolution can be defined through a partial differential
equation or, equivalently, as the gradient flow in the Wasserstein space of
probability distributions. Earlier work shows that (under some regularity
assumptions), the mean field description is accurate as soon as the number of
hidden units is much larger than the dimension $D$. In this paper we establish
stronger and more general approximation guarantees. First of all, we show that
the number of hidden units only needs to be larger than a quantity dependent on
the regularity properties of the data, and independent of the dimensions. Next,
we generalize this analysis to the case of unbounded activation functions,
which was not covered by earlier bounds. We extend our results to noisy
stochastic gradient descent.
Finally, we show that kernel ridge regression can be recovered as a special
limit of the mean field analysis.},
added-at = {2019-09-26T15:18:02.000+0200},
author = {Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
biburl = {https://www.bibsonomy.org/bibtex/2a5d68ca685aaaaf39d67d0013f8993ab/kirk86},
description = {[1902.06015] Mean-field theory of two-layers neural networks: dimension-free bounds and kernel limit},
interhash = {197edaf5b69b663cb6ecaf5c0e51173d},
intrahash = {a5d68ca685aaaaf39d67d0013f8993ab},
keywords = {approximate dynamic generalization optimization probability readings},
note = {cite arxiv:1902.06015Comment: 61 pages},
timestamp = {2019-09-26T15:18:40.000+0200},
title = {Mean-field theory of two-layers neural networks: dimension-free bounds
and kernel limit},
url = {http://arxiv.org/abs/1902.06015},
year = 2019
}