We consider the question of what functions can be captured by ReLU networks
with an unbounded number of units (infinite width), but where the overall
network Euclidean norm (sum of squares of all weights in the system, except for
an unregularized bias term for each unit) is bounded; or equivalently what is
the minimal norm required to approximate a given function. For functions $f :
R R$ and a single hidden layer, we show that the
minimal network norm for representing $f$ is $\max(|f''(x)| dx,
|f'(-ınfty) + f'(+ınfty)|)$, and hence the minimal norm fit for a sample is
given by a linear spline interpolation.
Description
[1902.05040] How do infinite width bounded norm networks look in function space?
%0 Journal Article
%1 savarese2019infinite
%A Savarese, Pedro
%A Evron, Itay
%A Soudry, Daniel
%A Srebro, Nathan
%D 2019
%K deep-learning foundations machine-learning stable theory
%T How do infinite width bounded norm networks look in function space?
%U http://arxiv.org/abs/1902.05040
%X We consider the question of what functions can be captured by ReLU networks
with an unbounded number of units (infinite width), but where the overall
network Euclidean norm (sum of squares of all weights in the system, except for
an unregularized bias term for each unit) is bounded; or equivalently what is
the minimal norm required to approximate a given function. For functions $f :
R R$ and a single hidden layer, we show that the
minimal network norm for representing $f$ is $\max(|f''(x)| dx,
|f'(-ınfty) + f'(+ınfty)|)$, and hence the minimal norm fit for a sample is
given by a linear spline interpolation.
@article{savarese2019infinite,
abstract = {We consider the question of what functions can be captured by ReLU networks
with an unbounded number of units (infinite width), but where the overall
network Euclidean norm (sum of squares of all weights in the system, except for
an unregularized bias term for each unit) is bounded; or equivalently what is
the minimal norm required to approximate a given function. For functions $f :
\mathbb R \rightarrow \mathbb R$ and a single hidden layer, we show that the
minimal network norm for representing $f$ is $\max(\int |f''(x)| dx,
|f'(-\infty) + f'(+\infty)|)$, and hence the minimal norm fit for a sample is
given by a linear spline interpolation.},
added-at = {2019-06-10T12:10:21.000+0200},
author = {Savarese, Pedro and Evron, Itay and Soudry, Daniel and Srebro, Nathan},
biburl = {https://www.bibsonomy.org/bibtex/262fd20a6d9b392e46621be1022b869a8/kirk86},
description = {[1902.05040] How do infinite width bounded norm networks look in function space?},
interhash = {4f244dc99785b912561df14dfc8718bd},
intrahash = {62fd20a6d9b392e46621be1022b869a8},
keywords = {deep-learning foundations machine-learning stable theory},
note = {cite arxiv:1902.05040},
timestamp = {2019-06-10T12:10:21.000+0200},
title = {How do infinite width bounded norm networks look in function space?},
url = {http://arxiv.org/abs/1902.05040},
year = 2019
}