The generalization error of deep neural networks via their classification
margin is studied in this work. Our approach is based on the Jacobian matrix of
a deep neural network and can be applied to networks with arbitrary
non-linearities and pooling layers, and to networks with different
architectures such as feed forward networks and residual networks. Our analysis
leads to the conclusion that a bounded spectral norm of the network's Jacobian
matrix in the neighbourhood of the training samples is crucial for a deep
neural network of arbitrary depth and width to generalize well. This is a
significant improvement over the current bounds in the literature, which imply
that the generalization error grows with either the width or the depth of the
network. Moreover, it shows that the recently proposed batch normalization and
weight normalization re-parametrizations enjoy good generalization properties,
and leads to a novel network regularizer based on the network's Jacobian
matrix. The analysis is supported with experimental results on the MNIST,
CIFAR-10, LaRED and ImageNet datasets.
Description
[1605.08254] Robust Large Margin Deep Neural Networks
%0 Journal Article
%1 sokolic2016robust
%A Sokolic, Jure
%A Giryes, Raja
%A Sapiro, Guillermo
%A Rodrigues, Miguel R. D.
%D 2016
%K generalization learning readings svms
%R 10.1109/TSP.2017.2708039
%T Robust Large Margin Deep Neural Networks
%U http://arxiv.org/abs/1605.08254
%X The generalization error of deep neural networks via their classification
margin is studied in this work. Our approach is based on the Jacobian matrix of
a deep neural network and can be applied to networks with arbitrary
non-linearities and pooling layers, and to networks with different
architectures such as feed forward networks and residual networks. Our analysis
leads to the conclusion that a bounded spectral norm of the network's Jacobian
matrix in the neighbourhood of the training samples is crucial for a deep
neural network of arbitrary depth and width to generalize well. This is a
significant improvement over the current bounds in the literature, which imply
that the generalization error grows with either the width or the depth of the
network. Moreover, it shows that the recently proposed batch normalization and
weight normalization re-parametrizations enjoy good generalization properties,
and leads to a novel network regularizer based on the network's Jacobian
matrix. The analysis is supported with experimental results on the MNIST,
CIFAR-10, LaRED and ImageNet datasets.
@article{sokolic2016robust,
abstract = {The generalization error of deep neural networks via their classification
margin is studied in this work. Our approach is based on the Jacobian matrix of
a deep neural network and can be applied to networks with arbitrary
non-linearities and pooling layers, and to networks with different
architectures such as feed forward networks and residual networks. Our analysis
leads to the conclusion that a bounded spectral norm of the network's Jacobian
matrix in the neighbourhood of the training samples is crucial for a deep
neural network of arbitrary depth and width to generalize well. This is a
significant improvement over the current bounds in the literature, which imply
that the generalization error grows with either the width or the depth of the
network. Moreover, it shows that the recently proposed batch normalization and
weight normalization re-parametrizations enjoy good generalization properties,
and leads to a novel network regularizer based on the network's Jacobian
matrix. The analysis is supported with experimental results on the MNIST,
CIFAR-10, LaRED and ImageNet datasets.},
added-at = {2019-10-22T14:16:53.000+0200},
author = {Sokolic, Jure and Giryes, Raja and Sapiro, Guillermo and Rodrigues, Miguel R. D.},
biburl = {https://www.bibsonomy.org/bibtex/2dcc92f6ff8712f08d481fd8849a95443/kirk86},
description = {[1605.08254] Robust Large Margin Deep Neural Networks},
doi = {10.1109/TSP.2017.2708039},
interhash = {ec94fc8dfd9da793db5b1e124aa221f1},
intrahash = {dcc92f6ff8712f08d481fd8849a95443},
keywords = {generalization learning readings svms},
note = {cite arxiv:1605.08254Comment: accepted to IEEE Transactions on Signal Processing},
timestamp = {2019-10-22T14:16:53.000+0200},
title = {Robust Large Margin Deep Neural Networks},
url = {http://arxiv.org/abs/1605.08254},
year = 2016
}