We provide an information-theoretic framework for studying the generalization
properties of machine learning algorithms. Our framework ties together existing
approaches, including uniform convergence bounds and recent methods for
adaptive data analysis. Specifically, we use Conditional Mutual Information
(CMI) to quantify how well the input (i.e., the training data) can be
recognized given the output (i.e., the trained model) of the learning
algorithm. We show that bounds on CMI can be obtained from VC dimension,
compression schemes, differential privacy, and other methods. We then show that
bounded CMI implies various forms of generalization.
Beschreibung
[2001.09122] Reasoning About Generalization via Conditional Mutual Information
%0 Journal Article
%1 steinke2020reasoning
%A Steinke, Thomas
%A Zakynthinou, Lydia
%D 2020
%K deep-learning generalization readings
%T Reasoning About Generalization via Conditional Mutual Information
%U http://arxiv.org/abs/2001.09122
%X We provide an information-theoretic framework for studying the generalization
properties of machine learning algorithms. Our framework ties together existing
approaches, including uniform convergence bounds and recent methods for
adaptive data analysis. Specifically, we use Conditional Mutual Information
(CMI) to quantify how well the input (i.e., the training data) can be
recognized given the output (i.e., the trained model) of the learning
algorithm. We show that bounds on CMI can be obtained from VC dimension,
compression schemes, differential privacy, and other methods. We then show that
bounded CMI implies various forms of generalization.
@article{steinke2020reasoning,
abstract = {We provide an information-theoretic framework for studying the generalization
properties of machine learning algorithms. Our framework ties together existing
approaches, including uniform convergence bounds and recent methods for
adaptive data analysis. Specifically, we use Conditional Mutual Information
(CMI) to quantify how well the input (i.e., the training data) can be
recognized given the output (i.e., the trained model) of the learning
algorithm. We show that bounds on CMI can be obtained from VC dimension,
compression schemes, differential privacy, and other methods. We then show that
bounded CMI implies various forms of generalization.},
added-at = {2020-01-27T12:53:20.000+0100},
author = {Steinke, Thomas and Zakynthinou, Lydia},
biburl = {https://www.bibsonomy.org/bibtex/235f01ff5c3e80ad61222966aa982f496/kirk86},
description = {[2001.09122] Reasoning About Generalization via Conditional Mutual Information},
interhash = {827d4ce0a68ad59709430aef6078819b},
intrahash = {35f01ff5c3e80ad61222966aa982f496},
keywords = {deep-learning generalization readings},
note = {cite arxiv:2001.09122},
timestamp = {2020-01-27T12:53:20.000+0100},
title = {Reasoning About Generalization via Conditional Mutual Information},
url = {http://arxiv.org/abs/2001.09122},
year = 2020
}