Contrastive unsupervised representation learning (CURL) is the
state-of-the-art technique to learn representations (as a set of features) from
unlabelled data. While CURL has collected several empirical successes recently,
theoretical understanding of its performance was still missing. In a recent
work, Arora et al. (2019) provide the first generalisation bounds for CURL,
relying on a Rademacher complexity. We extend their framework to the flexible
PAC-Bayes setting, allowing us to deal with the non-iid setting. We present
PAC-Bayesian generalisation bounds for CURL, which are then used to derive a
new representation learning algorithm. Numerical experiments on real-life
datasets illustrate that our algorithm achieves competitive accuracy, and
yields non-vacuous generalisation bounds.
%0 Journal Article
%1 nozawa2019pacbayesian
%A Nozawa, Kento
%A Germain, Pascal
%A Guedj, Benjamin
%D 2019
%K bayesian bounds generalization readings theory unsupervised
%T PAC-Bayesian Contrastive Unsupervised Representation Learning
%U http://arxiv.org/abs/1910.04464
%X Contrastive unsupervised representation learning (CURL) is the
state-of-the-art technique to learn representations (as a set of features) from
unlabelled data. While CURL has collected several empirical successes recently,
theoretical understanding of its performance was still missing. In a recent
work, Arora et al. (2019) provide the first generalisation bounds for CURL,
relying on a Rademacher complexity. We extend their framework to the flexible
PAC-Bayes setting, allowing us to deal with the non-iid setting. We present
PAC-Bayesian generalisation bounds for CURL, which are then used to derive a
new representation learning algorithm. Numerical experiments on real-life
datasets illustrate that our algorithm achieves competitive accuracy, and
yields non-vacuous generalisation bounds.
@article{nozawa2019pacbayesian,
abstract = {Contrastive unsupervised representation learning (CURL) is the
state-of-the-art technique to learn representations (as a set of features) from
unlabelled data. While CURL has collected several empirical successes recently,
theoretical understanding of its performance was still missing. In a recent
work, Arora et al. (2019) provide the first generalisation bounds for CURL,
relying on a Rademacher complexity. We extend their framework to the flexible
PAC-Bayes setting, allowing us to deal with the non-iid setting. We present
PAC-Bayesian generalisation bounds for CURL, which are then used to derive a
new representation learning algorithm. Numerical experiments on real-life
datasets illustrate that our algorithm achieves competitive accuracy, and
yields non-vacuous generalisation bounds.},
added-at = {2020-05-15T14:20:33.000+0200},
author = {Nozawa, Kento and Germain, Pascal and Guedj, Benjamin},
biburl = {https://www.bibsonomy.org/bibtex/2225b0c610b91536a081d74f7c9f95d37/kirk86},
description = {[1910.04464] PAC-Bayesian Contrastive Unsupervised Representation Learning},
interhash = {812c3d3cc6a0434db9560757c3f68bd7},
intrahash = {225b0c610b91536a081d74f7c9f95d37},
keywords = {bayesian bounds generalization readings theory unsupervised},
note = {cite arxiv:1910.04464Comment: 16 pages},
timestamp = {2020-05-15T14:22:26.000+0200},
title = {PAC-Bayesian Contrastive Unsupervised Representation Learning},
url = {http://arxiv.org/abs/1910.04464},
year = 2019
}