Training of discrete latent variable models remains challenging because
passing gradient information through discrete units is difficult. We propose a
new class of smoothing transformations based on a mixture of two overlapping
distributions, and show that the proposed transformation can be used for
training binary latent models with either directed or undirected priors. We
derive a new variational bound to efficiently train with Boltzmann machine
priors. Using this bound, we develop DVAE++, a generative model with a global
discrete prior and a hierarchy of convolutional continuous variables.
Experiments on several benchmarks show that overlapping transformations
outperform other recent continuous relaxations of discrete latent variables
including Gumbel-Softmax (Maddison et al., 2016; Jang et al., 2016), and
discrete variational autoencoders (Rolfe 2016).
Description
DVAE++: Discrete Variational Autoencoders with Overlapping Transformations
%0 Generic
%1 vahdat2018discrete
%A Vahdat, Arash
%A Macready, William G.
%A Bian, Zhengbing
%A Khoshaman, Amir
%D 2018
%K autoencoder to_read unsupervised variational-ae
%T DVAE++: Discrete Variational Autoencoders with Overlapping
Transformations
%U http://arxiv.org/abs/1802.04920
%X Training of discrete latent variable models remains challenging because
passing gradient information through discrete units is difficult. We propose a
new class of smoothing transformations based on a mixture of two overlapping
distributions, and show that the proposed transformation can be used for
training binary latent models with either directed or undirected priors. We
derive a new variational bound to efficiently train with Boltzmann machine
priors. Using this bound, we develop DVAE++, a generative model with a global
discrete prior and a hierarchy of convolutional continuous variables.
Experiments on several benchmarks show that overlapping transformations
outperform other recent continuous relaxations of discrete latent variables
including Gumbel-Softmax (Maddison et al., 2016; Jang et al., 2016), and
discrete variational autoencoders (Rolfe 2016).
@misc{vahdat2018discrete,
abstract = {Training of discrete latent variable models remains challenging because
passing gradient information through discrete units is difficult. We propose a
new class of smoothing transformations based on a mixture of two overlapping
distributions, and show that the proposed transformation can be used for
training binary latent models with either directed or undirected priors. We
derive a new variational bound to efficiently train with Boltzmann machine
priors. Using this bound, we develop DVAE++, a generative model with a global
discrete prior and a hierarchy of convolutional continuous variables.
Experiments on several benchmarks show that overlapping transformations
outperform other recent continuous relaxations of discrete latent variables
including Gumbel-Softmax (Maddison et al., 2016; Jang et al., 2016), and
discrete variational autoencoders (Rolfe 2016).},
added-at = {2018-02-15T11:24:59.000+0100},
author = {Vahdat, Arash and Macready, William G. and Bian, Zhengbing and Khoshaman, Amir},
biburl = {https://www.bibsonomy.org/bibtex/2f4073c8b823c3c982ada9682e6992442/jk_itwm},
description = {DVAE++: Discrete Variational Autoencoders with Overlapping Transformations},
interhash = {7c18bc243eb8de64d8b2e7d833f0b54e},
intrahash = {f4073c8b823c3c982ada9682e6992442},
keywords = {autoencoder to_read unsupervised variational-ae},
note = {cite arxiv:1802.04920},
timestamp = {2018-02-15T11:24:59.000+0100},
title = {DVAE++: Discrete Variational Autoencoders with Overlapping
Transformations},
url = {http://arxiv.org/abs/1802.04920},
year = 2018
}