While normalizing flows have led to significant advances in modeling
high-dimensional continuous distributions, their applicability to discrete
distributions remains unknown. In this paper, we show that flows can in fact be
extended to discrete events---and under a simple change-of-variables formula
not requiring log-determinant-Jacobian computations. Discrete flows have
numerous applications. We consider two flow architectures: discrete
autoregressive flows that enable bidirectionality, allowing, for example,
tokens in text to depend on both left-to-right and right-to-left contexts in an
exact language model; and discrete bipartite flows that enable efficient
non-autoregressive generation as in RealNVP. Empirically, we find that discrete
autoregressive flows outperform autoregressive baselines on synthetic discrete
distributions, an addition task, and Potts models; and bipartite flows can
obtain competitive performance with autoregressive baselines on character-level
language modeling for Penn Tree Bank and text8.
Description
[1905.10347] Discrete Flows: Invertible Generative Models of Discrete Data
%0 Journal Article
%1 tran2019discrete
%A Tran, Dustin
%A Vafa, Keyon
%A Agrawal, Kumar Krishna
%A Dinh, Laurent
%A Poole, Ben
%D 2019
%K 2019 arxiv deep-learning discrete flow
%T Discrete Flows: Invertible Generative Models of Discrete Data
%U http://arxiv.org/abs/1905.10347
%X While normalizing flows have led to significant advances in modeling
high-dimensional continuous distributions, their applicability to discrete
distributions remains unknown. In this paper, we show that flows can in fact be
extended to discrete events---and under a simple change-of-variables formula
not requiring log-determinant-Jacobian computations. Discrete flows have
numerous applications. We consider two flow architectures: discrete
autoregressive flows that enable bidirectionality, allowing, for example,
tokens in text to depend on both left-to-right and right-to-left contexts in an
exact language model; and discrete bipartite flows that enable efficient
non-autoregressive generation as in RealNVP. Empirically, we find that discrete
autoregressive flows outperform autoregressive baselines on synthetic discrete
distributions, an addition task, and Potts models; and bipartite flows can
obtain competitive performance with autoregressive baselines on character-level
language modeling for Penn Tree Bank and text8.
@article{tran2019discrete,
abstract = {While normalizing flows have led to significant advances in modeling
high-dimensional continuous distributions, their applicability to discrete
distributions remains unknown. In this paper, we show that flows can in fact be
extended to discrete events---and under a simple change-of-variables formula
not requiring log-determinant-Jacobian computations. Discrete flows have
numerous applications. We consider two flow architectures: discrete
autoregressive flows that enable bidirectionality, allowing, for example,
tokens in text to depend on both left-to-right and right-to-left contexts in an
exact language model; and discrete bipartite flows that enable efficient
non-autoregressive generation as in RealNVP. Empirically, we find that discrete
autoregressive flows outperform autoregressive baselines on synthetic discrete
distributions, an addition task, and Potts models; and bipartite flows can
obtain competitive performance with autoregressive baselines on character-level
language modeling for Penn Tree Bank and text8.},
added-at = {2019-12-07T05:15:14.000+0100},
author = {Tran, Dustin and Vafa, Keyon and Agrawal, Kumar Krishna and Dinh, Laurent and Poole, Ben},
biburl = {https://www.bibsonomy.org/bibtex/2d2d5640b13a61715d57dc2ae7b9060e1/analyst},
description = {[1905.10347] Discrete Flows: Invertible Generative Models of Discrete Data},
interhash = {312cc58d7aa0e076b837c3438518da98},
intrahash = {d2d5640b13a61715d57dc2ae7b9060e1},
keywords = {2019 arxiv deep-learning discrete flow},
note = {cite arxiv:1905.10347},
timestamp = {2019-12-07T05:15:14.000+0100},
title = {Discrete Flows: Invertible Generative Models of Discrete Data},
url = {http://arxiv.org/abs/1905.10347},
year = 2019
}