This work explores hypernetworks: an approach of using a one network, also
known as a hypernetwork, to generate the weights for another network.
Hypernetworks provide an abstraction that is similar to what is found in
nature: the relationship between a genotype - the hypernetwork - and a
phenotype - the main network. Though they are also reminiscent of HyperNEAT in
evolution, our hypernetworks are trained end-to-end with backpropagation and
thus are usually faster. The focus of this work is to make hypernetworks useful
for deep convolutional networks and long recurrent networks, where
hypernetworks can be viewed as relaxed form of weight-sharing across layers.
Our main result is that hypernetworks can generate non-shared weights for LSTM
and achieve near state-of-the-art results on a variety of sequence modelling
tasks including character-level language modelling, handwriting generation and
neural machine translation, challenging the weight-sharing paradigm for
recurrent networks. Our results also show that hypernetworks applied to
convolutional networks still achieve respectable results for image recognition
tasks compared to state-of-the-art baseline models while requiring fewer
learnable parameters.
%0 Generic
%1 ha2016hypernetworks
%A Ha, David
%A Dai, Andrew
%A Le, Quoc V.
%D 2016
%K final thema:Convolutional2DKnowledgeGraphEmbeddings
%T HyperNetworks
%U http://arxiv.org/abs/1609.09106
%X This work explores hypernetworks: an approach of using a one network, also
known as a hypernetwork, to generate the weights for another network.
Hypernetworks provide an abstraction that is similar to what is found in
nature: the relationship between a genotype - the hypernetwork - and a
phenotype - the main network. Though they are also reminiscent of HyperNEAT in
evolution, our hypernetworks are trained end-to-end with backpropagation and
thus are usually faster. The focus of this work is to make hypernetworks useful
for deep convolutional networks and long recurrent networks, where
hypernetworks can be viewed as relaxed form of weight-sharing across layers.
Our main result is that hypernetworks can generate non-shared weights for LSTM
and achieve near state-of-the-art results on a variety of sequence modelling
tasks including character-level language modelling, handwriting generation and
neural machine translation, challenging the weight-sharing paradigm for
recurrent networks. Our results also show that hypernetworks applied to
convolutional networks still achieve respectable results for image recognition
tasks compared to state-of-the-art baseline models while requiring fewer
learnable parameters.
@misc{ha2016hypernetworks,
abstract = {This work explores hypernetworks: an approach of using a one network, also
known as a hypernetwork, to generate the weights for another network.
Hypernetworks provide an abstraction that is similar to what is found in
nature: the relationship between a genotype - the hypernetwork - and a
phenotype - the main network. Though they are also reminiscent of HyperNEAT in
evolution, our hypernetworks are trained end-to-end with backpropagation and
thus are usually faster. The focus of this work is to make hypernetworks useful
for deep convolutional networks and long recurrent networks, where
hypernetworks can be viewed as relaxed form of weight-sharing across layers.
Our main result is that hypernetworks can generate non-shared weights for LSTM
and achieve near state-of-the-art results on a variety of sequence modelling
tasks including character-level language modelling, handwriting generation and
neural machine translation, challenging the weight-sharing paradigm for
recurrent networks. Our results also show that hypernetworks applied to
convolutional networks still achieve respectable results for image recognition
tasks compared to state-of-the-art baseline models while requiring fewer
learnable parameters.},
added-at = {2021-01-17T16:36:17.000+0100},
author = {Ha, David and Dai, Andrew and Le, Quoc V.},
biburl = {https://www.bibsonomy.org/bibtex/2c3215a35ae51402b2ab6ab27b9efc905/peggyschnetter},
interhash = {4edc2172fe4952b39dc3b0618eb0a33e},
intrahash = {c3215a35ae51402b2ab6ab27b9efc905},
keywords = {final thema:Convolutional2DKnowledgeGraphEmbeddings},
timestamp = {2021-01-17T16:36:17.000+0100},
title = {HyperNetworks},
url = {http://arxiv.org/abs/1609.09106},
year = 2016
}