%0 Generic %1 maclaurin2015gradientbased %A Maclaurin, Dougal %A Duvenaud, David %A Adams, Ryan P. %D 2015 %K ai gradient optimization %T Gradient-based Hyperparameter Optimization through Reversible Learning %U http://arxiv.org/abs/1502.03492 %X Tuning hyperparameters of learning algorithms is hard because gradients are usually unavailable. We compute exact gradients of cross-validation performance with respect to all hyperparameters by chaining derivatives backwards through the entire training procedure. These gradients allow us to optimize thousands of hyperparameters, including step-size and momentum schedules, weight initialization distributions, richly parameterized regularization schemes, and neural network architectures. We compute hyperparameter gradients by exactly reversing the dynamics of stochastic gradient descent with momentum.

@misc{maclaurin2015gradientbased, abstract = {Tuning hyperparameters of learning algorithms is hard because gradients are usually unavailable. We compute exact gradients of cross-validation performance with respect to all hyperparameters by chaining derivatives backwards through the entire training procedure. These gradients allow us to optimize thousands of hyperparameters, including step-size and momentum schedules, weight initialization distributions, richly parameterized regularization schemes, and neural network architectures. We compute hyperparameter gradients by exactly reversing the dynamics of stochastic gradient descent with momentum.}, added-at = {2020-12-04T10:28:31.000+0100}, author = {Maclaurin, Dougal and Duvenaud, David and Adams, Ryan P.}, biburl = {https://www.bibsonomy.org/bibtex/22aadb1e35680c91d51d96691e698aa40/louissf}, description = {Gradient-based Hyperparameter Optimization through Reversible Learning}, interhash = {b495e45fe88acc024d5da7d997ad9137}, intrahash = {2aadb1e35680c91d51d96691e698aa40}, keywords = {ai gradient optimization}, note = {cite arxiv:1502.03492Comment: 10 figures. Submitted to ICML}, timestamp = {2020-12-04T10:28:31.000+0100}, title = {Gradient-based Hyperparameter Optimization through Reversible Learning}, url = {http://arxiv.org/abs/1502.03492}, year = 2015 }