%0 Journal Article %1 braverman2019calibration %A Braverman, Mark %A Chen, Xinyi %A Kakade, Sham M. %A Narasimhan, Karthik %A Zhang, Cyril %A Zhang, Yi %D 2019 %K calibration memory nlp %T Calibration, Entropy Rates, and Memory in Language Models %U http://arxiv.org/abs/1906.05664 %X Building accurate language models that capture meaningful long-term dependencies is a core challenge in natural language processing. Towards this end, we present a calibration-based approach to measure long-term discrepancies between a generative sequence model and the true distribution, and use these discrepancies to improve the model. Empirically, we show that state-of-the-art language models, including LSTMs and Transformers, are miscalibrated: the entropy rates of their generations drift dramatically upward over time. We then provide provable methods to mitigate this phenomenon. Furthermore, we show how this calibration-based approach can also be used to measure the amount of memory that language models use for prediction.

@article{braverman2019calibration, abstract = {Building accurate language models that capture meaningful long-term dependencies is a core challenge in natural language processing. Towards this end, we present a calibration-based approach to measure long-term discrepancies between a generative sequence model and the true distribution, and use these discrepancies to improve the model. Empirically, we show that state-of-the-art language models, including LSTMs and Transformers, are \emph{miscalibrated}: the entropy rates of their generations drift dramatically upward over time. We then provide provable methods to mitigate this phenomenon. Furthermore, we show how this calibration-based approach can also be used to measure the amount of memory that language models use for prediction.}, added-at = {2019-06-29T22:42:48.000+0200}, author = {Braverman, Mark and Chen, Xinyi and Kakade, Sham M. and Narasimhan, Karthik and Zhang, Cyril and Zhang, Yi}, biburl = {https://www.bibsonomy.org/bibtex/28d4b0a7f9d46f4106af88496a24be080/kirk86}, description = {[1906.05664] Calibration, Entropy Rates, and Memory in Language Models}, interhash = {f484fa1fb7dc37293c5f35526072fe49}, intrahash = {8d4b0a7f9d46f4106af88496a24be080}, keywords = {calibration memory nlp}, note = {cite arxiv:1906.05664}, timestamp = {2019-06-29T22:42:48.000+0200}, title = {Calibration, Entropy Rates, and Memory in Language Models}, url = {http://arxiv.org/abs/1906.05664}, year = 2019 }