%0 Generic %1 gulcehre2023reinforced %A Gulcehre, Caglar %A Paine, Tom Le %A Srinivasan, Srivatsan %A Konyushkova, Ksenia %A Weerts, Lotte %A Sharma, Abhishek %A Siddhant, Aditya %A Ahern, Alex %A Wang, Miaosen %A Gu, Chenjie %A Macherey, Wolfgang %A Doucet, Arnaud %A Firat, Orhan %A de Freitas, Nando %D 2023 %K deep-learning nlp %T Reinforced Self-Training (ReST) for Language Modeling %U http://arxiv.org/abs/2308.08998 %X Reinforcement learning from human feedback (RLHF) can improve the quality of large language model's (LLM) outputs by aligning them with human preferences. We propose a simple algorithm for aligning LLMs with human preferences inspired by growing batch reinforcement learning (RL), which we call Reinforced Self-Training (ReST). Given an initial LLM policy, ReST produces a dataset by generating samples from the policy, which are then used to improve the LLM policy using offline RL algorithms. ReST is more efficient than typical online RLHF methods because the training dataset is produced offline, which allows data reuse. While ReST is a general approach applicable to all generative learning settings, we focus on its application to machine translation. Our results show that ReST can substantially improve translation quality, as measured by automated metrics and human evaluation on machine translation benchmarks in a compute and sample-efficient manner.

@misc{gulcehre2023reinforced, abstract = {Reinforcement learning from human feedback (RLHF) can improve the quality of large language model's (LLM) outputs by aligning them with human preferences. We propose a simple algorithm for aligning LLMs with human preferences inspired by growing batch reinforcement learning (RL), which we call Reinforced Self-Training (ReST). Given an initial LLM policy, ReST produces a dataset by generating samples from the policy, which are then used to improve the LLM policy using offline RL algorithms. ReST is more efficient than typical online RLHF methods because the training dataset is produced offline, which allows data reuse. While ReST is a general approach applicable to all generative learning settings, we focus on its application to machine translation. Our results show that ReST can substantially improve translation quality, as measured by automated metrics and human evaluation on machine translation benchmarks in a compute and sample-efficient manner.}, added-at = {2023-08-22T12:23:43.000+0200}, author = {Gulcehre, Caglar and Paine, Tom Le and Srinivasan, Srivatsan and Konyushkova, Ksenia and Weerts, Lotte and Sharma, Abhishek and Siddhant, Aditya and Ahern, Alex and Wang, Miaosen and Gu, Chenjie and Macherey, Wolfgang and Doucet, Arnaud and Firat, Orhan and de Freitas, Nando}, biburl = {https://www.bibsonomy.org/bibtex/2055121def15f2c407eb1ccce39ba9551/tomvoelker}, description = {Reinforced Self-Training (ReST) for Language Modeling}, interhash = {67d274721609782942753a29213ce293}, intrahash = {055121def15f2c407eb1ccce39ba9551}, keywords = {deep-learning nlp}, note = {cite arxiv:2308.08998Comment: 23 pages, 16 figures}, timestamp = {2023-08-22T12:23:43.000+0200}, title = {Reinforced Self-Training (ReST) for Language Modeling}, url = {http://arxiv.org/abs/2308.08998}, year = 2023 }