%0 Generic %1 richemond2024offline %A Richemond, Pierre Harvey %A Tang, Yunhao %A Guo, Daniel %A Calandriello, Daniele %A Azar, Mohammad Gheshlaghi %A Rafailov, Rafael %A Pires, Bernardo Avila %A Tarassov, Eugene %A Spangher, Lucas %A Ellsworth, Will %A Severyn, Aliaksei %A Mallinson, Jonathan %A Shani, Lior %A Shamir, Gil %A Joshi, Rishabh %A Liu, Tianqi %A Munos, Remi %A Piot, Bilal %D 2024 %K dro nlp rl %T Offline Regularised Reinforcement Learning for Large Language Models Alignment %U https://arxiv.org/abs/2405.19107 %X The dominant framework for alignment of large language models (LLM), whether through reinforcement learning from human feedback or direct preference optimisation, is to learn from preference data. This involves building datasets where each element is a quadruplet composed of a prompt, two independent responses (completions of the prompt) and a human preference between the two independent responses, yielding a preferred and a dis-preferred response. Such data is typically scarce and expensive to collect. On the other hand, single-trajectory datasets where each element is a triplet composed of a prompt, a response and a human feedback is naturally more abundant. The canonical element of such datasets is for instance an LLM's response to a user's prompt followed by a user's feedback such as a thumbs-up/down. Consequently, in this work, we propose DRO, or Direct Reward Optimisation, as a framework and associated algorithms that do not require pairwise preferences. DRO uses a simple mean-squared objective that can be implemented in various ways. We validate our findings empirically, using T5 encoder-decoder language models, and show DRO's performance over selected baselines such as Kahneman-Tversky Optimization (KTO). Thus, we confirm that DRO is a simple and empirically compelling method for single-trajectory policy optimisation.

@misc{richemond2024offline, abstract = {The dominant framework for alignment of large language models (LLM), whether through reinforcement learning from human feedback or direct preference optimisation, is to learn from preference data. This involves building datasets where each element is a quadruplet composed of a prompt, two independent responses (completions of the prompt) and a human preference between the two independent responses, yielding a preferred and a dis-preferred response. Such data is typically scarce and expensive to collect. On the other hand, \emph{single-trajectory} datasets where each element is a triplet composed of a prompt, a response and a human feedback is naturally more abundant. The canonical element of such datasets is for instance an LLM's response to a user's prompt followed by a user's feedback such as a thumbs-up/down. Consequently, in this work, we propose DRO, or \emph{Direct Reward Optimisation}, as a framework and associated algorithms that do not require pairwise preferences. DRO uses a simple mean-squared objective that can be implemented in various ways. We validate our findings empirically, using T5 encoder-decoder language models, and show DRO's performance over selected baselines such as Kahneman-Tversky Optimization (KTO). Thus, we confirm that DRO is a simple and empirically compelling method for single-trajectory policy optimisation.}, added-at = {2024-06-10T13:29:09.000+0200}, archiveprefix = {arXiv}, author = {Richemond, Pierre Harvey and Tang, Yunhao and Guo, Daniel and Calandriello, Daniele and Azar, Mohammad Gheshlaghi and Rafailov, Rafael and Pires, Bernardo Avila and Tarassov, Eugene and Spangher, Lucas and Ellsworth, Will and Severyn, Aliaksei and Mallinson, Jonathan and Shani, Lior and Shamir, Gil and Joshi, Rishabh and Liu, Tianqi and Munos, Remi and Piot, Bilal}, biburl = {https://www.bibsonomy.org/bibtex/2805146666902567c3c4afb258c9643db/tomvoelker}, description = {Offline Regularised Reinforcement Learning for Large Language Models Alignment}, eprint = {2405.19107}, interhash = {78485d1edf577c41e6f47fc750446abf}, intrahash = {805146666902567c3c4afb258c9643db}, keywords = {dro nlp rl}, month = may, primaryclass = {cs.LG}, timestamp = {2024-06-10T13:29:09.000+0200}, title = {Offline Regularised Reinforcement Learning for Large Language Models Alignment}, url = {https://arxiv.org/abs/2405.19107}, year = 2024 }