%0 Conference Paper %1 gruson2019offline %A Gruson, Alois %A Chandar, Praveen %A Charbuillet, Christophe %A McInerney, James %A Hansen, Samantha %A Tardieu, Damien %A Carterette, Ben %B Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining %D 2019 %I ACM %K bandits carousel recommender %P 420-428 %R 10.1145/3289600.3291027 %T Offline Evaluation to Make Decisions About Playlist Recommendation Algorithms %U https://doi.org/10.1145%2F3289600.3291027 %X Evaluating algorithmic recommendations is an important, but difficult, problem. Evaluations conducted offline using data collected from user interactions with an online system often suffer from biases arising from the user interface or the recommendation engine. Online evaluation (A/B testing) can more easily address problems of bias, but depending on setting can be time-consuming and incur risk of negatively impacting the user experience, not to mention that it is generally more difficult when access to a large user base is not taken as granted. A compromise based on counterfactual analysis is to present some subset of online users with recommendation results that have been randomized or otherwise manipulated, log their interactions, and then use those to de-bias offline evaluations on historical data. However, previous work does not offer clear conclusions on how well such methods correlate with and are able to predict the results of online A/B tests. Understanding this is crucial to widespread adoption of new offline evaluation techniques in recommender systems. In this work we present a comparison of offline and online evaluation results for a particular recommendation problem: recommending playlists of tracks to a user looking for music. We describe two different ways to think about de-biasing offline collections for more accurate evaluation. Our results show that, contrary to much of the previous work on this topic, properly-conducted offline experiments do correlate well to A/B test results, and moreover that we can expect an offline evaluation to identify the best candidate systems for online testing with high probability.

@inproceedings{gruson2019offline, abstract = {Evaluating algorithmic recommendations is an important, but difficult, problem. Evaluations conducted offline using data collected from user interactions with an online system often suffer from biases arising from the user interface or the recommendation engine. Online evaluation (A/B testing) can more easily address problems of bias, but depending on setting can be time-consuming and incur risk of negatively impacting the user experience, not to mention that it is generally more difficult when access to a large user base is not taken as granted. A compromise based on \em counterfactual analysis is to present some subset of online users with recommendation results that have been randomized or otherwise manipulated, log their interactions, and then use those to de-bias offline evaluations on historical data. However, previous work does not offer clear conclusions on how well such methods correlate with and are able to predict the results of online A/B tests. Understanding this is crucial to widespread adoption of new offline evaluation techniques in recommender systems. In this work we present a comparison of offline and online evaluation results for a particular recommendation problem: recommending playlists of tracks to a user looking for music. We describe two different ways to think about de-biasing offline collections for more accurate evaluation. Our results show that, contrary to much of the previous work on this topic, properly-conducted offline experiments do correlate well to A/B test results, and moreover that we can expect an offline evaluation to identify the best candidate systems for online testing with high probability. }, added-at = {2021-07-09T23:28:32.000+0200}, author = {Gruson, Alois and Chandar, Praveen and Charbuillet, Christophe and McInerney, James and Hansen, Samantha and Tardieu, Damien and Carterette, Ben}, biburl = {https://www.bibsonomy.org/bibtex/20ab8ae08f8db1be0dab70d82f28d4784/brusilovsky}, booktitle = {Proceedings of the Twelfth {ACM} International Conference on Web Search and Data Mining}, description = {Offline Evaluation to Make Decisions About PlaylistRecommendation Algorithms | Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining}, doi = {10.1145/3289600.3291027}, interhash = {b635234245ee3f71c02f11bb98f2e499}, intrahash = {0ab8ae08f8db1be0dab70d82f28d4784}, keywords = {bandits carousel recommender}, month = jan, pages = {420-428}, publisher = {{ACM}}, timestamp = {2022-08-12T03:58:46.000+0200}, title = {Offline Evaluation to Make Decisions About Playlist Recommendation Algorithms}, url = {https://doi.org/10.1145%2F3289600.3291027}, year = 2019 }