In many commercial systems, the 'best bet' recommendations are shown, but the predicted rating values are not. This is usually referred to as a top-N recommendation task, where the goal of the recommender system is to find a few specific items which are supposed to be most appealing to the user. Common methodologies based on error metrics (such as RMSE) are not a natural fit for evaluating the top-N recommendation task. Rather, top-N performance can be directly measured by alternative methodologies based on accuracy metrics (such as precision/recall). An extensive evaluation of several state-of-the art recommender algorithms suggests that algorithms optimized for minimizing RMSE do not necessarily perform as expected in terms of top-N recommendation task. Results show that improvements in RMSE often do not translate into accuracy improvements. In particular, a naive non-personalized algorithm can outperform some common recommendation approaches and almost match the accuracy of sophisticated algorithms. Another finding is that the very few top popular items can skew the top-N performance. The analysis points out that when evaluating a recommender algorithm on the top-N recommendation task, the test set should be chosen carefully in order to not bias accuracy metrics towards non-personalized solutions. Finally, we offer practitioners new variants of two collaborative filtering algorithms that, regardless of their RMSE, significantly outperform other recommender algorithms in pursuing the top-N recommendation task, with offering additional practical advantages. This comes at surprise given the simplicity of these two methods.
Description
Performance of recommender algorithms on top-n recommendation tasks
%0 Conference Paper
%1 cremonesi2010performance
%A Cremonesi, Paolo
%A Koren, Yehuda
%A Turrin, Roberto
%B Proceedings of the Fourth ACM Conference on Recommender Systems
%C New York, NY, USA
%D 2010
%I ACM
%K evaluation performance recommender
%P 39--46
%R 10.1145/1864708.1864721
%T Performance of Recommender Algorithms on Top-n Recommendation Tasks
%U http://doi.acm.org/10.1145/1864708.1864721
%X In many commercial systems, the 'best bet' recommendations are shown, but the predicted rating values are not. This is usually referred to as a top-N recommendation task, where the goal of the recommender system is to find a few specific items which are supposed to be most appealing to the user. Common methodologies based on error metrics (such as RMSE) are not a natural fit for evaluating the top-N recommendation task. Rather, top-N performance can be directly measured by alternative methodologies based on accuracy metrics (such as precision/recall). An extensive evaluation of several state-of-the art recommender algorithms suggests that algorithms optimized for minimizing RMSE do not necessarily perform as expected in terms of top-N recommendation task. Results show that improvements in RMSE often do not translate into accuracy improvements. In particular, a naive non-personalized algorithm can outperform some common recommendation approaches and almost match the accuracy of sophisticated algorithms. Another finding is that the very few top popular items can skew the top-N performance. The analysis points out that when evaluating a recommender algorithm on the top-N recommendation task, the test set should be chosen carefully in order to not bias accuracy metrics towards non-personalized solutions. Finally, we offer practitioners new variants of two collaborative filtering algorithms that, regardless of their RMSE, significantly outperform other recommender algorithms in pursuing the top-N recommendation task, with offering additional practical advantages. This comes at surprise given the simplicity of these two methods.
%@ 978-1-60558-906-0
@inproceedings{cremonesi2010performance,
abstract = {In many commercial systems, the 'best bet' recommendations are shown, but the predicted rating values are not. This is usually referred to as a top-N recommendation task, where the goal of the recommender system is to find a few specific items which are supposed to be most appealing to the user. Common methodologies based on error metrics (such as RMSE) are not a natural fit for evaluating the top-N recommendation task. Rather, top-N performance can be directly measured by alternative methodologies based on accuracy metrics (such as precision/recall). An extensive evaluation of several state-of-the art recommender algorithms suggests that algorithms optimized for minimizing RMSE do not necessarily perform as expected in terms of top-N recommendation task. Results show that improvements in RMSE often do not translate into accuracy improvements. In particular, a naive non-personalized algorithm can outperform some common recommendation approaches and almost match the accuracy of sophisticated algorithms. Another finding is that the very few top popular items can skew the top-N performance. The analysis points out that when evaluating a recommender algorithm on the top-N recommendation task, the test set should be chosen carefully in order to not bias accuracy metrics towards non-personalized solutions. Finally, we offer practitioners new variants of two collaborative filtering algorithms that, regardless of their RMSE, significantly outperform other recommender algorithms in pursuing the top-N recommendation task, with offering additional practical advantages. This comes at surprise given the simplicity of these two methods.},
acmid = {1864721},
added-at = {2014-08-04T18:02:03.000+0200},
address = {New York, NY, USA},
author = {Cremonesi, Paolo and Koren, Yehuda and Turrin, Roberto},
biburl = {https://www.bibsonomy.org/bibtex/2aeab7f02942cfeb97ccc7ae0a1d60801/sdo},
booktitle = {Proceedings of the Fourth ACM Conference on Recommender Systems},
description = {Performance of recommender algorithms on top-n recommendation tasks},
doi = {10.1145/1864708.1864721},
interhash = {04cb3373b65b03e03225f447250e7873},
intrahash = {aeab7f02942cfeb97ccc7ae0a1d60801},
isbn = {978-1-60558-906-0},
keywords = {evaluation performance recommender},
location = {Barcelona, Spain},
numpages = {8},
pages = {39--46},
publisher = {ACM},
series = {RecSys '10},
timestamp = {2014-08-04T18:02:03.000+0200},
title = {Performance of Recommender Algorithms on Top-n Recommendation Tasks},
url = {http://doi.acm.org/10.1145/1864708.1864721},
year = 2010
}