%0 Conference Paper %1 cremonesi2010performance %A Cremonesi, Paolo %A Koren, Yehuda %A Turrin, Roberto %B Proceedings of the Fourth ACM Conference on Recommender Systems %C New York, NY, USA %D 2010 %I ACM %K evaluation performance recommender %P 39--46 %R 10.1145/1864708.1864721 %T Performance of Recommender Algorithms on Top-n Recommendation Tasks %U http://doi.acm.org/10.1145/1864708.1864721 %X In many commercial systems, the 'best bet' recommendations are shown, but the predicted rating values are not. This is usually referred to as a top-N recommendation task, where the goal of the recommender system is to find a few specific items which are supposed to be most appealing to the user. Common methodologies based on error metrics (such as RMSE) are not a natural fit for evaluating the top-N recommendation task. Rather, top-N performance can be directly measured by alternative methodologies based on accuracy metrics (such as precision/recall). An extensive evaluation of several state-of-the art recommender algorithms suggests that algorithms optimized for minimizing RMSE do not necessarily perform as expected in terms of top-N recommendation task. Results show that improvements in RMSE often do not translate into accuracy improvements. In particular, a naive non-personalized algorithm can outperform some common recommendation approaches and almost match the accuracy of sophisticated algorithms. Another finding is that the very few top popular items can skew the top-N performance. The analysis points out that when evaluating a recommender algorithm on the top-N recommendation task, the test set should be chosen carefully in order to not bias accuracy metrics towards non-personalized solutions. Finally, we offer practitioners new variants of two collaborative filtering algorithms that, regardless of their RMSE, significantly outperform other recommender algorithms in pursuing the top-N recommendation task, with offering additional practical advantages. This comes at surprise given the simplicity of these two methods. %@ 978-1-60558-906-0

@inproceedings{cremonesi2010performance, abstract = {In many commercial systems, the 'best bet' recommendations are shown, but the predicted rating values are not. This is usually referred to as a top-N recommendation task, where the goal of the recommender system is to find a few specific items which are supposed to be most appealing to the user. Common methodologies based on error metrics (such as RMSE) are not a natural fit for evaluating the top-N recommendation task. Rather, top-N performance can be directly measured by alternative methodologies based on accuracy metrics (such as precision/recall). An extensive evaluation of several state-of-the art recommender algorithms suggests that algorithms optimized for minimizing RMSE do not necessarily perform as expected in terms of top-N recommendation task. Results show that improvements in RMSE often do not translate into accuracy improvements. In particular, a naive non-personalized algorithm can outperform some common recommendation approaches and almost match the accuracy of sophisticated algorithms. Another finding is that the very few top popular items can skew the top-N performance. The analysis points out that when evaluating a recommender algorithm on the top-N recommendation task, the test set should be chosen carefully in order to not bias accuracy metrics towards non-personalized solutions. Finally, we offer practitioners new variants of two collaborative filtering algorithms that, regardless of their RMSE, significantly outperform other recommender algorithms in pursuing the top-N recommendation task, with offering additional practical advantages. This comes at surprise given the simplicity of these two methods.}, acmid = {1864721}, added-at = {2014-08-04T18:02:03.000+0200}, address = {New York, NY, USA}, author = {Cremonesi, Paolo and Koren, Yehuda and Turrin, Roberto}, biburl = {https://www.bibsonomy.org/bibtex/2aeab7f02942cfeb97ccc7ae0a1d60801/sdo}, booktitle = {Proceedings of the Fourth ACM Conference on Recommender Systems}, description = {Performance of recommender algorithms on top-n recommendation tasks}, doi = {10.1145/1864708.1864721}, interhash = {04cb3373b65b03e03225f447250e7873}, intrahash = {aeab7f02942cfeb97ccc7ae0a1d60801}, isbn = {978-1-60558-906-0}, keywords = {evaluation performance recommender}, location = {Barcelona, Spain}, numpages = {8}, pages = {39--46}, publisher = {ACM}, series = {RecSys '10}, timestamp = {2014-08-04T18:02:03.000+0200}, title = {Performance of Recommender Algorithms on Top-n Recommendation Tasks}, url = {http://doi.acm.org/10.1145/1864708.1864721}, year = 2010 }