%0 Conference Paper %1 ozertem2011evaluating %A Ozertem, Umut %A Jones, Rosie %A Dumoulin, Benoit %B Proceedings of the 20th international conference on World wide web %C New York, NY, USA %D 2011 %I ACM %K check clickdata clickthrough evaluation implicit-feedback %P 397--406 %R 10.1145/1963405.1963463 %T Evaluating new search engine configurations with pre-existing judgments and clicks %U http://doi.acm.org/10.1145/1963405.1963463 %X We provide a novel method of evaluating search results, which allows us to combine existing editorial judgments with the relevance estimates generated by click-based user browsing models. There are evaluation methods in the literature that use clicks and editorial judgments together, but our approach is novel in the sense that it allows us to predict the impact of <i>unseen</i> search models <i>without</i> online tests to collect clicks and without requesting new editorial data, since we are only re-using <i>existing</i> editorial data, and clicks observed for previous result set configurations. Since the user browsing model and the pre-existing editorial data cannot provide relevance estimates for all documents for the selected set of queries, one important challenge is to obtain this performance estimation where there are a lot of ranked documents with missing relevance values. We introduce a query and rank based smoothing to overcome this problem. We show that a hybrid of these smoothing techniques performs better than both query and position based smoothing, and despite the high percentage of missing judgments, the resulting method is significantly correlated (0.74) with <i>DCG</i> values evaluated using fully judged datasets, and approaches inter-annotator agreement. We show that previously published techniques, applicable to frequent queries, degrade when applied to a random sample of queries, with a correlation of only 0.29. While our experiments focus on evaluation using <i>DCG</i>, our method is also applicable to other commonly used metrics. %@ 978-1-4503-0632-4

@inproceedings{ozertem2011evaluating, abstract = {We provide a novel method of evaluating search results, which allows us to combine existing editorial judgments with the relevance estimates generated by click-based user browsing models. There are evaluation methods in the literature that use clicks and editorial judgments together, but our approach is novel in the sense that it allows us to predict the impact of <i>unseen</i> search models <i>without</i> online tests to collect clicks and without requesting new editorial data, since we are only re-using <i>existing</i> editorial data, and clicks observed for previous result set configurations. Since the user browsing model and the pre-existing editorial data cannot provide relevance estimates for all documents for the selected set of queries, one important challenge is to obtain this performance estimation where there are a lot of ranked documents with missing relevance values. We introduce a query and rank based smoothing to overcome this problem. We show that a hybrid of these smoothing techniques performs better than both query and position based smoothing, and despite the high percentage of missing judgments, the resulting method is significantly correlated (0.74) with <i>DCG</i> values evaluated using fully judged datasets, and approaches inter-annotator agreement. We show that previously published techniques, applicable to frequent queries, degrade when applied to a random sample of queries, with a correlation of only 0.29. While our experiments focus on evaluation using <i>DCG</i>, our method is also applicable to other commonly used metrics.}, acmid = {1963463}, added-at = {2011-07-29T16:33:28.000+0200}, address = {New York, NY, USA}, author = {Ozertem, Umut and Jones, Rosie and Dumoulin, Benoit}, biburl = {https://www.bibsonomy.org/bibtex/2cd5ef067c6f300604a5cfb77eeae26cf/beate}, booktitle = {Proceedings of the 20th international conference on World wide web}, description = {Evaluating new search engine configurations with pre-existing judgments and clicks}, doi = {10.1145/1963405.1963463}, interhash = {57c345e630805ca52499b6512212a639}, intrahash = {cd5ef067c6f300604a5cfb77eeae26cf}, isbn = {978-1-4503-0632-4}, keywords = {check clickdata clickthrough evaluation implicit-feedback}, location = {Hyderabad, India}, numpages = {10}, pages = {397--406}, publisher = {ACM}, series = {WWW '11}, timestamp = {2011-07-29T16:33:28.000+0200}, title = {Evaluating new search engine configurations with pre-existing judgments and clicks}, url = {http://doi.acm.org/10.1145/1963405.1963463}, year = 2011 }