%0 Conference Paper %1 yue2010learning %A Yue, Yisong %A Gao, Yue %A Chapelle, Oliver %A Zhang, Ya %A Joachims, Thorsten %B Proceeding of the 33rd international ACM SIGIR conference on Research and development in information retrieval %C New York, NY, USA %D 2010 %I ACM %K check clickdata clickthrough evaluation implicit-feedback %P 507--514 %R 10.1145/1835449.1835534 %T Learning more powerful test statistics for click-based retrieval evaluation %U http://doi.acm.org/10.1145/1835449.1835534 %X Interleaving experiments are an attractive methodology for evaluating retrieval functions through implicit feedback. Designed as a blind and unbiased test for eliciting a preference between two retrieval functions, an interleaved ranking of the results of two retrieval functions is presented to the users. It is then observed whether the users click more on results from one retrieval function or the other. While it was shown that such interleaving experiments reliably identify the better of the two retrieval functions, the naive approach of counting all clicks equally leads to a suboptimal test. We present new methods for learning how to score different types of clicks so that the resulting test statistic optimizes the statistical power of the experiment. This can lead to substantial savings in the amount of data required for reaching a target confidence level. Our methods are evaluated on an operational search engine over a collection of scientific articles. %@ 978-1-4503-0153-4

@inproceedings{yue2010learning, abstract = {Interleaving experiments are an attractive methodology for evaluating retrieval functions through implicit feedback. Designed as a blind and unbiased test for eliciting a preference between two retrieval functions, an interleaved ranking of the results of two retrieval functions is presented to the users. It is then observed whether the users click more on results from one retrieval function or the other. While it was shown that such interleaving experiments reliably identify the better of the two retrieval functions, the naive approach of counting all clicks equally leads to a suboptimal test. We present new methods for learning how to score different types of clicks so that the resulting test statistic optimizes the statistical power of the experiment. This can lead to substantial savings in the amount of data required for reaching a target confidence level. Our methods are evaluated on an operational search engine over a collection of scientific articles.}, acmid = {1835534}, added-at = {2011-07-29T16:36:28.000+0200}, address = {New York, NY, USA}, author = {Yue, Yisong and Gao, Yue and Chapelle, Oliver and Zhang, Ya and Joachims, Thorsten}, biburl = {https://www.bibsonomy.org/bibtex/25e0c409b9538db6c3d0f402c8d696eb2/beate}, booktitle = {Proceeding of the 33rd international ACM SIGIR conference on Research and development in information retrieval}, description = {Learning more powerful test statistics for click-based retrieval evaluation}, doi = {10.1145/1835449.1835534}, interhash = {314382cc2d6609772cfd3eb82e43f25f}, intrahash = {5e0c409b9538db6c3d0f402c8d696eb2}, isbn = {978-1-4503-0153-4}, keywords = {check clickdata clickthrough evaluation implicit-feedback}, location = {Geneva, Switzerland}, numpages = {8}, pages = {507--514}, publisher = {ACM}, series = {SIGIR '10}, timestamp = {2011-07-29T16:36:28.000+0200}, title = {Learning more powerful test statistics for click-based retrieval evaluation}, url = {http://doi.acm.org/10.1145/1835449.1835534}, year = 2010 }