%0 Journal Article %1 raykar2010learning %A Raykar, Vikas C. %A Yu, Shipeng %A Zhao, Linda H. %A Valadez, Gerardo Hermosillo %A Florin, Charles %A Bogoni, Luca %A Moy, Linda %D 2010 %I JMLR.org %J Journal of Machine Learning Research %K %P 1297--1322 %T Learning From Crowds %U http://dl.acm.org/citation.cfm?id=1756006.1859894 %V 11 %X For many supervised learning tasks it may be infeasible (or very expensive) to obtain objective and reliable labels. Instead, we can collect subjective (possibly noisy) labels from multiple experts or annotators. In practice, there is a substantial amount of disagreement among the annotators, and hence it is of great practical interest to address conventional supervised learning problems in this scenario. In this paper we describe a probabilistic approach for supervised learning when we have multiple annotators providing (possibly noisy) labels but no absolute gold standard. The proposed algorithm evaluates the different experts and also gives an estimate of the actual hidden labels. Experimental results indicate that the proposed method is superior to the commonly used majority voting baseline.

@article{raykar2010learning, abstract = {For many supervised learning tasks it may be infeasible (or very expensive) to obtain objective and reliable labels. Instead, we can collect subjective (possibly noisy) labels from multiple experts or annotators. In practice, there is a substantial amount of disagreement among the annotators, and hence it is of great practical interest to address conventional supervised learning problems in this scenario. In this paper we describe a probabilistic approach for supervised learning when we have multiple annotators providing (possibly noisy) labels but no absolute gold standard. The proposed algorithm evaluates the different experts and also gives an estimate of the actual hidden labels. Experimental results indicate that the proposed method is superior to the commonly used majority voting baseline.}, acmid = {1859894}, added-at = {2012-06-20T11:58:03.000+0200}, author = {Raykar, Vikas C. and Yu, Shipeng and Zhao, Linda H. and Valadez, Gerardo Hermosillo and Florin, Charles and Bogoni, Luca and Moy, Linda}, biburl = {https://www.bibsonomy.org/bibtex/214220abe8babfab01c0cdd5ebd5e4b7c/jaeschke}, interhash = {8113daf47997fddf48e4c6c79f2eba56}, intrahash = {14220abe8babfab01c0cdd5ebd5e4b7c}, issn = {1532-4435}, issue_date = {3/1/2010}, journal = {Journal of Machine Learning Research}, keywords = {}, month = aug, numpages = {26}, pages = {1297--1322}, publisher = {JMLR.org}, timestamp = {2012-06-20T11:58:03.000+0200}, title = {Learning From Crowds}, url = {http://dl.acm.org/citation.cfm?id=1756006.1859894}, volume = 11, year = 2010 }