%0 Journal Article %1 eugenio2004 %A Eugenio, B. Di %A Glass, M. %C Cambridge, MA, USA %D 2004 %I MIT Press %J Computational Linguistics %K TOREAD linguistics mathematics reading-group %N 1 %P 95--101 %R http://dx.doi.org/10.1162/089120104773633402 %T The kappa statistic: a second look %U http://portal.acm.org/citation.cfm?id=1005385&dl=# %V 30 %X In recent years, the kappa coefficient of agreement has become the de facto standard for evaluating intercoder agreement for tagging tasks. In this squib, we highlight issues that affect κ and that the community has largely neglected. First, we discuss the assumptions underlying different computations of the expected agreement component of κ. Second, we discuss how prevalence and bias affect the κ measure.

@article{eugenio2004, abstract = {In recent years, the kappa coefficient of agreement has become the de facto standard for evaluating intercoder agreement for tagging tasks. In this squib, we highlight issues that affect κ and that the community has largely neglected. First, we discuss the assumptions underlying different computations of the expected agreement component of κ. Second, we discuss how prevalence and bias affect the κ measure.}, added-at = {2008-06-03T09:56:37.000+0200}, address = {Cambridge, MA, USA}, author = {Eugenio, B. Di and Glass, M.}, biburl = {https://www.bibsonomy.org/bibtex/27f2f0f656b9fce77906c0857e15fee68/mstrohm}, description = {The kappa statistic}, doi = {http://dx.doi.org/10.1162/089120104773633402}, interhash = {d09f376b70a157341f6abd6dbe6c40f1}, intrahash = {7f2f0f656b9fce77906c0857e15fee68}, issn = {0891-2017}, journal = {Computational Linguistics}, keywords = {TOREAD linguistics mathematics reading-group}, number = 1, pages = {95--101}, publisher = {MIT Press}, timestamp = {2008-06-03T09:56:37.000+0200}, title = {The kappa statistic: a second look}, url = {http://portal.acm.org/citation.cfm?id=1005385&dl=#}, volume = 30, year = 2004 }