%0 Journal Article %1 silberzahn2018analysts %A Silberzahn, R. %A Uhlmann, E. L. %A Martin, D. P. %A Anselmi, P. %A Aust, F. %A Awtrey, E. %A Bahn\'ık, S. %A Bai, F. %A Bannard, C. %A Bonnier, E. %A Carlsson, R. %A Cheung, F. %A Christensen, G. %A Clay, R. %A Craig, M. A. %A Rosa, A. Dalla %A Dam, L. %A Evans, M. H. %A Cervantes, I. Flores %A Fong, N. %A Gamez-Djokic, M. %A Glenz, A. %A Gordon-McKeon, S. %A Heaton, T. J. %A Hederos, K. %A Heene, M. %A Mohr, A. J. Hofelich %A Högden, F. %A Hui, K. %A Johannesson, M. %A Kalodimos, J. %A Kaszubowski, E. %A Kennedy, D. M. %A Lei, R. %A Lindsay, T. A. %A Liverani, S. %A Madan, C. R. %A Molden, D. %A Molleman, E. %A Morey, R. D. %A Mulder, L. B. %A Nijstad, B. R. %A Pope, N. G. %A Pope, B. %A Prenoveau, J. M. %A Rink, F. %A Robusto, E. %A Roderique, H. %A Sandberg, A. %A Schlüter, E. %A Schönbrodt, F. D. %A Sherman, M. F. %A Sommer, S. A. %A Sotak, K. %A Spain, S. %A Spörlein, C. %A Stafford, T. %A Stefanutti, L. %A Tauber, S. %A Ullrich, J. %A Vianello, M. %A Wagenmakers, E.-J. %A Witkowiak, M. %A Yoon, S. %A Nosek, B. A. %D 2018 %I SAGE Publications %J Advances in Methods and Practices in Psychological Science %K crowdsourcing crowdsourcing_science data_analysis open_data open_materials transparency %N 3 %P 337--356 %R 10.1177/2515245917747646 %T Many Analysts, One Data Set: Making Transparent How Variations in Analytic Choices Affect Results %U https://journals.sagepub.com/doi/10.1177/2515245917747646 %V 1 %X Twenty-nine teams involving 61 analysts used the same data set to address the same research question: whether soccer referees are more likely to give red cards to dark-skin-toned players than to light-skin-toned players. Analytic approaches varied widely across the teams, and the estimated effect sizes ranged from 0.89 to 2.93 (Mdn = 1.31) in odds-ratio units. Twenty teams (69%) found a statistically significant positive effect, and 9 teams (31%) did not observe a significant relationship. Overall, the 29 different analyses used 21 unique combinations of covariates. Neither analysts’ prior beliefs about the effect of interest nor their level of expertise readily explained the variation in the outcomes of the analyses. Peer ratings of the quality of the analyses also did not account for the variability. These findings suggest that significant variation in the results of analyses of complex data may be difficult to avoid, even by experts with honest intentions. Crowdsourcing data analysis, a strategy in which numerous research teams are recruited to simultaneously investigate the same research question, makes transparent how defensible, yet subjective, analytic choices influence research results.

@article{silberzahn2018analysts, abstract = {Twenty-nine teams involving 61 analysts used the same data set to address the same research question: whether soccer referees are more likely to give red cards to dark-skin-toned players than to light-skin-toned players. Analytic approaches varied widely across the teams, and the estimated effect sizes ranged from 0.89 to 2.93 (Mdn = 1.31) in odds-ratio units. Twenty teams (69%) found a statistically significant positive effect, and 9 teams (31%) did not observe a significant relationship. Overall, the 29 different analyses used 21 unique combinations of covariates. Neither analysts’ prior beliefs about the effect of interest nor their level of expertise readily explained the variation in the outcomes of the analyses. Peer ratings of the quality of the analyses also did not account for the variability. These findings suggest that significant variation in the results of analyses of complex data may be difficult to avoid, even by experts with honest intentions. Crowdsourcing data analysis, a strategy in which numerous research teams are recruited to simultaneously investigate the same research question, makes transparent how defensible, yet subjective, analytic choices influence research results.}, added-at = {2019-04-07T10:57:51.000+0200}, author = {Silberzahn, R. and Uhlmann, E. L. and Martin, D. P. and Anselmi, P. and Aust, F. and Awtrey, E. and Bahn{\'{\i}}k, {\v{S}}. and Bai, F. and Bannard, C. and Bonnier, E. and Carlsson, R. and Cheung, F. and Christensen, G. and Clay, R. and Craig, M. A. and Rosa, A. Dalla and Dam, L. and Evans, M. H. and Cervantes, I. Flores and Fong, N. and Gamez-Djokic, M. and Glenz, A. and Gordon-McKeon, S. and Heaton, T. J. and Hederos, K. and Heene, M. and Mohr, A. J. Hofelich and Högden, F. and Hui, K. and Johannesson, M. and Kalodimos, J. and Kaszubowski, E. and Kennedy, D. M. and Lei, R. and Lindsay, T. A. and Liverani, S. and Madan, C. R. and Molden, D. and Molleman, E. and Morey, R. D. and Mulder, L. B. and Nijstad, B. R. and Pope, N. G. and Pope, B. and Prenoveau, J. M. and Rink, F. and Robusto, E. and Roderique, H. and Sandberg, A. and Schlüter, E. and Schönbrodt, F. D. and Sherman, M. F. and Sommer, S. A. and Sotak, K. and Spain, S. and Spörlein, C. and Stafford, T. and Stefanutti, L. and Tauber, S. and Ullrich, J. and Vianello, M. and Wagenmakers, E.-J. and Witkowiak, M. and Yoon, S. and Nosek, B. A.}, biburl = {https://www.bibsonomy.org/bibtex/28895dd2440b4e4c10566130dbb82a2d9/meneteqel}, doi = {10.1177/2515245917747646}, interhash = {7d24c4a1071248c7001e37b1099544c1}, intrahash = {8895dd2440b4e4c10566130dbb82a2d9}, journal = {Advances in Methods and Practices in Psychological Science}, keywords = {crowdsourcing crowdsourcing_science data_analysis open_data open_materials transparency}, language = {eng}, month = aug, number = 3, pages = {337--356}, publisher = {{SAGE} Publications}, timestamp = {2019-04-07T10:57:51.000+0200}, title = {Many Analysts, One Data Set: Making Transparent How Variations in Analytic Choices Affect Results}, url = {https://journals.sagepub.com/doi/10.1177/2515245917747646}, volume = 1, year = 2018 }