OBJECTIVE: To evaluate the impact of non-blinded outcome assessment on estimated treatment effects in randomised clinical trials with binary outcomes. DESIGN: Systematic review of trials with both blinded and non-blinded assessment of the same binary outcome. For each trial we calculated the ratio of the odds ratios--the odds ratio from non-blinded assessments relative to the corresponding odds ratio from blinded assessments. A ratio of odds ratios <1 indicated that non-blinded assessors generated more optimistic effect estimates than blinded assessors. We pooled the individual ratios of odds ratios with inverse variance random effects meta-analysis and explored reasons for variation in ratios of odds ratios with meta-regression. We also analysed rates of agreement between blinded and non-blinded assessors and calculated the number of patients needed to be reclassified to neutralise any bias. DATA SOURCES: PubMed, Embase, PsycINFO, CINAHL, Cochrane Central Register of Controlled Trials, HighWire Press, and Google Scholar. ELIGIBILITY CRITERIA FOR SELECTING STUDIES: Randomised clinical trials with blinded and non-blinded assessment of the same binary outcome. RESULTS: We included 21 trials in the main analysis (with 4391 patients); eight trials provided individual patient data. Outcomes in most trials were subjective--for example, qualitative assessment of the patient's function. The ratio of the odds ratios ranged from 0.02 to 14.4. The pooled ratio of odds ratios was 0.64 (95% confidence interval 0.43 to 0.96), indicating an average exaggeration of the non-blinded odds ratio by 36%. We found no significant association between low ratios of odds ratios and scores for outcome subjectivity (P=0.27); non-blinded assessor's overall involvement in the trial (P=0.60); or outcome vulnerability to non-blinded patients (P=0.52). Blinded and non-blinded assessors agreed in a median of 78% of assessments (interquartile range 64-90%) in the 12 trials with available data. The exaggeration of treatment effects associated with non-blinded assessors was induced by the misclassification of a median of 3% of the assessed patients per trial (1-7%). CONCLUSIONS: On average, non-blinded assessors of subjective binary outcomes generated substantially biased effect estimates in randomised clinical trials, exaggerating odds ratios by 36%. This bias was compatible with a high rate of agreement between blinded and non-blinded outcome assessors and driven by the misclassification of few patients.
%0 Journal Article
%1 Hrobjartsson2012
%A Hróbjartsson, Asbjørn
%A Thomsen, Ann Sofia Skou
%A Emanuelsson, Frida
%A Tendal, Britta
%A Hilden, Jørgen
%A Boutron, Isabelle
%A Ravaud, Philippe
%A Brorson, Stig
%D 2012
%J BMJ (Clinical research ed.)
%K Double-BlindMethod Humans ObserverVariation OddsRatio OutcomeAssessment(HealthCare) OutcomeAssessment(HealthCare):methods Single-BlindMethod TreatmentOutcome RCT
%P e1119
%R 10.1136/bmj.e1119
%T Observer bias in randomised clinical trials with binary outcomes: systematic review of trials with both blinded and non-blinded outcome assessors.
%U http://www.ncbi.nlm.nih.gov/pubmed/22371859
%V 344
%X OBJECTIVE: To evaluate the impact of non-blinded outcome assessment on estimated treatment effects in randomised clinical trials with binary outcomes. DESIGN: Systematic review of trials with both blinded and non-blinded assessment of the same binary outcome. For each trial we calculated the ratio of the odds ratios--the odds ratio from non-blinded assessments relative to the corresponding odds ratio from blinded assessments. A ratio of odds ratios <1 indicated that non-blinded assessors generated more optimistic effect estimates than blinded assessors. We pooled the individual ratios of odds ratios with inverse variance random effects meta-analysis and explored reasons for variation in ratios of odds ratios with meta-regression. We also analysed rates of agreement between blinded and non-blinded assessors and calculated the number of patients needed to be reclassified to neutralise any bias. DATA SOURCES: PubMed, Embase, PsycINFO, CINAHL, Cochrane Central Register of Controlled Trials, HighWire Press, and Google Scholar. ELIGIBILITY CRITERIA FOR SELECTING STUDIES: Randomised clinical trials with blinded and non-blinded assessment of the same binary outcome. RESULTS: We included 21 trials in the main analysis (with 4391 patients); eight trials provided individual patient data. Outcomes in most trials were subjective--for example, qualitative assessment of the patient's function. The ratio of the odds ratios ranged from 0.02 to 14.4. The pooled ratio of odds ratios was 0.64 (95% confidence interval 0.43 to 0.96), indicating an average exaggeration of the non-blinded odds ratio by 36%. We found no significant association between low ratios of odds ratios and scores for outcome subjectivity (P=0.27); non-blinded assessor's overall involvement in the trial (P=0.60); or outcome vulnerability to non-blinded patients (P=0.52). Blinded and non-blinded assessors agreed in a median of 78% of assessments (interquartile range 64-90%) in the 12 trials with available data. The exaggeration of treatment effects associated with non-blinded assessors was induced by the misclassification of a median of 3% of the assessed patients per trial (1-7%). CONCLUSIONS: On average, non-blinded assessors of subjective binary outcomes generated substantially biased effect estimates in randomised clinical trials, exaggerating odds ratios by 36%. This bias was compatible with a high rate of agreement between blinded and non-blinded outcome assessors and driven by the misclassification of few patients.
%@ 1756-1833; 0959-535X
@article{Hrobjartsson2012,
abstract = {OBJECTIVE: To evaluate the impact of non-blinded outcome assessment on estimated treatment effects in randomised clinical trials with binary outcomes. DESIGN: Systematic review of trials with both blinded and non-blinded assessment of the same binary outcome. For each trial we calculated the ratio of the odds ratios--the odds ratio from non-blinded assessments relative to the corresponding odds ratio from blinded assessments. A ratio of odds ratios <1 indicated that non-blinded assessors generated more optimistic effect estimates than blinded assessors. We pooled the individual ratios of odds ratios with inverse variance random effects meta-analysis and explored reasons for variation in ratios of odds ratios with meta-regression. We also analysed rates of agreement between blinded and non-blinded assessors and calculated the number of patients needed to be reclassified to neutralise any bias. DATA SOURCES: PubMed, Embase, PsycINFO, CINAHL, Cochrane Central Register of Controlled Trials, HighWire Press, and Google Scholar. ELIGIBILITY CRITERIA FOR SELECTING STUDIES: Randomised clinical trials with blinded and non-blinded assessment of the same binary outcome. RESULTS: We included 21 trials in the main analysis (with 4391 patients); eight trials provided individual patient data. Outcomes in most trials were subjective--for example, qualitative assessment of the patient's function. The ratio of the odds ratios ranged from 0.02 to 14.4. The pooled ratio of odds ratios was 0.64 (95% confidence interval 0.43 to 0.96), indicating an average exaggeration of the non-blinded odds ratio by 36%. We found no significant association between low ratios of odds ratios and scores for outcome subjectivity (P=0.27); non-blinded assessor's overall involvement in the trial (P=0.60); or outcome vulnerability to non-blinded patients (P=0.52). Blinded and non-blinded assessors agreed in a median of 78% of assessments (interquartile range 64-90%) in the 12 trials with available data. The exaggeration of treatment effects associated with non-blinded assessors was induced by the misclassification of a median of 3% of the assessed patients per trial (1-7%). CONCLUSIONS: On average, non-blinded assessors of subjective binary outcomes generated substantially biased effect estimates in randomised clinical trials, exaggerating odds ratios by 36%. This bias was compatible with a high rate of agreement between blinded and non-blinded outcome assessors and driven by the misclassification of few patients.},
added-at = {2023-02-03T11:44:35.000+0100},
author = {Hróbjartsson, Asbjørn and Thomsen, Ann Sofia Skou and Emanuelsson, Frida and Tendal, Britta and Hilden, Jørgen and Boutron, Isabelle and Ravaud, Philippe and Brorson, Stig},
biburl = {https://www.bibsonomy.org/bibtex/2870a962e0573007adb073b5bf344928c/jepcastel},
city = {Nordic Cochrane Centre, Rigshospitalet Department 3343, Blegdamsvej 9, 2100 Copenhagen O, Denmark. ah@cochrane.dk},
doi = {10.1136/bmj.e1119},
interhash = {c377d533cd190be0d0b0904b1a27e892},
intrahash = {870a962e0573007adb073b5bf344928c},
isbn = {1756-1833; 0959-535X},
issn = {1756-1833},
journal = {BMJ (Clinical research ed.)},
keywords = {Double-BlindMethod Humans ObserverVariation OddsRatio OutcomeAssessment(HealthCare) OutcomeAssessment(HealthCare):methods Single-BlindMethod TreatmentOutcome RCT},
month = {1},
note = {6709<m:linebreak></m:linebreak>JID: 8900488; epublish;<m:linebreak></m:linebreak>Emmascarament; Bias},
pages = {e1119},
pmid = {22371859},
timestamp = {2023-05-04T08:59:38.000+0200},
title = {Observer bias in randomised clinical trials with binary outcomes: systematic review of trials with both blinded and non-blinded outcome assessors.},
url = {http://www.ncbi.nlm.nih.gov/pubmed/22371859},
volume = 344,
year = 2012
}