%0 Journal Article %1 vandewiele2021overly %A Vandewiele, Gilles %A Dehaene, Isabelle %A Kovács, György %A Sterckx, Lucas %A Janssens, Olivier %A Ongenae, Femke %A De Backere, Femke %A De Turck, Filip %A Roelens, Kristien %A Decruyenaere, Johan %A Van Hoecke, Sofie %A Demeester, Thomas %D 2021 %J Artificial Intelligence in Medicine %K imbalance learning machine ml oversampling sampling %P 101987 %R https://doi.org/10.1016/j.artmed.2020.101987 %T Overly optimistic prediction results on imbalanced data: a case study of flaws and benefits when applying over-sampling %U https://www.sciencedirect.com/science/article/pii/S0933365720312525 %V 111 %X Information extracted from electrohysterography recordings could potentially prove to be an interesting additional source of information to estimate the risk on preterm birth. Recently, a large number of studies have reported near-perfect results to distinguish between recordings of patients that will deliver term or preterm using a public resource, called the Term/Preterm Electrohysterogram database. However, we argue that these results are overly optimistic due to a methodological flaw being made. In this work, we focus on one specific type of methodological flaw: applying over-sampling before partitioning the data into mutually exclusive training and testing sets. We show how this causes the results to be biased using two artificial datasets and reproduce results of studies in which this flaw was identified. Moreover, we evaluate the actual impact of over-sampling on predictive performance, when applied prior to data partitioning, using the same methodologies of related studies, to provide a realistic view of these methodologies’ generalization capabilities. We make our research reproducible by providing all the code under an open license.

@article{vandewiele2021overly, abstract = {Information extracted from electrohysterography recordings could potentially prove to be an interesting additional source of information to estimate the risk on preterm birth. Recently, a large number of studies have reported near-perfect results to distinguish between recordings of patients that will deliver term or preterm using a public resource, called the Term/Preterm Electrohysterogram database. However, we argue that these results are overly optimistic due to a methodological flaw being made. In this work, we focus on one specific type of methodological flaw: applying over-sampling before partitioning the data into mutually exclusive training and testing sets. We show how this causes the results to be biased using two artificial datasets and reproduce results of studies in which this flaw was identified. Moreover, we evaluate the actual impact of over-sampling on predictive performance, when applied prior to data partitioning, using the same methodologies of related studies, to provide a realistic view of these methodologies’ generalization capabilities. We make our research reproducible by providing all the code under an open license.}, added-at = {2021-08-16T15:31:44.000+0200}, author = {Vandewiele, Gilles and Dehaene, Isabelle and Kovács, György and Sterckx, Lucas and Janssens, Olivier and Ongenae, Femke and {De Backere}, Femke and {De Turck}, Filip and Roelens, Kristien and Decruyenaere, Johan and {Van Hoecke}, Sofie and Demeester, Thomas}, biburl = {https://www.bibsonomy.org/bibtex/29e7e53f39521d465e4ae516d4b90ffb6/jaeschke}, description = {Overly optimistic prediction results on imbalanced data: a case study of flaws and benefits when applying over-sampling - ScienceDirect}, doi = {https://doi.org/10.1016/j.artmed.2020.101987}, interhash = {d1b4624130503315247d6d2be8dabbaa}, intrahash = {9e7e53f39521d465e4ae516d4b90ffb6}, issn = {0933-3657}, journal = {Artificial Intelligence in Medicine}, keywords = {imbalance learning machine ml oversampling sampling}, pages = 101987, timestamp = {2021-08-16T15:31:44.000+0200}, title = {Overly optimistic prediction results on imbalanced data: a case study of flaws and benefits when applying over-sampling}, url = {https://www.sciencedirect.com/science/article/pii/S0933365720312525}, volume = 111, year = 2021 }