%0 Journal Article %1 Braga-Neto:2004:CBR:1090591.1090602 %A Braga-Neto, Ulisses %A Hashimoto, Ronaldo %A Dougherty, Edward R. %A Nguyen, Danh V. %A Carroll, Raymond J. %C Oxford, UK %D 2004 %I Oxford University Press %J Bioinformatics %K classification discriminant discriminant-analysis discrimination gene genetics high-dimensional high-dimensional-data %P 253--258 %R 10.1093/bioinformatics/btg399 %T Is cross-validation better than resubstitution for ranking genes? %U http://dx.doi.org/10.1093/bioinformatics/btg399 %V 20 %X <b>Motivation:</b> Ranking gene feature sets is a key issue for both phenotype classification, for instance, tumor classification in a DNA microarray experiment, and prediction in the context of genetic regulatory networks. Two broad methods are available to estimate the error (misclassification rate) of a classifier. Resubstitution fits a single classifier to the data, and applies this classifier in turn to each data observation. Cross-validation (in leave-one-out form) removes each observation in turn, constructs the classifier, and then computes whether this leave-one-out classifier correctly classifies the deleted observation. Resubstitution typically underestimates classifier error, severely so in many cases. Cross-validation has the advantage of producing an effectively unbiased error estimate, but the estimate is highly variable. In many applications it is not the misclassification rate <i>per se</i> that is of interest, but rather the construction of gene sets that have the potential to classify or predict. Hence, one needs to rank feature sets based on their performance.</p> <p><b>Results:</b> A model-based approach is used to compare the ranking performances of resubstitution and cross-validation for classification based on real-valued feature sets and for prediction in the context of probabilistic Boolean networks (PBNs). For classification, a Gaussian model is considered, along with classification via linear discriminant analysis and the 3-nearest-neighbor classification rule. Prediction is examined in the steady-distribution of a PBN. Three metrics are proposed to compare feature-set ranking based on error estimation with ranking based on the true error, which is known owing to the model-based approach. In all cases, resubstitution is competitive with cross-validation relative to ranking accuracy. This is in addition to the enormous savings in computation time afforded by resubstitution.

@article{Braga-Neto:2004:CBR:1090591.1090602, abstract = {<b>Motivation:</b> Ranking gene feature sets is a key issue for both phenotype classification, for instance, tumor classification in a DNA microarray experiment, and prediction in the context of genetic regulatory networks. Two broad methods are available to estimate the error (misclassification rate) of a classifier. Resubstitution fits a single classifier to the data, and applies this classifier in turn to each data observation. Cross-validation (in leave-one-out form) removes each observation in turn, constructs the classifier, and then computes whether this leave-one-out classifier correctly classifies the deleted observation. Resubstitution typically underestimates classifier error, severely so in many cases. Cross-validation has the advantage of producing an effectively unbiased error estimate, but the estimate is highly variable. In many applications it is not the misclassification rate <i>per se</i> that is of interest, but rather the construction of gene sets that have the potential to classify or predict. Hence, one needs to rank feature sets based on their performance.</p> <p><b>Results:</b> A model-based approach is used to compare the ranking performances of resubstitution and cross-validation for classification based on real-valued feature sets and for prediction in the context of probabilistic Boolean networks (PBNs). For classification, a Gaussian model is considered, along with classification via linear discriminant analysis and the 3-nearest-neighbor classification rule. Prediction is examined in the steady-distribution of a PBN. Three metrics are proposed to compare feature-set ranking based on error estimation with ranking based on the true error, which is known owing to the model-based approach. In all cases, resubstitution is competitive with cross-validation relative to ranking accuracy. This is in addition to the enormous savings in computation time afforded by resubstitution.}, acmid = {1090602}, added-at = {2011-05-09T18:21:45.000+0200}, address = {Oxford, UK}, author = {Braga-Neto, Ulisses and Hashimoto, Ronaldo and Dougherty, Edward R. and Nguyen, Danh V. and Carroll, Raymond J.}, biburl = {https://www.bibsonomy.org/bibtex/2b150cb261ae44bfd884363b64348db6b/vivion}, description = {Is cross-validation better than resubstitution for ranking genes?}, doi = {10.1093/bioinformatics/btg399}, interhash = {415a5eedfc25da5935ab9f9477a270c6}, intrahash = {b150cb261ae44bfd884363b64348db6b}, issn = {1367-4803}, issue = {2}, journal = {Bioinformatics}, keywords = {classification discriminant discriminant-analysis discrimination gene genetics high-dimensional high-dimensional-data}, month = {January}, numpages = {6}, pages = {253--258}, publisher = {Oxford University Press}, timestamp = {2011-05-09T18:21:45.000+0200}, title = {Is cross-validation better than resubstitution for ranking genes?}, url = {http://dx.doi.org/10.1093/bioinformatics/btg399}, volume = 20, year = 2004 }