%0 Journal Article %1 anda09pre %A Anda, Bente C.D. %A Sjoberg, Dag I.K. %A Mockus, Audris %C Los Alamitos, CA, USA %D 5555 %I IEEE Computer Society %J IEEE Transactions on Software Engineering %K imported %N 1 %T Variability and Reproducibility in Software Engineering: A Study of Four Companies that Developed the Same System %U http://doi.ieeecomputersociety.org/10.1109/TSE.2008.89 %V 99 %X The scientific study of a phenomenon requires it to be reproducible. Mature engineering industries are recognized by projects and products that are, to some extent, reproducible. However, reproducibility in software engineering (SE) has not been thoroughly investigated, although generalizing the results of SE studies depend on SE phenomena being reproducible. We report a longitudinal multiple-case study of variations and reproducibility in software development, from bidding to deployment, on the basis of the same requirements specification. In a call for tender to 81 consultancy companies, 35 responded. Four of them developed the system independently. The firm price, planned schedule, and planned development process, had, respectively, "low", "low", and "medium" reproducibility. The contractor's internal costs, actual lead time, and schedule overrun of the projects had, respectively, "medium", "high", and "low" reproducibility. The quality dimensions of the delivered products, reliability, usability, and maintainability had, respectively, "low", "high", and "low" reproducibility. We proposed a coarse-grained model that uses software process inputs to predict key project outcomes. The comparisons of the model's predictions with the actual outcomes indicate some reproducibility. This initial work may contribute to developing more accurate models, but making SE more reproducible remains a great challenge for SE research, education, and industry.

@article{anda09pre, abstract = {The scientific study of a phenomenon requires it to be reproducible. Mature engineering industries are recognized by projects and products that are, to some extent, reproducible. However, reproducibility in software engineering (SE) has not been thoroughly investigated, although generalizing the results of SE studies depend on SE phenomena being reproducible. We report a longitudinal multiple-case study of variations and reproducibility in software development, from bidding to deployment, on the basis of the same requirements specification. In a call for tender to 81 consultancy companies, 35 responded. Four of them developed the system independently. The firm price, planned schedule, and planned development process, had, respectively, "low", "low", and "medium" reproducibility. The contractor's internal costs, actual lead time, and schedule overrun of the projects had, respectively, "medium", "high", and "low" reproducibility. The quality dimensions of the delivered products, reliability, usability, and maintainability had, respectively, "low", "high", and "low" reproducibility. We proposed a coarse-grained model that uses software process inputs to predict key project outcomes. The comparisons of the model's predictions with the actual outcomes indicate some reproducibility. This initial work may contribute to developing more accurate models, but making SE more reproducible remains a great challenge for SE research, education, and industry.}, added-at = {2008-12-18T18:33:14.000+0100}, address = {Los Alamitos, CA, USA}, author = {Anda, Bente C.D. and Sjoberg, Dag I.K. and Mockus, Audris}, biburl = {https://www.bibsonomy.org/bibtex/2b91900ab2723abd62935af3877134532/neilernst}, description = {Digital Library}, interhash = {40e75964bf862706c32d827bc56e0736}, intrahash = {b91900ab2723abd62935af3877134532}, issn = {0098-5589}, journal = {IEEE Transactions on Software Engineering}, keywords = {imported}, number = 1, publisher = {IEEE Computer Society}, timestamp = {2008-12-18T18:33:14.000+0100}, title = {Variability and Reproducibility in Software Engineering: A Study of Four Companies that Developed the Same System}, url = {http://doi.ieeecomputersociety.org/10.1109/TSE.2008.89}, volume = 99, year = 5555 }