%0 Journal Article %1 Yin2023 %A Yin, Zongyu %A Reuben, Federico %A Stepney, Susan %A Collins, Tom %D 2023 %J Machine Learning %K comparison meta %R 10.1007/s10994-023-06309-w %T Deep learning's shallow gains: a comparative evaluation of algorithms for automatic music generation %U https://doi.org/10.1007/s10994-023-06309-w %X Deep learning methods are recognised as state-of-the-art for many applications of machine learning. Recently, deep learning methods have emerged as a solution to the task of automatic music generation (AMG) using symbolic tokens in a target style, but their superiority over non-deep learning methods has not been demonstrated. Here, we conduct a listening study to comparatively evaluate several music generation systems along six musical dimensions: stylistic success, aesthetic pleasure, repetition or self-reference, melody, harmony, and rhythm. A range of models, both deep learning algorithms and other methods, are used to generate 30-s excerpts in the style of Classical string quartets and classical piano improvisations. Fifty participants with relatively high musical knowledge rate unlabelled samples of computer-generated and human-composed excerpts for the six musical dimensions. We use non-parametric Bayesian hypothesis testing to interpret the results, allowing the possibility of finding meaningful non-differences between systems' performance. We find that the strongest deep learning method, a reimplemented version of Music Transformer, has equivalent performance to a non-deep learning method, MAIA Markov, demonstrating that to date, deep learning does not outperform other methods for AMG. We also find there still remains a significant gap between any algorithmic method and human-composed excerpts.

@article{Yin2023, abstract = {Deep learning methods are recognised as state-of-the-art for many applications of machine learning. Recently, deep learning methods have emerged as a solution to the task of automatic music generation (AMG) using symbolic tokens in a target style, but their superiority over non-deep learning methods has not been demonstrated. Here, we conduct a listening study to comparatively evaluate several music generation systems along six musical dimensions: stylistic success, aesthetic pleasure, repetition or self-reference, melody, harmony, and rhythm. A range of models, both deep learning algorithms and other methods, are used to generate 30-s excerpts in the style of Classical string quartets and classical piano improvisations. Fifty participants with relatively high musical knowledge rate unlabelled samples of computer-generated and human-composed excerpts for the six musical dimensions. We use non-parametric Bayesian hypothesis testing to interpret the results, allowing the possibility of finding meaningful non-differences between systems' performance. We find that the strongest deep learning method, a reimplemented version of Music Transformer, has equivalent performance to a non-deep learning method, MAIA Markov, demonstrating that to date, deep learning does not outperform other methods for AMG. We also find there still remains a significant gap between any algorithmic method and human-composed excerpts.}, added-at = {2023-04-05T16:26:39.000+0200}, author = {Yin, Zongyu and Reuben, Federico and Stepney, Susan and Collins, Tom}, biburl = {https://www.bibsonomy.org/bibtex/2634917c31842f46625a2fb4691089d34/alex_h}, day = 21, description = {Deep learning’s shallow gains: a comparative evaluation of algorithms for automatic music generation | SpringerLink}, doi = {10.1007/s10994-023-06309-w}, interhash = {9cf0fd76cb56779c23ba39daabe15407}, intrahash = {634917c31842f46625a2fb4691089d34}, issn = {1573-0565}, journal = {Machine Learning}, keywords = {comparison meta}, month = mar, timestamp = {2023-04-05T16:26:39.000+0200}, title = {Deep learning's shallow gains: a comparative evaluation of algorithms for automatic music generation}, url = {https://doi.org/10.1007/s10994-023-06309-w}, year = 2023 }