%0 Journal Article %1 10.1162/coli_a_00482 %A Choenni, Rochelle %A Garrette, Dan %A Shutova, Ekaterina %D 2023 %J Computational Linguistics %K low-resource-nlp related_works %P 1-29 %R 10.1162/coli_a_00482 %T Cross-Lingual Transfer with Language-Specific Subnetworks for Low-Resource Dependency Parsing %U https://doi.org/10.1162/coli\_a\_00482 %X Large multilingual language models typically share their parameters across all languages, which enables cross-lingual task transfer, but learning can also be hindered when training updates from different languages are in conflict. In this article, we propose novel methods for using language-specific subnetworks, which control cross-lingual parameter sharing, to reduce conflicts and increase positive transfer during fine-tuning. We introduce dynamic subnetworks, which are jointly updated with the model, and we combine our methods with meta-learning, an established, but complementary, technique for improving cross-lingual transfer. Finally, we provide extensive analyses of how each of our methods affects the models.

@article{10.1162/coli_a_00482, abstract = {{Large multilingual language models typically share their parameters across all languages, which enables cross-lingual task transfer, but learning can also be hindered when training updates from different languages are in conflict. In this article, we propose novel methods for using language-specific subnetworks, which control cross-lingual parameter sharing, to reduce conflicts and increase positive transfer during fine-tuning. We introduce dynamic subnetworks, which are jointly updated with the model, and we combine our methods with meta-learning, an established, but complementary, technique for improving cross-lingual transfer. Finally, we provide extensive analyses of how each of our methods affects the models.}}, added-at = {2023-11-09T09:32:42.000+0100}, author = {Choenni, Rochelle and Garrette, Dan and Shutova, Ekaterina}, biburl = {https://www.bibsonomy.org/bibtex/2fb4870a21e0ef8a4eb48182258bb9c68/tomvoelker}, doi = {10.1162/coli_a_00482}, eprint = {https://direct.mit.edu/coli/article-pdf/doi/10.1162/coli\_a\_00482/2141924/coli\_a\_00482.pdf}, interhash = {e9088d3de51abe1f9d5d59b36bbcde51}, intrahash = {fb4870a21e0ef8a4eb48182258bb9c68}, issn = {0891-2017}, journal = {Computational Linguistics}, keywords = {low-resource-nlp related_works}, month = {07}, pages = {1-29}, timestamp = {2023-11-09T09:32:42.000+0100}, title = {Cross-Lingual Transfer with Language-Specific Subnetworks for Low-Resource Dependency Parsing}, url = {https://doi.org/10.1162/coli\_a\_00482}, year = 2023 }