%0 Conference Paper %1 pmlr-v139-touvron21a %A Touvron, Hugo %A Cord, Matthieu %A Douze, Matthijs %A Massa, Francisco %A Sablayrolles, Alexandre %A Jegou, Herve %B Proceedings of the 38th International Conference on Machine Learning %D 2021 %E Meila, Marina %E Zhang, Tong %I PMLR %K %P 10347--10357 %T Training data-efficient image transformers & distillation through attention %U https://proceedings.mlr.press/v139/touvron21a.html %V 139 %X Recently, neural networks purely based on attention were shown to address image understanding tasks such as image classification. These high-performing vision transformers are pre-trained with hundreds of millions of images using a large infrastructure, thereby limiting their adoption. In this work, we produce competitive convolution-free transformers trained on ImageNet only using a single computer in less than 3 days. Our reference vision transformer (86M parameters) achieves top-1 accuracy of 83.1% (single-crop) on ImageNet with no external data. We also introduce a teacher-student strategy specific to transformers. It relies on a distillation token ensuring that the student learns from the teacher through attention, typically from a convnet teacher. The learned transformers are competitive (85.2% top-1 acc.) with the state of the art on ImageNet, and similarly when transferred to other tasks. We will share our code and models.

@inproceedings{pmlr-v139-touvron21a, abstract = {Recently, neural networks purely based on attention were shown to address image understanding tasks such as image classification. These high-performing vision transformers are pre-trained with hundreds of millions of images using a large infrastructure, thereby limiting their adoption. In this work, we produce competitive convolution-free transformers trained on ImageNet only using a single computer in less than 3 days. Our reference vision transformer (86M parameters) achieves top-1 accuracy of 83.1% (single-crop) on ImageNet with no external data. We also introduce a teacher-student strategy specific to transformers. It relies on a distillation token ensuring that the student learns from the teacher through attention, typically from a convnet teacher. The learned transformers are competitive (85.2% top-1 acc.) with the state of the art on ImageNet, and similarly when transferred to other tasks. We will share our code and models.}, added-at = {2022-07-11T20:00:04.000+0200}, author = {Touvron, Hugo and Cord, Matthieu and Douze, Matthijs and Massa, Francisco and Sablayrolles, Alexandre and Jegou, Herve}, biburl = {https://www.bibsonomy.org/bibtex/2e432cd6b9882f5169202f17023c759dd/simonh}, booktitle = {Proceedings of the 38th International Conference on Machine Learning}, description = {Training data-efficient image transformers & distillation through attention}, editor = {Meila, Marina and Zhang, Tong}, interhash = {d0078fd7ff7fc87997d38aab0b6b536e}, intrahash = {e432cd6b9882f5169202f17023c759dd}, keywords = {}, month = {18--24 Jul}, pages = {10347--10357}, pdf = {http://proceedings.mlr.press/v139/touvron21a/touvron21a.pdf}, publisher = {PMLR}, series = {Proceedings of Machine Learning Research}, timestamp = {2022-07-12T10:09:24.000+0200}, title = {Training data-efficient image transformers & distillation through attention}, url = {https://proceedings.mlr.press/v139/touvron21a.html}, volume = 139, year = 2021 }