%0 Conference Paper %1 Gabbay2024 %A Gabbay, Hagit %A Cohen, Anat %B Proceedings of the Eleventh ACM Conference on Learning @ Scale %C New York, NY, USA %D 2024 %I Association for Computing Machinery %K chatgpt feedback las2024 llm progtutor %P 177–187 %R 10.1145/3657604.3662040 %T Combining LLM-Generated and Test-Based Feedback in a MOOC for Programming %U https://doi.org/10.1145/3657604.3662040 %X In large-scale programming courses, providing learners with immediate and effective feedback is a significant challenge. This study explores the potential of Large Language Models (LLMs) to generate feedback on code assignments and to address the gaps in Automated Test-based Feedback (ATF) tools commonly employed in programming courses. We applied dedicated metrics in a Massive Open Online Course (MOOC) on programming to assess the correctness of feedback generated by two models, GPT-3.5-turbo and GPT-4, using a reliable ATF as a benchmark. The findings point to effective error detection, yet the feedback is often inaccurate, with GPT-4 outperforming GPT-3.5-turbo. We used insights gained from the prompt practices to develop Gipy, an application for submitting course assignments and obtaining LLM-generated feedback. Learners participating in a field experiment perceived the feedback provided by Gipy as moderately valuable, while at the same time recognizing its potential to complement ATF. Given the learners' critique and their awareness of the limitations of LLM-generated feedback, the studied implementation may be able to take advantage of the best of both ATF and LLMs as feedback resources. Further research is needed to assess the impact of LLM-generated feedback on learning outcomes and explore the capabilities of more advanced models. %@ 9798400706332

@inproceedings{Gabbay2024, abstract = {In large-scale programming courses, providing learners with immediate and effective feedback is a significant challenge. This study explores the potential of Large Language Models (LLMs) to generate feedback on code assignments and to address the gaps in Automated Test-based Feedback (ATF) tools commonly employed in programming courses. We applied dedicated metrics in a Massive Open Online Course (MOOC) on programming to assess the correctness of feedback generated by two models, GPT-3.5-turbo and GPT-4, using a reliable ATF as a benchmark. The findings point to effective error detection, yet the feedback is often inaccurate, with GPT-4 outperforming GPT-3.5-turbo. We used insights gained from the prompt practices to develop Gipy, an application for submitting course assignments and obtaining LLM-generated feedback. Learners participating in a field experiment perceived the feedback provided by Gipy as moderately valuable, while at the same time recognizing its potential to complement ATF. Given the learners' critique and their awareness of the limitations of LLM-generated feedback, the studied implementation may be able to take advantage of the best of both ATF and LLMs as feedback resources. Further research is needed to assess the impact of LLM-generated feedback on learning outcomes and explore the capabilities of more advanced models.}, added-at = {2024-07-19T18:15:37.000+0200}, address = {New York, NY, USA}, author = {Gabbay, Hagit and Cohen, Anat}, biburl = {https://www.bibsonomy.org/bibtex/28da516a78a3e520ea06c350f0fb1ccc0/brusilovsky}, booktitle = {Proceedings of the Eleventh ACM Conference on Learning @ Scale}, day = 15, description = {Combining LLM-Generated and Test-Based Feedback in a MOOC for Programming | Proceedings of the Eleventh ACM Conference on Learning @ Scale}, doi = {10.1145/3657604.3662040}, interhash = {cf3c7de177f358b13c486949377c85f8}, intrahash = {8da516a78a3e520ea06c350f0fb1ccc0}, isbn = {9798400706332}, keywords = {chatgpt feedback las2024 llm progtutor}, location = {Atlanta, GA, USA}, month = {7}, pages = {177–187}, publisher = {Association for Computing Machinery}, series = {L@S '24}, timestamp = {2024-07-19T18:15:37.000+0200}, title = {Combining LLM-Generated and Test-Based Feedback in a MOOC for Programming}, url = {https://doi.org/10.1145/3657604.3662040}, year = 2024 }