%0 Conference Paper %1 Ou_2023 %A Ou, Changkun %A Mayer, Sven %A Butz, Andreas Martin %B Proceedings of the 28th International Conference on Intelligent User Interfaces %D 2023 %I ACM %K expertise human-in-the-loop iui2023 %P 307-321 %R 10.1145/3581641.3584040 %T The Impact of Expertise in the Loop for Exploring Machine Rationality %U https://doi.org/10.1145%2F3581641.3584040 %X Human-in-the-loop optimization utilizes human expertise to guide machine optimizers iteratively and search for an optimal solution in a solution space. While prior empirical studies mainly investigated novices, we analyzed the impact of the levels of expertise on the outcome quality and corresponding subjective satisfaction. We conducted a study (N=60) in text, photo, and 3D mesh optimization contexts. We found that novices can achieve an expert level of quality performance, but participants with higher expertise led to more optimization iteration with more explicit preference while keeping satisfaction low. In contrast, novices were more easily satisfied and terminated faster. Therefore, we identified that experts seek more diverse outcomes while the machine reaches optimal results, and the observed behavior can be used as a performance indicator for human-in-the-loop system designers to improve underlying models. We inform future research to be cautious about the impact of user expertise when designing human-in-the-loop systems.

@inproceedings{Ou_2023, abstract = {Human-in-the-loop optimization utilizes human expertise to guide machine optimizers iteratively and search for an optimal solution in a solution space. While prior empirical studies mainly investigated novices, we analyzed the impact of the levels of expertise on the outcome quality and corresponding subjective satisfaction. We conducted a study (N=60) in text, photo, and 3D mesh optimization contexts. We found that novices can achieve an expert level of quality performance, but participants with higher expertise led to more optimization iteration with more explicit preference while keeping satisfaction low. In contrast, novices were more easily satisfied and terminated faster. Therefore, we identified that experts seek more diverse outcomes while the machine reaches optimal results, and the observed behavior can be used as a performance indicator for human-in-the-loop system designers to improve underlying models. We inform future research to be cautious about the impact of user expertise when designing human-in-the-loop systems. }, added-at = {2023-06-05T01:14:52.000+0200}, author = {Ou, Changkun and Mayer, Sven and Butz, Andreas Martin}, biburl = {https://www.bibsonomy.org/bibtex/241475d1540fb30bfc65f5c8723f23260/brusilovsky}, booktitle = {Proceedings of the 28th International Conference on Intelligent User Interfaces}, description = {The Impact of Expertise in the Loop for Exploring Machine Rationality | Proceedings of the 28th International Conference on Intelligent User Interfaces}, doi = {10.1145/3581641.3584040}, interhash = {1558ffb9b712b649d408422212100454}, intrahash = {41475d1540fb30bfc65f5c8723f23260}, keywords = {expertise human-in-the-loop iui2023}, month = mar, pages = {307-321}, publisher = {{ACM}}, timestamp = {2023-06-05T01:14:52.000+0200}, title = {The Impact of Expertise in the Loop for Exploring Machine Rationality}, url = {https://doi.org/10.1145%2F3581641.3584040}, year = 2023 }