@techreport {pub5781,
	title = {The Illusion of Competence: Evaluating the Effect of Explanations On Users{\textquoteright} Mental Models of Visual Question Answering Systems},
	author = {Sina Zarriess AND Nazia Attari AND Heiko Wersing AND Judith Sieker AND Simeon Sch{\"u}z AND Hendrik Buschmeier},
	year = {2024},
	month = {June},
	abstract = {In our study, we examine how participants/users perceive the limitations of an AI system when it encounters tasks it cannot perform perfectly. Our objective is to investigate whether providing explanations alongside model answers aids users in building an appropriate mental model of an AI{\textquoteright}s limitations. To accomplish this, we employ a visual question explanation task and evaluate both the accuracy of the models{\textquoteright} answers and the effectiveness of their accompanying explanations. In our study, we control the AI{\textquoteright}s limitations by manipulating the visual stimuli{\textquoteright}s presentation: during inference, systems either view the image in full color or solely in grayscale. Our goal is to determine whether participants can perceive the constraint of the AI, i.e., whether they can identify that it lacks the ability to perceive colors. To investigate the effect of providing model-generated explanations alongside the model answers, we undertake two studies: one where participants view both the AI{\textquoteright}s answer and the accompanying explanation, and another where participants only observe the AI{\textquoteright}s answer without the explanation. We hypothesize that users will gain a clearer understanding of a AI{\textquoteright}s capabilities and, thus will get a better mental model of the system when explanations are provided alongside its answers. Our results, however, show that explanations do not have this effect. Rather than allowing users to assess the AI{\textquoteright}s limitations in a more differentiated way, explanations generally improve the users{\textquoteright} perception of system competence, regardless of its actual performance.},
	publisher = {arXiv}
}