@inproceedings {pub3937,
	title = {From Explainability to Explanation: Using a Dialogue Setting to Elicit Annotations with Justifications.},
	author = {Nazia Attari AND Martin Ernst Heckmann AND David Schlangen},
	year = {2019},
	month = {September},
	abstract = {With the increasing use of machine learning techniques to support decision making comes an increasing demand for making their predictions more transparent. The field of explainable AI has set itself the goal to open the black box of current prediction methods, as it were, in order to provide not just predictions, but also justifications for them. Typically, however, already the training data for supervised machine learning is collected in a manner that treats the annotator as a black box, the internal workings of which remains unobserved. We present an annotation method where a task is given to a pair of annotators who collaborate on finding the best response. The hypothesis is that this a) leads to better quality responses, compared to non-collaborative annotation, as the annotators can think together and potentially catch each other{\textquoteright}s mistakes, and that b) this thinking together provides useful information in itself, as it at least partially reveals their reasoning steps. We have set up a small test with three different tasks (two rule-induction tasks and one text comprehension task) which we presented to pairs of annotators via a dialogue crowd sourcing tool. For comparison, we also presented the same tasks to crowd workers working on their own. Our analysis of the collected data shows that the difficulty of the task has a significant impact on the quality and quantity of annotators{\textquoteright} discussions. It is also important to set up the task in a manner such that the annotators know exactly what the goal is and how to achieve it.},
	publisher = {Association for Computational Linguistics},
	booktitle = {Proceedings of the 20th Annual SIGdial Meeting on Discourse and Dialogue},
	city = {Stockholm, Sweden}
}