@inproceedings{3f3cc42cae0d465c980d6b357a459558,
title = "Annotating multiparty discourse: Challenges for agreement metrics",
abstract = "To computationally model discourse phenomena such as argumentation we need corpora with reliable annotation of the phenomena under study. Annotating complex discourse phenomena poses two challenges: fuzziness of unit boundaries and the need for multiple annotators. We show that current metrics for inter-annotator agreement (IAA) such as P/R/F1 and Krippendorff's α provide inconsistent results for the same text. In addition, IAA metrics do not tell us what parts of a text are easier or harder for human judges to annotate and so do not provide sufficiently specific information for evaluating systems that automatically identify discourse units. We propose a hierarchical clustering approach that aggregates overlapping text segments of text identified by multiple annotators; the more annotators who identify a text segment, the easier we assume that the text segment is to annotate. The clusters make it possible to quantify the extent of agreement judges show about text segments; this information can be used to assess the output of systems that automatically identify discourse units.",
author = "Nina Wacholder and Smaranda Muresan and Debanjan Ghosh and Mark Aakhus",
note = "Publisher Copyright: {\textcopyright} LAW 2014 - 8th Linguistic Annotation Workshop, in conjunction with COLING 2014 - Proceedings of the Workshop. All rights reserved.; 8th Linguistic Annotation Workshop, LAW 2014, in conjunction with COLING 2014 ; Conference date: 23-08-2014 Through 24-08-2014",
year = "2020",
language = "English (US)",
series = "LAW 2014 - 8th Linguistic Annotation Workshop, in conjunction with COLING 2014 - Proceedings of the Workshop",
publisher = "Association for Computational Linguistics (ACL)",
pages = "120--128",
editor = "Lori Levin and Manfred Stede",
booktitle = "LAW 2014 - 8th Linguistic Annotation Workshop, in conjunction with COLING 2014 - Proceedings of the Workshop",
}