Source code for hydra.utils.evaluation

"""Model evaluation utilities."""

import urllib.parse
import json

from collections import Counter
from typing import Any, override

from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, confusion_matrix

from morfeusz2 import Morfeusz

from .hybrid_tree import tree2dict, correct_lemma, TreeNode
from .ner import get_nes
from .constants import MASK_VALUE, HEADS, DEPRELS, GRAPH_HEADS, GRAPH_DEPRELS, LEMMAS, TAGS, NE_IOBS
from .types import TokenList, LabelDict

def _collect_spans(
    tree: TreeNode,
    spans: list[tuple[tuple[int, ...], str, bool]],
    labeled: bool,
    headed: bool,
    discontinuous_only: bool,
    skip_categories: list[str] | None = None
) -> set[int]:
    if headed and not labeled:
        raise RuntimeError('cant’t use headed=True with labeled=False')
    if not tree.children:
        assert (tree.from_index is not None)
        return {tree.from_index}
    if skip_categories is None:
        skip_categories = []
    span = set()
    for child in tree.children:
        span.update(_collect_spans(child, spans, labeled, headed, discontinuous_only, skip_categories))
    if tree.category not in skip_categories:
        sorted_span = tuple(sorted(span))
        if discontinuous_only and sorted_span == tuple(range(sorted_span[0], sorted_span[-1] + 1)):
            # continuous constituent
            pass
        else:
            assert (tree.category is not None)
            spans += [(sorted_span, tree.category if labeled else 'SPAN', tree.is_head if headed else False)]
    return span

def _tree2spans(
    tree: TreeNode,
    labeled: bool = True,
    headed: bool = False,
    discontinuous_only: bool = False,
    skip_categories: list[str] | None = None
) -> set[tuple[tuple[int, ...], str, bool]]:
    spans: list[tuple[tuple[int, ...], str, bool]] = []
    _collect_spans(tree, spans, labeled, headed, discontinuous_only, skip_categories)
    # TODO
    #try:
    #    assert(len(spans) == len(set(spans)))
    #except:
    #    show_tree(tree)
    #    (display(spans))
    #    1/0
    return set(spans)

def _collect_edges(tree: TreeNode, edges: list[tuple[str, str, int, int, str]]) -> int:
    if not tree.children:
        assert (tree.from_index is not None)
        return tree.from_index
    head_index = None
    child_edges = []
    for child in tree.children:
        child_head_index = _collect_edges(child, edges)
        child_cat = child.category if child.children else '<TERMINAL>'
        assert (child_cat is not None)
        deprel = child.attributes['deprel'] if not child.is_head else '<HEAD>'
        child_edges.append((child_head_index, child_cat, deprel))
        if child.is_head:
            assert(head_index is None)
            head_index = child_head_index
    assert(head_index is not None)
    assert(tree.category is not None)
    for child_head_index, child_cat, deprel in child_edges:
        assert (child_cat is not None)
        edges.append((tree.category, child_cat, head_index, child_head_index, deprel))
    return head_index

def _tree2edges(tree: TreeNode) -> set[tuple[str, str, int, int, str]]:
    edges: list[tuple[str, str, int, int, str]] = []
    _collect_edges(tree, edges)
    #TODO?
    #if len(edges) != len(set(edges)):
    #    #show_tree(tree)
    #    #1/0
    return set(edges)

[docs] class EvaluationMetric(object): """Base class for any evaluation metric.""" def __init__(self, name: str) -> None: self.name = name self.total = 0 self.masked = 0
[docs] def update( self, tokens: TokenList, tags_true: LabelDict, tags_pred: LabelDict, tree_true: TreeNode | None, tree_pred: TreeNode | None, ) -> None: """Gather any statistics needed for calculating this metric from a true/predicted sentence pair. Subclasses must implement this method. :raises NotImplementedError: Always. """ raise NotImplementedError
[docs] def calculate(self) -> dict[str, Any]: """Calculate the overall metric result after being fed all the data. Returns: any scores and statistics assoaciated with the metric, number of instances: total seen and excluded because of masking. """ return { 'result' : self.do_calculate(), 'total instances' : self.total, 'masked instances' : self.masked, }
[docs] def do_calculate(self) -> Any: """Perform metric-specific calculations of the overall metric result after being fed all the data. Subclasses must implement this method. :raises NotImplementedError: Always. """ raise NotImplementedError
[docs] class TokenMetric(EvaluationMetric): """Base class for any evaluation metric operating on token level.""" def __init__(self, name: str) -> None: super().__init__(name) self.true: list[Any] = [] self.pred: list[Any] = []
[docs] def get_labels(self, tags: LabelDict) -> list[Any] | None: """Gather token labels to be compared between gold and predicted sequences. Subclasses must implement this method. :raises NotImplementedError: Always. """ raise NotImplementedError
[docs] @override def update( self, tokens: TokenList, tags_true: LabelDict, tags_pred: LabelDict, tree_true: TreeNode | None, tree_pred: TreeNode | None, ) -> None: """Gather token-level labels needed for calculating this metric from a true/predicted sentence pair. Metric-specific label gathering logic must be implemented by overriding :meth:`~hydra.utils.evaluation.TokenMetric.get_labels`. """ self.total += 1 labels_true = self.get_labels(tags_true) if labels_true is not None: self.true += labels_true labels_pred = self.get_labels(tags_pred) assert (labels_pred is not None) self.pred += labels_pred else: self.masked += 1
[docs] def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any: """Calculate a specific metric operating on token level. Subclasses must implement this method. :raises NotImplementedError: Always. """ raise NotImplementedError
[docs] @override def do_calculate(self) -> Any: """Perform metric-specific calculations of the overall metric result after being fed all the data. Metric-specific calculation logic must be implemented by overriding :meth:`~hydra.utils.evaluation.TokenMetric.calculate_token_metric`. Returns: calculated result or `'N/A'` if not possible. """ # for Składnica merged data with masked deprels... filtered = [(t, p) for t, p in zip(self.true, self.pred) if t is not None] if filtered: true, pred = map(list, zip(*filtered)) return self.calculate_token_metric(true, pred) else: return 'N/A'
[docs] class AccuracyMetric(TokenMetric): """Base class for any evaluation metric calculating token-level accuracy.""" def __init__(self, name: str) -> None: super().__init__(name)
[docs] @override def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any: """Calculate the accuracy score. Returns: the numeric score or `'N/A'` if no unmasked data was provided. """ if not (true or pred): return 'N/A' return accuracy_score(true, pred)
[docs] class UAS(AccuracyMetric): """Unlabeled Attachment Score.""" def __init__(self) -> None: super().__init__('UAS')
[docs] @override def get_labels(self, tags: LabelDict) -> list[int] | None: """Gather dependency head indices. Returns: indices or `None` if masked. """ if MASK_VALUE in tags[HEADS]: return None return [h if h is not None else -1 for h in tags[HEADS]]
[docs] class LAS(AccuracyMetric): """Labeled Attachment Score.""" def __init__(self) -> None: super().__init__('LAS')
[docs] @override def get_labels(self, tags: LabelDict) -> list[str] | None: """Gather dependency head indices and dependency relation labels. Strips UD-style language-specific dependency relation sublabels (anything after a semicolon). Returns: indices with deprels or `None` if masked. """ # for Składnica merged data with masked deprels... if MASK_VALUE in tags[HEADS] or None in tags[DEPRELS]: return None return [f'{h}#{d.split(":")[0]}' for h, d in zip(tags[HEADS], tags[DEPRELS])] # type: ignore[union-attr] # all deprels are not None
[docs] class Tagging(AccuracyMetric): """Tagging accuracy.""" def __init__(self) -> None: super().__init__('tagging')
[docs] @override def get_labels(self, tags: LabelDict) -> list[str] | None: """Gather inflectional tags. Returns: tags or `None` if masked. """ if None in tags[TAGS]: return None return tags[TAGS] # type: ignore[return-value] # all tags are not None
[docs] class TaggingCorr(AccuracyMetric): """Tagging accuracy when Morfeusz correction is applied.""" def __init__(self, morfeusz: Morfeusz) -> None: super().__init__('tagging*') self.morfeusz = morfeusz # TODO test: this looks unnecessary since we override update()? or use correct_lemma in get_labels? #def get_labels(self, tags: LabelDict) -> list[str] | None: # if MASK_VALUE in tags[TAGS]: # return None # return tags[TAGS] # type: ignore[return-value] # all tags are not None
[docs] @override def update( self, tokens: TokenList, tags_true: LabelDict, tags_pred: LabelDict, tree_true: TreeNode | None, tree_pred: TreeNode | None, ) -> None: """Override :meth:`~hydra.utils.evaluation.TokenMetric.update` to inject Morfeusz correction.""" self.total += 1 if None in tags_true[TAGS]: self.masked += 1 else: self.true += tags_true[TAGS] self.pred += [ correct_lemma(*x, self.morfeusz)[1] for x in zip( # type: ignore [arg-type] tokens, tags_pred[LEMMAS], tags_pred[TAGS]) ]
[docs] class Lemmatisation(AccuracyMetric): """Lemmatisation accuracy.""" def __init__(self) -> None: super().__init__('lemmatisation')
[docs] @override def get_labels(self, tags: LabelDict) -> list[str] | None: """Gather lemmata. Returns: lemmata or `None` if masked. """ if None in tags[LEMMAS]: return None return tags[LEMMAS] # type: ignore[return-value] # all lemmas are not None
[docs] class LemmatisationCorr(AccuracyMetric): """Lemmatisation accuracy when Morfeusz correction is applied.""" def __init__(self, morfeusz: Morfeusz) -> None: super().__init__('lemmatisation*') self.morfeusz = morfeusz
[docs] @override def update( self, tokens: TokenList, tags_true: LabelDict, tags_pred: LabelDict, tree_true: TreeNode | None, tree_pred: TreeNode | None, ) -> None: """Override :meth:`~hydra.utils.evaluation.TokenMetric.update` to inject Morfeusz correction.""" self.total += 1 if None in tags_true[LEMMAS]: self.masked += 1 else: self.true += tags_true[LEMMAS] self.pred += [ correct_lemma(*x, self.morfeusz)[0] for x in zip( # type: ignore [arg-type] tokens, tags_pred[LEMMAS], tags_pred[TAGS]) ]
[docs] class PrecisionRecallFscore(EvaluationMetric): """Precision, recall and F1-score reported for entire data and specific types of predicted units.""" def __init__(self, name: str) -> None: super().__init__(name) self.true_positive = 0 self.precision_denominator = 0 self.recall_denominator = 0 self.partial_true_positives: dict[Any, int] = Counter() self.partial_precision_denominators: dict[Any, int] = Counter() self.partial_recall_denominators: dict[Any, int] = Counter() self.unit_keys: set[Any] = set()
[docs] def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[Any] | None: """Gather predicted units (spans, constituency edges...) to be compared between gold and predicted sequences. Subclasses must implement this method. :raises NotImplementedError: Always. """ raise NotImplementedError
[docs] def get_unit_key(self, unit: Any) -> Any: """Get unit type for calculating partial scores (e.g. a constituent type for tree spans). Subclasses must implement this method. :raises NotImplementedError: Always. """ raise NotImplementedError
[docs] @override def update( self, tokens: TokenList, tags_true: LabelDict, tags_pred: LabelDict, tree_true: TreeNode | None, tree_pred: TreeNode | None, ) -> None: """Gather units needed for calculating this metric from a true/predicted sentence pair. Metric-specific unit gathering logic must be implemented by overriding :meth:`~hydra.utils.evaluation.PrecisionRecallFscore.get_units` and :meth:`~hydra.utils.evaluation.PrecisionRecallFscore.get_unit_key`. """ self.total += 1 units_true = self.get_units(tree_true, tags_true) if units_true is None: self.masked += 1 else: units_pred = self.get_units(tree_pred, tags_pred) assert (units_pred is not None) common = units_true.intersection(units_pred) self.true_positive += len(units_true.intersection(units_pred)) self.precision_denominator += len(units_pred) self.recall_denominator += len(units_true) for unit in units_true.union(units_pred): key = self.get_unit_key(unit) self.unit_keys.add(key) if unit in units_true: self.partial_recall_denominators[key] += 1 if unit in units_pred: self.partial_precision_denominators[key] += 1 if unit in common: self.partial_true_positives[key] += 1
[docs] @override def do_calculate(self) -> Any: """Perform calculations of the overall metric results after being fed all the data. Returns: calculated results. """ f: float | str = 0 p = self.true_positive / self.precision_denominator if self.precision_denominator else 'N/A' r = self.true_positive / self.recall_denominator if self.recall_denominator else 'N/A' if self.precision_denominator and self.recall_denominator: assert (isinstance(p, float)) assert (isinstance(r, float)) if p or r: f = (2 * p * r) / (p + r) else: f = 'N/A' ret = { 'total' : { 'P' : p, 'R' : r, 'F1': f, } } for key in self.unit_keys: pd = self.partial_precision_denominators[key] pr = self.partial_recall_denominators[key] p = (self.partial_true_positives[key] / pd) if pd else 'N/A' r = (self.partial_true_positives[key] / pr) if pr else 'N/A' if pd and pr: assert (isinstance(p, float)) assert (isinstance(r, float)) f = ((2 * p * r) / (p + r)) if p or r else 0.0 else: f = 'N/A' ret[key] = { 'P' : p, 'R' : r, 'F1' : f, 'n' : self.partial_recall_denominators[key], } return ret
[docs] class Spans(PrecisionRecallFscore): """Base class for precision/recall/F1 metrics operating on tree constituents. Implements the logic shared between specific metrics. """ def __init__( self, name: str, labeled: bool, headed: bool, discontinuous_only: bool = False, skip_categories: list[str] | None = None, ) -> None: super().__init__(name) self.labeled = labeled self.headed = headed self.discontinuous_only = discontinuous_only self.skip_categories = skip_categories if skip_categories is not None else []
[docs] @override def get_unit_key(self, unit: tuple[tuple[int, ...], str, bool]) -> str: # unit: <span, cat, is_head> return unit[1]# if len(unit[0]) > 1 else '<PRETERMINAL>'
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[tuple[int, ...], str, bool]] | None: if not tree: return None return _tree2spans( tree, labeled=self.labeled, headed=self.headed, discontinuous_only=self.discontinuous_only, skip_categories=self.skip_categories )
[docs] class HeadedSpans(Spans): """Precision/recall/F1 metrics operating on tree constituents. Takes token spans, syntactic categories and head marking into account. """ def __init__(self, **kwargs: Any) -> None: super().__init__('headed constit.', labeled=True, headed=True, **kwargs)
[docs] class NonHeadedSpans(Spans): """Precision/recall/F1 metrics operating on tree constituents. Takes token spans and syntactic categories into account. """ def __init__(self, **kwargs: Any) -> None: super().__init__('constituents', labeled=True, headed=False, **kwargs)
[docs] class UnlabeledSpans(Spans): """Precision/recall/F1 metrics operating on tree constituents. Only takes token spans into account. """ def __init__(self, **kwargs: Any) -> None: super().__init__('bracketing', labeled=False, headed=False, **kwargs)
[docs] class HeadedSpansDiscontinuous(Spans): """Like :class:`~hydra.utils.evaluation.HeadedSpans`, but considers only discontinuous constituents.""" def __init__(self, **kwargs: Any) -> None: super().__init__('disc. headed constit.', labeled=True, headed=True, discontinuous_only=True, **kwargs)
[docs] class NonHeadedSpansDiscontinuous(Spans): """Like :class:`~hydra.utils.evaluation.NonHeadedSpans`, but considers only discontinuous constituents.""" def __init__(self, **kwargs: Any) -> None: super().__init__('disc. constituents', labeled=True, headed=False, discontinuous_only=True, **kwargs)
[docs] class UnlabeledSpansDiscontinuous(Spans): """Like :class:`~hydra.utils.evaluation.UnlabeledSpans`, but considers only discontinuous constituents.""" def __init__(self, **kwargs: Any) -> None: super().__init__('disc. bracketing', labeled=False, headed=False, discontinuous_only=True, **kwargs)
[docs] class Edges(PrecisionRecallFscore): """Precision/recall/F1 metrics operating on hybrid tree edges.""" def __init__(self) -> None: super().__init__('edges')
[docs] @override def get_unit_key(self, unit: tuple[str, str, int, int, str]) -> tuple[str, str, str]: # unit = from_cat, to_cat, from_index, to_index, deprel return (unit[0], unit[1], unit[4])
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[str, str, int, int, str]] | None: if not tree or None in tags[DEPRELS]: return None return _tree2edges(tree)
[docs] class Deprels(PrecisionRecallFscore): """Precision/recall/F1 metrics for dependency relation labels.""" def __init__(self) -> None: super().__init__('deprels')
[docs] @override def get_unit_key(self, unit: tuple[int, str]) -> str: # unit = idx, deprel return unit[1]
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, str]] | None: if None in tags[DEPRELS]: return None return set(enumerate(tags[DEPRELS])) # type: ignore [arg-type]
[docs] class DepArcs(PrecisionRecallFscore): """Precision/recall/F1 metrics for dependency arcs.""" def __init__(self) -> None: super().__init__('deparcs')
[docs] @override def get_unit_key(self, unit: tuple[int, tuple[str, int]]) -> str: # unit = idx, (deprel, parent) return unit[1][0]
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None: if None in tags[DEPRELS]: return None return set(enumerate(zip(tags[DEPRELS], tags[HEADS]))) # type: ignore [arg-type]
[docs] class EUAS(PrecisionRecallFscore): """Enhanced Unlabeled Attachment Score (standard and enhanced edges).""" def __init__(self) -> None: super().__init__('EUAS')
[docs] @override def get_unit_key(self, unit: tuple[int, int]) -> str: # unit = idx, parent # no grouping here return ''
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, int | None]] | None: if MASK_VALUE in tags[HEADS] or [MASK_VALUE] in tags[GRAPH_HEADS]: return None ret = set(enumerate(tags[HEADS])) # In EUD, standard i->j edge implies no enhanced i->j edge, # but in predicate-argument graphs it’s possible (at most one enhanced i->j). # For now, treat them as one *unlabeled* edge (and differentiate in ELAS). # Alternative: index the enhanced ones as, e.g. (i, f'{j}e') for idx, heads in enumerate(tags[GRAPH_HEADS]): for h in heads: ret.add((idx, h)) return ret
[docs] class ELAS(PrecisionRecallFscore): """Enhanced Labeled Attachment Score (standard and enhanced edges).""" def __init__(self) -> None: super().__init__('ELAS')
[docs] @override def get_unit_key(self, unit: tuple[int, tuple[str, int]]) -> str: # unit = idx, (deprel, parent) return unit[1][0]
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None: if MASK_VALUE in tags[HEADS] or [MASK_VALUE] in tags[GRAPH_HEADS]: return None ret = set(enumerate(zip(tags[DEPRELS], tags[HEADS]))) for idx, (deprels, heads) in enumerate(zip(tags[GRAPH_DEPRELS], tags[GRAPH_HEADS])): for d, h in zip(deprels, heads): ret.add((idx, (d, h))) return ret # type: ignore [return-value] # all deprels are not None
[docs] class GUAS(PrecisionRecallFscore): """Graph Unlabeled Attachment Score (enhanced edges only).""" def __init__(self) -> None: super().__init__('GUAS')
[docs] @override def get_unit_key(self, unit: tuple[int, int]) -> str: # unit = idx, parent # no grouping here return ''
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, int | None]] | None: if [MASK_VALUE] in tags[GRAPH_HEADS]: return None ret = set() for idx, heads in enumerate(tags[GRAPH_HEADS]): for h in heads: ret.add((idx, h)) return ret
[docs] class GLAS(PrecisionRecallFscore): """Graph Labeled Attachment Score (enhanced edges only).""" def __init__(self) -> None: super().__init__('GLAS')
[docs] @override def get_unit_key(self, unit: tuple[int, tuple[str, int]]) -> str: # unit = idx, (deprel, parent) return unit[1][0]
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None: if [MASK_VALUE] in tags[GRAPH_HEADS]: return None ret = set() # In EUD, standard i->j edge implies no enhanced i->j edge, # but in predicate-argument graphs it’s possible (at most one enhanced i->j). # For now, treat them as one *unlabeled* edge (and differentiate in ELAS). # Alternative: index the enhanced ones as, e.g. (i, f'{j}e') for idx, (deprels, heads) in enumerate(zip(tags[GRAPH_DEPRELS], tags[GRAPH_HEADS])): for d, h in zip(deprels, heads): assert (d is not None) ret.add((idx, (d, h))) return ret
[docs] class NER(PrecisionRecallFscore): """Precision/recall/F1 metrics for named entities.""" def __init__(self) -> None: super().__init__('NER')
[docs] @override def get_unit_key(self, unit: tuple[tuple[int, ...], str]) -> str: # unit = span, ne_type return unit[1]
[docs] @override def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None: if None in tags[NE_IOBS]: return None ne_tags = tags[NE_IOBS] # tokens in returned units will have indices, so anything works as text form here nes = get_nes(['_' for _ in ne_tags], ne_tags) return set((tuple(idx for idx, _ in tokens), ne_type) for tokens, ne_type in nes)
[docs] class ConfusionMatrix(TokenMetric): """Confusion matrix calculation.""" def __init__(self, name: str) -> None: super().__init__(name)
[docs] @override def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any: labels = sorted(set(true + pred)) errors = Counter((t, p) for t, p in zip(true, pred) if t != p) return confusion_matrix(true, pred, labels=labels), labels, errors
[docs] class DeprelsConfusionMatrix(ConfusionMatrix): """Confusion matrix for dependency relation labels.""" def __init__(self) -> None: super().__init__('deprels_cf')
[docs] @override def get_labels(self, tags: LabelDict) -> list[str] | None: if None in tags[DEPRELS]: return None return tags[DEPRELS] # type: ignore[return-value] # all deprels are not None
[docs] class DepArcsConfusionMatrix(ConfusionMatrix): """Confusion matrix for dependency arcs (heads and relation labels).""" def __init__(self) -> None: super().__init__('deparcs_cf')
[docs] @override def get_labels(self, tags: LabelDict) -> list[tuple[str, int]] | None: if None in tags[DEPRELS]: return None return list(zip(tags[DEPRELS], tags[HEADS])) # type: ignore [arg-type]
[docs] @override def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any: new_true, new_pred = [], [] i = 0 for t, p in zip(true, pred): new_true.append(t[0]) new_pred.append(p[0] if p[1] == t[1] else '—') true, pred = new_true, new_pred labels = sorted(set(true + pred)) errors = Counter((t, p) for t, p in zip(true, pred) if t != p) return confusion_matrix(true, pred, labels=labels), labels, errors