"""Model evaluation utilities."""
import urllib.parse
import json
from collections import Counter
from typing import Any, override
from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, confusion_matrix
from morfeusz2 import Morfeusz
from .hybrid_tree import tree2dict, correct_lemma, TreeNode
from .ner import get_nes
from .constants import MASK_VALUE, HEADS, DEPRELS, GRAPH_HEADS, GRAPH_DEPRELS, LEMMAS, TAGS, NE_IOBS
from .types import TokenList, LabelDict
def _collect_spans(
tree: TreeNode,
spans: list[tuple[tuple[int, ...], str, bool]],
labeled: bool,
headed: bool,
discontinuous_only: bool,
skip_categories: list[str] | None = None
) -> set[int]:
if headed and not labeled:
raise RuntimeError('cant’t use headed=True with labeled=False')
if not tree.children:
assert (tree.from_index is not None)
return {tree.from_index}
if skip_categories is None:
skip_categories = []
span = set()
for child in tree.children:
span.update(_collect_spans(child, spans, labeled, headed, discontinuous_only, skip_categories))
if tree.category not in skip_categories:
sorted_span = tuple(sorted(span))
if discontinuous_only and sorted_span == tuple(range(sorted_span[0], sorted_span[-1] + 1)):
# continuous constituent
pass
else:
assert (tree.category is not None)
spans += [(sorted_span, tree.category if labeled else 'SPAN', tree.is_head if headed else False)]
return span
def _tree2spans(
tree: TreeNode,
labeled: bool = True,
headed: bool = False,
discontinuous_only: bool = False,
skip_categories: list[str] | None = None
) -> set[tuple[tuple[int, ...], str, bool]]:
spans: list[tuple[tuple[int, ...], str, bool]] = []
_collect_spans(tree, spans, labeled, headed, discontinuous_only, skip_categories)
# TODO
#try:
# assert(len(spans) == len(set(spans)))
#except:
# show_tree(tree)
# (display(spans))
# 1/0
return set(spans)
def _collect_edges(tree: TreeNode, edges: list[tuple[str, str, int, int, str]]) -> int:
if not tree.children:
assert (tree.from_index is not None)
return tree.from_index
head_index = None
child_edges = []
for child in tree.children:
child_head_index = _collect_edges(child, edges)
child_cat = child.category if child.children else '<TERMINAL>'
assert (child_cat is not None)
deprel = child.attributes['deprel'] if not child.is_head else '<HEAD>'
child_edges.append((child_head_index, child_cat, deprel))
if child.is_head:
assert(head_index is None)
head_index = child_head_index
assert(head_index is not None)
assert(tree.category is not None)
for child_head_index, child_cat, deprel in child_edges:
assert (child_cat is not None)
edges.append((tree.category, child_cat, head_index, child_head_index, deprel))
return head_index
def _tree2edges(tree: TreeNode) -> set[tuple[str, str, int, int, str]]:
edges: list[tuple[str, str, int, int, str]] = []
_collect_edges(tree, edges)
#TODO?
#if len(edges) != len(set(edges)):
# #show_tree(tree)
# #1/0
return set(edges)
[docs]
class EvaluationMetric(object):
"""Base class for any evaluation metric."""
def __init__(self, name: str) -> None:
self.name = name
self.total = 0
self.masked = 0
[docs]
def update(
self,
tokens: TokenList,
tags_true: LabelDict,
tags_pred: LabelDict,
tree_true: TreeNode | None,
tree_pred: TreeNode | None,
) -> None:
"""Gather any statistics needed for calculating this metric from a true/predicted sentence pair.
Subclasses must implement this method.
:raises NotImplementedError: Always.
"""
raise NotImplementedError
[docs]
def calculate(self) -> dict[str, Any]:
"""Calculate the overall metric result after being fed all the data.
Returns:
any scores and statistics assoaciated with the metric, number of instances: total seen and excluded because of masking.
"""
return {
'result' : self.do_calculate(),
'total instances' : self.total,
'masked instances' : self.masked,
}
[docs]
def do_calculate(self) -> Any:
"""Perform metric-specific calculations of the overall metric result after being fed all the data.
Subclasses must implement this method.
:raises NotImplementedError: Always.
"""
raise NotImplementedError
[docs]
class TokenMetric(EvaluationMetric):
"""Base class for any evaluation metric operating on token level."""
def __init__(self, name: str) -> None:
super().__init__(name)
self.true: list[Any] = []
self.pred: list[Any] = []
[docs]
def get_labels(self, tags: LabelDict) -> list[Any] | None:
"""Gather token labels to be compared between gold and predicted sequences.
Subclasses must implement this method.
:raises NotImplementedError: Always.
"""
raise NotImplementedError
[docs]
@override
def update(
self,
tokens: TokenList,
tags_true: LabelDict,
tags_pred: LabelDict,
tree_true: TreeNode | None,
tree_pred: TreeNode | None,
) -> None:
"""Gather token-level labels needed for calculating this metric from a true/predicted sentence pair.
Metric-specific label gathering logic must be implemented by overriding :meth:`~hydra.utils.evaluation.TokenMetric.get_labels`.
"""
self.total += 1
labels_true = self.get_labels(tags_true)
if labels_true is not None:
self.true += labels_true
labels_pred = self.get_labels(tags_pred)
assert (labels_pred is not None)
self.pred += labels_pred
else:
self.masked += 1
[docs]
def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any:
"""Calculate a specific metric operating on token level.
Subclasses must implement this method.
:raises NotImplementedError: Always.
"""
raise NotImplementedError
[docs]
@override
def do_calculate(self) -> Any:
"""Perform metric-specific calculations of the overall metric result after being fed all the data.
Metric-specific calculation logic must be implemented by overriding :meth:`~hydra.utils.evaluation.TokenMetric.calculate_token_metric`.
Returns:
calculated result or `'N/A'` if not possible.
"""
# for Składnica merged data with masked deprels...
filtered = [(t, p) for t, p in zip(self.true, self.pred) if t is not None]
if filtered:
true, pred = map(list, zip(*filtered))
return self.calculate_token_metric(true, pred)
else:
return 'N/A'
[docs]
class AccuracyMetric(TokenMetric):
"""Base class for any evaluation metric calculating token-level accuracy."""
def __init__(self, name: str) -> None:
super().__init__(name)
[docs]
@override
def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any:
"""Calculate the accuracy score.
Returns:
the numeric score or `'N/A'` if no unmasked data was provided.
"""
if not (true or pred):
return 'N/A'
return accuracy_score(true, pred)
[docs]
class UAS(AccuracyMetric):
"""Unlabeled Attachment Score."""
def __init__(self) -> None:
super().__init__('UAS')
[docs]
@override
def get_labels(self, tags: LabelDict) -> list[int] | None:
"""Gather dependency head indices.
Returns:
indices or `None` if masked.
"""
if MASK_VALUE in tags[HEADS]:
return None
return [h if h is not None else -1 for h in tags[HEADS]]
[docs]
class LAS(AccuracyMetric):
"""Labeled Attachment Score."""
def __init__(self) -> None:
super().__init__('LAS')
[docs]
@override
def get_labels(self, tags: LabelDict) -> list[str] | None:
"""Gather dependency head indices and dependency relation labels.
Strips UD-style language-specific dependency relation sublabels (anything after a semicolon).
Returns:
indices with deprels or `None` if masked.
"""
# for Składnica merged data with masked deprels...
if MASK_VALUE in tags[HEADS] or None in tags[DEPRELS]:
return None
return [f'{h}#{d.split(":")[0]}' for h, d in zip(tags[HEADS], tags[DEPRELS])] # type: ignore[union-attr] # all deprels are not None
[docs]
class Tagging(AccuracyMetric):
"""Tagging accuracy."""
def __init__(self) -> None:
super().__init__('tagging')
[docs]
@override
def get_labels(self, tags: LabelDict) -> list[str] | None:
"""Gather inflectional tags.
Returns:
tags or `None` if masked.
"""
if None in tags[TAGS]:
return None
return tags[TAGS] # type: ignore[return-value] # all tags are not None
[docs]
class TaggingCorr(AccuracyMetric):
"""Tagging accuracy when Morfeusz correction is applied."""
def __init__(self, morfeusz: Morfeusz) -> None:
super().__init__('tagging*')
self.morfeusz = morfeusz
# TODO test: this looks unnecessary since we override update()? or use correct_lemma in get_labels?
#def get_labels(self, tags: LabelDict) -> list[str] | None:
# if MASK_VALUE in tags[TAGS]:
# return None
# return tags[TAGS] # type: ignore[return-value] # all tags are not None
[docs]
@override
def update(
self,
tokens: TokenList,
tags_true: LabelDict,
tags_pred: LabelDict,
tree_true: TreeNode | None,
tree_pred: TreeNode | None,
) -> None:
"""Override :meth:`~hydra.utils.evaluation.TokenMetric.update` to inject Morfeusz correction."""
self.total += 1
if None in tags_true[TAGS]:
self.masked += 1
else:
self.true += tags_true[TAGS]
self.pred += [
correct_lemma(*x, self.morfeusz)[1] for x in zip( # type: ignore [arg-type]
tokens, tags_pred[LEMMAS], tags_pred[TAGS])
]
[docs]
class Lemmatisation(AccuracyMetric):
"""Lemmatisation accuracy."""
def __init__(self) -> None:
super().__init__('lemmatisation')
[docs]
@override
def get_labels(self, tags: LabelDict) -> list[str] | None:
"""Gather lemmata.
Returns:
lemmata or `None` if masked.
"""
if None in tags[LEMMAS]:
return None
return tags[LEMMAS] # type: ignore[return-value] # all lemmas are not None
[docs]
class LemmatisationCorr(AccuracyMetric):
"""Lemmatisation accuracy when Morfeusz correction is applied."""
def __init__(self, morfeusz: Morfeusz) -> None:
super().__init__('lemmatisation*')
self.morfeusz = morfeusz
[docs]
@override
def update(
self,
tokens: TokenList,
tags_true: LabelDict,
tags_pred: LabelDict,
tree_true: TreeNode | None,
tree_pred: TreeNode | None,
) -> None:
"""Override :meth:`~hydra.utils.evaluation.TokenMetric.update` to inject Morfeusz correction."""
self.total += 1
if None in tags_true[LEMMAS]:
self.masked += 1
else:
self.true += tags_true[LEMMAS]
self.pred += [
correct_lemma(*x, self.morfeusz)[0] for x in zip( # type: ignore [arg-type]
tokens, tags_pred[LEMMAS], tags_pred[TAGS])
]
[docs]
class PrecisionRecallFscore(EvaluationMetric):
"""Precision, recall and F1-score reported for entire data and specific types of predicted units."""
def __init__(self, name: str) -> None:
super().__init__(name)
self.true_positive = 0
self.precision_denominator = 0
self.recall_denominator = 0
self.partial_true_positives: dict[Any, int] = Counter()
self.partial_precision_denominators: dict[Any, int] = Counter()
self.partial_recall_denominators: dict[Any, int] = Counter()
self.unit_keys: set[Any] = set()
[docs]
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[Any] | None:
"""Gather predicted units (spans, constituency edges...) to be compared between gold and predicted sequences.
Subclasses must implement this method.
:raises NotImplementedError: Always.
"""
raise NotImplementedError
[docs]
def get_unit_key(self, unit: Any) -> Any:
"""Get unit type for calculating partial scores (e.g. a constituent type for tree spans).
Subclasses must implement this method.
:raises NotImplementedError: Always.
"""
raise NotImplementedError
[docs]
@override
def update(
self,
tokens: TokenList,
tags_true: LabelDict,
tags_pred: LabelDict,
tree_true: TreeNode | None,
tree_pred: TreeNode | None,
) -> None:
"""Gather units needed for calculating this metric from a true/predicted sentence pair.
Metric-specific unit gathering logic must be implemented by overriding
:meth:`~hydra.utils.evaluation.PrecisionRecallFscore.get_units`
and :meth:`~hydra.utils.evaluation.PrecisionRecallFscore.get_unit_key`.
"""
self.total += 1
units_true = self.get_units(tree_true, tags_true)
if units_true is None:
self.masked += 1
else:
units_pred = self.get_units(tree_pred, tags_pred)
assert (units_pred is not None)
common = units_true.intersection(units_pred)
self.true_positive += len(units_true.intersection(units_pred))
self.precision_denominator += len(units_pred)
self.recall_denominator += len(units_true)
for unit in units_true.union(units_pred):
key = self.get_unit_key(unit)
self.unit_keys.add(key)
if unit in units_true:
self.partial_recall_denominators[key] += 1
if unit in units_pred:
self.partial_precision_denominators[key] += 1
if unit in common:
self.partial_true_positives[key] += 1
[docs]
@override
def do_calculate(self) -> Any:
"""Perform calculations of the overall metric results after being fed all the data.
Returns:
calculated results.
"""
f: float | str = 0
p = self.true_positive / self.precision_denominator if self.precision_denominator else 'N/A'
r = self.true_positive / self.recall_denominator if self.recall_denominator else 'N/A'
if self.precision_denominator and self.recall_denominator:
assert (isinstance(p, float))
assert (isinstance(r, float))
if p or r:
f = (2 * p * r) / (p + r)
else:
f = 'N/A'
ret = {
'total' : {
'P' : p,
'R' : r,
'F1': f,
}
}
for key in self.unit_keys:
pd = self.partial_precision_denominators[key]
pr = self.partial_recall_denominators[key]
p = (self.partial_true_positives[key] / pd) if pd else 'N/A'
r = (self.partial_true_positives[key] / pr) if pr else 'N/A'
if pd and pr:
assert (isinstance(p, float))
assert (isinstance(r, float))
f = ((2 * p * r) / (p + r)) if p or r else 0.0
else:
f = 'N/A'
ret[key] = {
'P' : p,
'R' : r,
'F1' : f,
'n' : self.partial_recall_denominators[key],
}
return ret
[docs]
class Spans(PrecisionRecallFscore):
"""Base class for precision/recall/F1 metrics operating on tree constituents.
Implements the logic shared between specific metrics.
"""
def __init__(
self,
name: str,
labeled: bool,
headed: bool,
discontinuous_only: bool = False,
skip_categories: list[str] | None = None,
) -> None:
super().__init__(name)
self.labeled = labeled
self.headed = headed
self.discontinuous_only = discontinuous_only
self.skip_categories = skip_categories if skip_categories is not None else []
[docs]
@override
def get_unit_key(self, unit: tuple[tuple[int, ...], str, bool]) -> str:
# unit: <span, cat, is_head>
return unit[1]# if len(unit[0]) > 1 else '<PRETERMINAL>'
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[tuple[int, ...], str, bool]] | None:
if not tree:
return None
return _tree2spans(
tree,
labeled=self.labeled,
headed=self.headed,
discontinuous_only=self.discontinuous_only,
skip_categories=self.skip_categories
)
[docs]
class HeadedSpans(Spans):
"""Precision/recall/F1 metrics operating on tree constituents.
Takes token spans, syntactic categories and head marking into account.
"""
def __init__(self, **kwargs: Any) -> None:
super().__init__('headed constit.', labeled=True, headed=True, **kwargs)
[docs]
class NonHeadedSpans(Spans):
"""Precision/recall/F1 metrics operating on tree constituents.
Takes token spans and syntactic categories into account.
"""
def __init__(self, **kwargs: Any) -> None:
super().__init__('constituents', labeled=True, headed=False, **kwargs)
[docs]
class UnlabeledSpans(Spans):
"""Precision/recall/F1 metrics operating on tree constituents.
Only takes token spans into account.
"""
def __init__(self, **kwargs: Any) -> None:
super().__init__('bracketing', labeled=False, headed=False, **kwargs)
[docs]
class HeadedSpansDiscontinuous(Spans):
"""Like :class:`~hydra.utils.evaluation.HeadedSpans`, but considers only discontinuous constituents."""
def __init__(self, **kwargs: Any) -> None:
super().__init__('disc. headed constit.', labeled=True, headed=True, discontinuous_only=True, **kwargs)
[docs]
class NonHeadedSpansDiscontinuous(Spans):
"""Like :class:`~hydra.utils.evaluation.NonHeadedSpans`, but considers only discontinuous constituents."""
def __init__(self, **kwargs: Any) -> None:
super().__init__('disc. constituents', labeled=True, headed=False, discontinuous_only=True, **kwargs)
[docs]
class UnlabeledSpansDiscontinuous(Spans):
"""Like :class:`~hydra.utils.evaluation.UnlabeledSpans`, but considers only discontinuous constituents."""
def __init__(self, **kwargs: Any) -> None:
super().__init__('disc. bracketing', labeled=False, headed=False, discontinuous_only=True, **kwargs)
[docs]
class Edges(PrecisionRecallFscore):
"""Precision/recall/F1 metrics operating on hybrid tree edges."""
def __init__(self) -> None:
super().__init__('edges')
[docs]
@override
def get_unit_key(self, unit: tuple[str, str, int, int, str]) -> tuple[str, str, str]:
# unit = from_cat, to_cat, from_index, to_index, deprel
return (unit[0], unit[1], unit[4])
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[str, str, int, int, str]] | None:
if not tree or None in tags[DEPRELS]:
return None
return _tree2edges(tree)
[docs]
class Deprels(PrecisionRecallFscore):
"""Precision/recall/F1 metrics for dependency relation labels."""
def __init__(self) -> None:
super().__init__('deprels')
[docs]
@override
def get_unit_key(self, unit: tuple[int, str]) -> str:
# unit = idx, deprel
return unit[1]
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, str]] | None:
if None in tags[DEPRELS]:
return None
return set(enumerate(tags[DEPRELS])) # type: ignore [arg-type]
[docs]
class DepArcs(PrecisionRecallFscore):
"""Precision/recall/F1 metrics for dependency arcs."""
def __init__(self) -> None:
super().__init__('deparcs')
[docs]
@override
def get_unit_key(self, unit: tuple[int, tuple[str, int]]) -> str:
# unit = idx, (deprel, parent)
return unit[1][0]
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None:
if None in tags[DEPRELS]:
return None
return set(enumerate(zip(tags[DEPRELS], tags[HEADS]))) # type: ignore [arg-type]
[docs]
class EUAS(PrecisionRecallFscore):
"""Enhanced Unlabeled Attachment Score (standard and enhanced edges)."""
def __init__(self) -> None:
super().__init__('EUAS')
[docs]
@override
def get_unit_key(self, unit: tuple[int, int]) -> str:
# unit = idx, parent
# no grouping here
return ''
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, int | None]] | None:
if MASK_VALUE in tags[HEADS] or [MASK_VALUE] in tags[GRAPH_HEADS]:
return None
ret = set(enumerate(tags[HEADS]))
# In EUD, standard i->j edge implies no enhanced i->j edge,
# but in predicate-argument graphs it’s possible (at most one enhanced i->j).
# For now, treat them as one *unlabeled* edge (and differentiate in ELAS).
# Alternative: index the enhanced ones as, e.g. (i, f'{j}e')
for idx, heads in enumerate(tags[GRAPH_HEADS]):
for h in heads:
ret.add((idx, h))
return ret
[docs]
class ELAS(PrecisionRecallFscore):
"""Enhanced Labeled Attachment Score (standard and enhanced edges)."""
def __init__(self) -> None:
super().__init__('ELAS')
[docs]
@override
def get_unit_key(self, unit: tuple[int, tuple[str, int]]) -> str:
# unit = idx, (deprel, parent)
return unit[1][0]
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None:
if MASK_VALUE in tags[HEADS] or [MASK_VALUE] in tags[GRAPH_HEADS]:
return None
ret = set(enumerate(zip(tags[DEPRELS], tags[HEADS])))
for idx, (deprels, heads) in enumerate(zip(tags[GRAPH_DEPRELS], tags[GRAPH_HEADS])):
for d, h in zip(deprels, heads):
ret.add((idx, (d, h)))
return ret # type: ignore [return-value] # all deprels are not None
[docs]
class GUAS(PrecisionRecallFscore):
"""Graph Unlabeled Attachment Score (enhanced edges only)."""
def __init__(self) -> None:
super().__init__('GUAS')
[docs]
@override
def get_unit_key(self, unit: tuple[int, int]) -> str:
# unit = idx, parent
# no grouping here
return ''
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, int | None]] | None:
if [MASK_VALUE] in tags[GRAPH_HEADS]:
return None
ret = set()
for idx, heads in enumerate(tags[GRAPH_HEADS]):
for h in heads:
ret.add((idx, h))
return ret
[docs]
class GLAS(PrecisionRecallFscore):
"""Graph Labeled Attachment Score (enhanced edges only)."""
def __init__(self) -> None:
super().__init__('GLAS')
[docs]
@override
def get_unit_key(self, unit: tuple[int, tuple[str, int]]) -> str:
# unit = idx, (deprel, parent)
return unit[1][0]
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None:
if [MASK_VALUE] in tags[GRAPH_HEADS]:
return None
ret = set()
# In EUD, standard i->j edge implies no enhanced i->j edge,
# but in predicate-argument graphs it’s possible (at most one enhanced i->j).
# For now, treat them as one *unlabeled* edge (and differentiate in ELAS).
# Alternative: index the enhanced ones as, e.g. (i, f'{j}e')
for idx, (deprels, heads) in enumerate(zip(tags[GRAPH_DEPRELS], tags[GRAPH_HEADS])):
for d, h in zip(deprels, heads):
assert (d is not None)
ret.add((idx, (d, h)))
return ret
[docs]
class NER(PrecisionRecallFscore):
"""Precision/recall/F1 metrics for named entities."""
def __init__(self) -> None:
super().__init__('NER')
[docs]
@override
def get_unit_key(self, unit: tuple[tuple[int, ...], str]) -> str:
# unit = span, ne_type
return unit[1]
[docs]
@override
def get_units(self, tree: TreeNode | None, tags: LabelDict) -> set[tuple[int, tuple[str, int | None]]] | None:
if None in tags[NE_IOBS]:
return None
ne_tags = tags[NE_IOBS]
# tokens in returned units will have indices, so anything works as text form here
nes = get_nes(['_' for _ in ne_tags], ne_tags)
return set((tuple(idx for idx, _ in tokens), ne_type) for tokens, ne_type in nes)
[docs]
class ConfusionMatrix(TokenMetric):
"""Confusion matrix calculation."""
def __init__(self, name: str) -> None:
super().__init__(name)
[docs]
@override
def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any:
labels = sorted(set(true + pred))
errors = Counter((t, p) for t, p in zip(true, pred) if t != p)
return confusion_matrix(true, pred, labels=labels), labels, errors
[docs]
class DeprelsConfusionMatrix(ConfusionMatrix):
"""Confusion matrix for dependency relation labels."""
def __init__(self) -> None:
super().__init__('deprels_cf')
[docs]
@override
def get_labels(self, tags: LabelDict) -> list[str] | None:
if None in tags[DEPRELS]:
return None
return tags[DEPRELS] # type: ignore[return-value] # all deprels are not None
[docs]
class DepArcsConfusionMatrix(ConfusionMatrix):
"""Confusion matrix for dependency arcs (heads and relation labels)."""
def __init__(self) -> None:
super().__init__('deparcs_cf')
[docs]
@override
def get_labels(self, tags: LabelDict) -> list[tuple[str, int]] | None:
if None in tags[DEPRELS]:
return None
return list(zip(tags[DEPRELS], tags[HEADS])) # type: ignore [arg-type]
[docs]
@override
def calculate_token_metric(self, true: list[Any], pred: list[Any]) -> Any:
new_true, new_pred = [], []
i = 0
for t, p in zip(true, pred):
new_true.append(t[0])
new_pred.append(p[0] if p[1] == t[1] else '—')
true, pred = new_true, new_pred
labels = sorted(set(true + pred))
errors = Counter((t, p) for t, p in zip(true, pred) if t != p)
return confusion_matrix(true, pred, labels=labels), labels, errors