"""Utilities for Hydra parser’s named entities module."""
from collections import defaultdict
from itertools import chain
from .constants import SEG_BEGIN, SEG_OUTSIDE
from .types import TokenList, StrLabelList
type NE = list[tuple[int, str]] # (token_idx, token)
[docs]
def get_nes(tokens: TokenList, ne_tags: StrLabelList) -> list[tuple[NE, str]]:
"""Retrieve named entities in a friendly format from model-predicted labels.
Returns:
list of named entities as pairs of <tokens, label>, where tokens are pairs of <id, form>.
"""
nes = []
curr_nes: defaultdict[str, NE] = defaultdict(list)
for i, (token, tag) in enumerate(zip(tokens, ne_tags)):
assert (tag is not None)
if tag == SEG_OUTSIDE:
continue
iobs = tag.split('#')
for iob in iobs:
iob_type, ne_type = iob.split(':')
if iob_type == SEG_BEGIN and curr_nes[ne_type]:
nes.append((curr_nes[ne_type], ne_type))
curr_nes[ne_type] = []
curr_nes[ne_type].append((i, token))
for ne_type, ne in curr_nes.items():
if ne:
nes.append((ne, ne_type))
assert ((set(ne_tags) == {SEG_OUTSIDE}) == (not nes))
return nes