Source code for hydra.utils.ner

"""Utilities for Hydra parser’s named entities module."""

from collections import defaultdict
from itertools import chain

from .constants import SEG_BEGIN, SEG_OUTSIDE
from .types import TokenList, StrLabelList

type NE = list[tuple[int, str]] # (token_idx, token)

[docs] def get_nes(tokens: TokenList, ne_tags: StrLabelList) -> list[tuple[NE, str]]: """Retrieve named entities in a friendly format from model-predicted labels. Returns: list of named entities as pairs of <tokens, label>, where tokens are pairs of <id, form>. """ nes = [] curr_nes: defaultdict[str, NE] = defaultdict(list) for i, (token, tag) in enumerate(zip(tokens, ne_tags)): assert (tag is not None) if tag == SEG_OUTSIDE: continue iobs = tag.split('#') for iob in iobs: iob_type, ne_type = iob.split(':') if iob_type == SEG_BEGIN and curr_nes[ne_type]: nes.append((curr_nes[ne_type], ne_type)) curr_nes[ne_type] = [] curr_nes[ne_type].append((i, token)) for ne_type, ne in curr_nes.items(): if ne: nes.append((ne, ne_type)) assert ((set(ne_tags) == {SEG_OUTSIDE}) == (not nes)) return nes