"""The main class for the Hydra parser is defined here."""
from __future__ import annotations
import json
import time
from collections import defaultdict
from itertools import chain
from math import floor
from typing import Any, Callable
import logging
logger = logging.getLogger(__name__)
logger.debug('Hydra says hi!')
import morfeusz2
import torch
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from datasets.features import Features
from .utils.hydra import load_config, category_names, get_labels, get_param
from .utils.dataset import masked_word_ids, morf_tokenize
from .utils.hybrid_tree import make_lemma, correct_lemma, EXC, drop_rozpodabniacz, get_heads, get_graph_edges, make_tree, tree2dict, get_tree_dict_yield
from .utils.conllu import get_conllu
from .utils.ner import get_nes
from .utils.model import get_model_class, MultiTargetTokenClassification
from .utils.constants import (
SEG_BEGIN,
SEGS,
LEMMAS,
LEMMA_CASES,
LEMMA_RULES,
TAGS,
HEADS,
ADJACENCY_MATRIX,
DEPRELS,
GRAPH_HEADS,
GRAPH_ADJACENCY_MATRIX,
GRAPH_DEPRELS,
SPINES,
NE_IOBS,
MASK_VALUE,
EAGER,
MST,
)
from .utils.types import TokenList, IntLabelList, StrLabelList, AnyLabelList, LabelDict, TreeDict, HydraAnalysis
[docs]
class Hydra(object):
"""A Hydra parser. Use of :meth:`~hydra.training_and_evaluation.Trainer.train` recommended for new instance creation."""
def __init__(
self,
pretrained_path: str,
model_type: str,
model: MultiTargetTokenClassification,
labels: dict[str, list[str]],
segmentation: bool = False,
lemmatisation: bool = False,
tagging: bool = False,
dependency: bool = False,
graph: bool = False,
hybrid: bool = False,
nonterminal_features: list[str] | None = None,
named_entities: bool = False,
morfeusz_dict_path: str | None ='polimorf',
morfeusz_dict_name: str | None = None,
pretrained_tokenizer: PreTrainedTokenizerBase | None = None,
tokenizer_kwargs: dict[str, Any] | None = None,
) -> None:
self.pretrained_path = pretrained_path
self.model_type = model_type
self.model = model
self.segmentation = segmentation
self.lemmatisation = lemmatisation
self.tagging = tagging
self.dependency = dependency
self.graph=graph
self.hybrid = hybrid
self.nonterminal_features = nonterminal_features if nonterminal_features is not None else []
self.named_entities = named_entities
self.morfeusz_dict_path = morfeusz_dict_path
self.morfeusz_dict_name = morfeusz_dict_name
self.categories = category_names(
segmentation, lemmatisation, tagging, dependency, graph, hybrid, nonterminal_features, named_entities)
self.labels = labels
self.tokenizer_kwargs = tokenizer_kwargs if tokenizer_kwargs is not None else {}
if pretrained_tokenizer is not None:
self.pretrained_tokenizer = pretrained_tokenizer
else:
self.pretrained_tokenizer = AutoTokenizer.from_pretrained(pretrained_path, **self.tokenizer_kwargs)
if self.segmentation or self.lemmatisation or self.tagging:
self.morfeusz = morfeusz2.Morfeusz(
generate=False,
dict_path=self.morfeusz_dict_path,
dict_name=self.morfeusz_dict_name,
expand_tags=True,
aggl='isolated'
)
else:
self.morfeusz = None
self.morfeusz_dict_path = None
self.morfeusz_dict_name = None
[docs]
def save(self, path: str) -> None:
"""Save a trained Hydra model to disk."""
self.model.save_pretrained(f'{path}/model')
config = {
'segmentation' : self.segmentation,
'lemmatisation' : self.lemmatisation,
'tagging' : self.tagging,
'dependency' : self.dependency,
'graph' : self.graph,
'hybrid' : self.hybrid,
'nonterminal_features' : self.nonterminal_features,
'named_entities' : self.named_entities,
'morfeusz_dict_path' : self.morfeusz_dict_path,
'morfeusz_dict_name' : self.morfeusz_dict_name,
'pretrained_path' : self.pretrained_path,
'tokenizer_kwargs' : self.tokenizer_kwargs,
'model_type' : self.model_type,
'labels' : self.labels,
}
with open(f'{path}/config.json', 'w') as f:
json.dump(config, f, ensure_ascii=False)
[docs]
@classmethod
def create(
cls,
pretrained_path: str,
model_type: str,
features: Features,
segmentation: bool = False,
lemmatisation: bool = False,
tagging: bool = False,
dependency: bool = False,
graph: bool = False,
hybrid: bool = False,
nonterminal_features: list[str] | None = None,
named_entities: bool = False,
morfeusz_dict_path: str | None = None,
morfeusz_dict_name: str | None = 'sgjp',
pretrained_tokenizer: PreTrainedTokenizerBase | None = None,
tokenizer_kwargs: dict[str, Any] | None = None,
) -> Hydra:
"""Create a new Hydra instance.
Args:
pretrained_path: pretrained language model to be fine-tuned.
model_type: pretrained language model type.
features: the features signature of the dataset to be used for training.
segmentation: enable segmentation module.
lemmatisation: enable lemmatisation module.
tagging: enable tagging module.
dependency: enable dependency parsing module.
graph: enable graph parsing module (extended/additional edges).
hybrid: enable hybrid syntactic parsing module.
nonterminal_features: nonterminal features to predict (experimental).
named_entities: enable NER module.
morfeusz_dict_path: path to directory containing Morfeusz dictionary (if not using the default one).
morfeusz_dict_name: name of the Morfeusz dictionary (if not using the default one).
pretrained_tokenizer: tokenizer for the language model.
tokenizer_kwargs: addditonal kwargs for the language model tokenizer.
Returns:
a Hydra instance.
"""
if hybrid and not dependency:
logger.warning('Hybrid parsing requires dependency parsing, setting `dependency` to True.')
dependency = True
categories = category_names(
segmentation, lemmatisation, tagging, dependency, graph, hybrid, nonterminal_features, named_entities)
labels = get_labels(features, categories)
model = get_model_class(model_type).from_pretrained(
pretrained_path,
categories=categories,
labels=labels,
)
return cls(
pretrained_path,
model_type,
model,
labels,
segmentation=segmentation,
lemmatisation=lemmatisation,
tagging=tagging,
dependency=dependency,
graph=graph,
hybrid=hybrid,
nonterminal_features=nonterminal_features,
named_entities=named_entities,
morfeusz_dict_path=morfeusz_dict_path,
morfeusz_dict_name=morfeusz_dict_name,
pretrained_tokenizer=pretrained_tokenizer,
tokenizer_kwargs=tokenizer_kwargs if tokenizer_kwargs is not None else {},
)
[docs]
@classmethod
def load(cls, path: str, morfeusz_dict_path: str | None = None) -> Hydra:
"""Load a trained Hydra model.
Returns:
a Hydra instance.
"""
config = load_config(path)
labels = get_param(config, 'labels')
segmentation = get_param(config, 'segmentation')
lemmatisation = get_param(config, 'lemmatisation')
tagging = get_param(config, 'tagging')
dependency = get_param(config, 'dependency')
graph = get_param(config, 'graph', default=False)
hybrid = get_param(config, 'hybrid', alt_keys=['spines'])
nonterminal_features = get_param(config, 'nonterminal_features')
named_entities = get_param(config, 'named_entities')
if not morfeusz_dict_path:
morfeusz_dict_path = get_param(config, 'morfeusz_dict_path', allow_none=True)
morfeusz_dict_name = get_param(config, 'morfeusz_dict_name', allow_none=True)
logger.debug(f'morfeusz_dict_path: {morfeusz_dict_path}, morfeusz_dict_name: {morfeusz_dict_name}')
# for backwards compatibility when only BERT was possible
pretrained_path = get_param(config, 'pretrained_path', alt_keys=['bert_path'])
model_type = get_param(config, 'model_type', default='bert')
tokenizer_kwargs = get_param(config, 'tokenizer_kwargs', default={})
categories = category_names(
segmentation, lemmatisation, tagging, dependency, graph, hybrid, nonterminal_features, named_entities)
if lemmatisation and len(labels[LEMMA_CASES]) == 2:
logger.warning('This model uses legacy rules for lemma casing, consider training a new model.')
model = get_model_class(model_type).from_pretrained(
f'{path}/model',
# use gpu if available
device_map='auto',
categories=categories,
labels=labels,
)
return cls(
pretrained_path,
model_type,
model,
labels,
segmentation=segmentation,
lemmatisation=lemmatisation,
tagging=tagging,
dependency=dependency,
graph=graph,
hybrid=hybrid,
nonterminal_features=nonterminal_features,
named_entities=named_entities,
morfeusz_dict_path=morfeusz_dict_path,
morfeusz_dict_name=morfeusz_dict_name,
tokenizer_kwargs=tokenizer_kwargs,
)
def _retokenize_mask(self, tokens: TokenList, seg: StrLabelList, min_tokens: TokenList) -> tuple[TokenList, IntLabelList]:
"""Extract model-predicted sentence segmentation.
Args:
tokens: tokens produced by the underlying language model’s tokeniser.
seg: model-predicted segmentation labels (SEG_BEGIN/SEG_INSIDE).
min_tokens: minimally fine segmentation required.
Returns:
model-predicted tokens (segments) and 1/None mask marking segment-initial tokens.
"""
tok2 = ''
index = 0
assert(len(tokens) == len(seg))
tokens2 = []
mask: list[int | None] = []
for token, seglabel in zip(tokens, seg):
if seglabel == SEG_BEGIN or tok2 == min_tokens[index]:
tokens2.append(token)
mask.append(1)
else:
tokens2[-1] += token
mask.append(None)
if tok2 == min_tokens[index]:
tok2 = ''
index += 1
tok2 += token
return tokens2, mask
def _select_unmasked(self, labels: AnyLabelList, mask: IntLabelList, category: str) -> AnyLabelList:
"""Filter out any masked labels.
Returns:
only the non-masked values.
"""
matrix = category in (ADJACENCY_MATRIX, GRAPH_ADJACENCY_MATRIX, GRAPH_DEPRELS)
return [ # type: ignore [return-value]
lbl if not matrix else self._select_unmasked(lbl, mask, '') # type: ignore[arg-type] # make matrix == False in the nested call
for lbl, m in zip(labels, mask) if m is not None
]
def _get_nps(self, sentence: str, tokens: TokenList) -> list[bool]:
"""Generate no-preceding-space annotation for a sentence.
Returns:
no-preceding-space labels.
"""
nps = []
for i, token in enumerate(tokens):
if i == 0:
nps.append(False)
else:
nps.append(sentence.startswith(token))
while not sentence.startswith(token):
sentence = sentence[1:]
assert(sentence.startswith(token))
sentence = sentence[len(token):]
assert (not sentence)
return nps
def _add_nps(self, tree_dict: TreeDict, nps: list[bool]) -> None:
"""Add no-preceding-space annotations to a tree json."""
terminals = sorted(get_tree_dict_yield(tree_dict), key=lambda x: x['span']['from'])
assert len(terminals) == len(nps)
for terminal, _nps in zip(terminals, nps):
if 'features' not in terminal:
terminal['features'] = {}
terminal['features']['nps'] = _nps
def _add_oov(self, tree_dict: TreeDict) -> None:
"""Add out-of-vocabulary annotations to a tree json."""
for terminal in get_tree_dict_yield(tree_dict):
orth = terminal['orth']
base = terminal['base']
tag = terminal['tag']
oov = (orth, base, tag) not in EXC
if oov:
for i, j, interp in self.morfeusz.analyse(orth):
o, b, t = interp[:3]
b = drop_rozpodabniacz(b, t)
if (o, b, t) == (orth, base, tag):
oov = False
if 'features' not in terminal:
terminal['features'] = {}
terminal['features']['oov'] = oov
[docs]
def process_labels(
self,
labels: LabelDict,
tokens: TokenList,
correct_lemmata: bool = False,
morph_postprocess: Callable[[str, str, str], tuple[str, str]] | None = None,
mst: str = MST,
allow_masked: bool = False,
) -> LabelDict:
"""Process model-predicted labels.
Args:
labels: model-predicted labels.
tokens: the processed sentence.
correct_lemmata: apply lemmata correction using Morfeusz.
morph_postprocess: custom postprocessing function for tags and lemmata.
mst: dependency tree decoding algorithm to use: :const:`~hydra.utils.constants.MST` or :const:`~hydra.utils.constants.EAGER`.
allow_masked: accept masked labels.
Returns:
labels that can be returned as parsing result or used to construct a tree.
"""
if self.lemmatisation:
rules = labels.pop(LEMMA_RULES)
cases = labels.pop(LEMMA_CASES)
legacy_casing = len(self.labels[LEMMA_CASES]) == 2
labels[LEMMAS] = [make_lemma(*x, legacy_casing=legacy_casing) for x in zip(tokens, cases, rules)] # type: ignore[arg-type]
else:
labels[LEMMAS] = ['_' for _ in tokens]
if correct_lemmata:
tags = labels[TAGS]
lemmas = labels[LEMMAS]
# may be an empty sentence...
if tags:
new_lemmas, new_tags = map(list, zip(*(correct_lemma(*x, self.morfeusz) for x in zip(tokens, lemmas, tags)))) # type: ignore[arg-type]
labels[LEMMAS] = new_lemmas
labels[TAGS] = new_tags
if morph_postprocess:
tags = labels[TAGS]
lemmas = labels[LEMMAS]
# may be an empty sentence...
if tags:
new_lemmas, new_tags = map(list, zip(*(morph_postprocess(*x) for x in zip(tokens, lemmas, tags)))) # type: ignore[arg-type]
labels[LEMMAS] = new_lemmas
labels[TAGS] = new_tags
if self.dependency:
matrix = labels.pop(ADJACENCY_MATRIX)
if allow_masked and set(chain.from_iterable(matrix)) == {MASK_VALUE}:
labels[HEADS] = [MASK_VALUE for _ in tokens]
else:
labels[HEADS] = get_heads(matrix, mst=mst) # type: ignore[arg-type] # ints/floats
# !!! dodane 26.07.2025
if labels[HEADS]: # check in case of an empty input string
labels[DEPRELS][labels[HEADS].index(None)] = 'root'
if self.graph:
graph = labels.pop(GRAPH_ADJACENCY_MATRIX)
deprels = labels.pop(GRAPH_DEPRELS)
if allow_masked and set(chain.from_iterable(graph)) == {MASK_VALUE}:
labels[GRAPH_HEADS] = [[MASK_VALUE] for _ in tokens]
labels[GRAPH_DEPRELS] = [[None] for _ in tokens]
else:
labels[GRAPH_HEADS], labels[GRAPH_DEPRELS] = get_graph_edges(graph, deprels)
return labels
[docs]
def parse(
self,
sentences: list[str] | str,
correct_lemmata: bool = False,
return_jsons: bool = False,
return_conllu: bool = False,
return_trees: bool = False,
return_labels: bool = False,
return_logits: bool = False,
root_label: str | None = None,
force_root_label: bool = False,
skip_long: bool = False,
is_tokenized: bool = False,
morph_postprocess: Callable[[str, str, str], tuple[str, str]] | None = None,
mst: str = MST,
return_times: bool = False,
mem_limit: int | None = None,
verbose: bool = False,
) -> list[HydraAnalysis] | tuple[list[HydraAnalysis], dict[str, int]]:
"""Parse a single sentence or a list of sentences.
Args:
sentences: sentence(s) to process.
correct_lemmata: apply lemmata correction using Morfeusz.
return_jsons: include full json outputs in the results. Only available for models trained with `hybrid=True`.
return_conllu: include ConLLu outputs in the results. Only available for models trained with `dependency=True`.
return_trees: include tree objects in the results. Only available for models trained with `hybrid=True`.
return_labels: include label lists in the results.
return_logits: include raw predicted logits in the results.
root_label: used by `force_root_label`.
force_root_label: enforce a specific root label in resulting trees.
skip_long: omit sentences with too many tokens (an error message will be logged in the sentences’ metadata).
is_tokenized: don’t perform segmentation (treat spaces as segment delimiters).
morph_postprocess: custom postprocessing function for tags and lemmata.
mst: dependency tree decoding algorithm to use: :const:`~hydra.utils.constants.MST` or :const:`~hydra.utils.constants.EAGER`.
return_times: include parsing time info in the results.
mem_limit: memory limit (experimental).
verbose: include additional info in the results.
Returns:
the parsing results.
"""
t1 = time.perf_counter_ns()
if return_logits and sum((return_jsons, return_conllu, return_trees, return_labels)):
logger.warning('`return_logits` can’t be used with other formats, disabling all other formats.')
return_jsons = False
return_conllu = False
return_trees = False
return_labels = False
if not is_tokenized and not self.segmentation:
raise RuntimeError('This model can’t tokenize, please use is_tokenized=True and pass a space-separated tokenized sentence, e.g ‘Miał em kota .’')
if correct_lemmata and not (self.lemmatisation and self.tagging):
logger.warning('This model can’t lemmatise and/or tag, setting `correct_lemmata` to False.')
correct_lemmata = False
if correct_lemmata and not (self.lemmatisation and self.tagging):
logger.warning('This model can’t lemmatise and/or tag, ignoring the `morph_postprocess` argument.')
morph_postprocess = None
if return_jsons and not self.hybrid:
logger.warning('This model can’t perform hybrid parsing and won’t return jsons, setting `return_jsons` to False.')
return_jsons = False
if return_trees and not self.hybrid:
logger.warning('This model can’t perform hybrid parsing and won’t return trees, setting `return_trees` to False.')
return_trees = False
if force_root_label and not self.hybrid:
logger.warning('This model can’t perform hybrid parsing, `force_root_label` will have no effect.')
force_root_label = False
if force_root_label and not root_label:
logger.critical('`force_root_label` was set to True with no `root_label` provided.')
raise RuntimeError
if return_conllu and not self.dependency:
logger.warning('This model can’t perform dependency parsing and won’t return conllu, setting `return_conllu` to False.')
return_conllu = False
if not return_jsons and not return_labels and self.named_entities:
logger.info('This model can do NER. Use return_jsons=True or return_labels=True to retrieve them.')
if verbose and not return_jsons:
logger.warning('verbose=True has no effect with return_jsons=False. Use return_jsons=True to get verbose info.')
if sum((return_jsons, return_conllu, return_trees, return_labels, return_logits)) == 0:
logger.critical('At least one of the parameters must be set to True: ' + \
'`return_jsons`, `return_conllu`, `return_trees`, `return_labels`, `return_logits`.')
raise RuntimeError
if isinstance(sentences, str):
sentences = [sentences]
# some tokenizers replace spaces with '▁'
sentences = [self.pretrained_tokenizer._tokenizer.normalizer.normalize_str(s).replace('▁', ' ').strip() for s in sentences]
tokens = [s.split() for s in sentences]
if self.segmentation and not is_tokenized:
#tokens = [morf_tokenize(' '.join(toks), self.morfeusz) for toks in tokens]
tokens = [list(chain.from_iterable(morf_tokenize(t, self.morfeusz) for t in toks)) for toks in tokens]
num_lm_tokens = self.pretrained_tokenizer(
tokens,
is_split_into_words=True,
truncation=False,
padding=False,
return_length=True
)['length']
max_seq_len = max(num_lm_tokens)
max_valid_len = self.pretrained_tokenizer.model_max_length
if max_seq_len > max_valid_len and not skip_long:
raise RuntimeError(f'LM tokenizer produced a sequence of {max_seq_len} tokens which exceeds the model’s limit ({self.pretrained_tokenizer.model_max_length}). Parse shorter sentences or call parse with skip_long=True.')
valid_indices = [i for i, l in enumerate(num_lm_tokens) if l <= max_valid_len]
invalid_indices = [i for i, l in enumerate(num_lm_tokens) if l > max_valid_len]
sequence_is_valid = [l <= max_valid_len for l in num_lm_tokens]
tokenized = self.pretrained_tokenizer(
[tokens[i] for i in valid_indices],
is_split_into_words=True,
return_offsets_mapping=True,
padding=True,
return_tensors='pt',
)
t2 = time.perf_counter_ns()
self.model.eval() # type: ignore[no-untyped-call]
device = self.model.device
num_samples = tokenized['input_ids'].shape[0]
if mem_limit is not None:
k = 64#2 * 128
batch_size = min(num_samples, max(1, floor(k * mem_limit / (max_seq_len * max_seq_len))))
else:
batch_size = num_samples
logits_lists = defaultdict(list)
with torch.no_grad():
for i in range(0, num_samples, batch_size):
batch = {k: v[i:(i + batch_size)].to(device) for k, v in tokenized.items()}
for k, v in self.model(**batch).logits.items():
logits_lists[k].append(v.cpu())
predicted = {k : torch.cat(v, dim=0) for k, v in logits_lists.items()}
t3 = time.perf_counter_ns()
# values are logits if return_logits == True, labels otherwise
labels = dict()
for cat, pred in predicted.items():
if return_logits and cat != SEGS:
_lbls = pred.numpy()
else:
if cat == ADJACENCY_MATRIX:
_lbls = torch.nn.functional.softmax(pred, dim=-1).numpy()
elif cat == GRAPH_ADJACENCY_MATRIX:
_lbls = torch.round(torch.nn.functional.sigmoid(pred)).numpy()
elif cat == GRAPH_DEPRELS:
label_ids = torch.argmax(pred, dim=-1).numpy()
_lbls = [[[self.labels[cat][i] for i in row_ids] for row_ids in instance_ids] for instance_ids in label_ids]
else:
label_ids = torch.argmax(pred, dim=-1).numpy()
_lbls = [[self.labels[cat][i] for i in l_ids] for l_ids in label_ids]
labels[cat] = _lbls
trees = []
idx = 0
for i, (is_valid, tkns, sentence, num_lm_toks) in enumerate(zip(sequence_is_valid, tokens, sentences, num_lm_tokens)):
TREE: HydraAnalysis = {
'sentence' : sentence,
'tokens' : tkns,
# leaving 'bert_tokens' key for backwards compatibility, 'lm_tokens' would be better
'metadata' : { 'bert_tokens' : num_lm_toks },
}
if not is_valid:
TREE['metadata']['error'] = 'too long'
trees.append(TREE)
continue
mask = masked_word_ids(tokenized.word_ids(idx))
# values are logits if return_logits == True, labels otherwise, except for segments (always labels!)
selected_labels = {cat : self._select_unmasked(lbls[idx], mask, cat) for cat, lbls in labels.items()}
nps = None
# empty sentence can happen for automatically sentenced text, eg. single '\u200b' bert-normalised to ''
if self.segmentation and sentence:
# remove the seg labels
seg: StrLabelList = selected_labels.pop(SEGS) # type: ignore[assignment] # we know they’re labels
if not is_tokenized:
# can’t start with ‘I’nside segment
seg[0] = SEG_BEGIN
tkns, mask = self._retokenize_mask(tkns, seg, sentence.split())
selected_labels = {cat : self._select_unmasked(lbl, mask, cat) for cat, lbl in selected_labels.items()}
nps = self._get_nps(sentence, tkns)
TREE['tokens'] = tkns
if return_logits:
TREE['logits'] = selected_labels
trees.append(TREE)
continue
processed_labels = self.process_labels(
selected_labels, # type: ignore[arg-type] # dict/LabelDict, but at this point we know they’re labels (return_logits == False)
tkns,
correct_lemmata=correct_lemmata,
morph_postprocess=morph_postprocess,
mst=mst
)
if return_trees or return_jsons:
if not sentence:
if return_trees:
TREE['tree'] = None
if return_jsons:
TREE['json'] = None
else:
tree, info, heads = make_tree(tkns, processed_labels, self.nonterminal_features,
root_label=root_label,
force_root_label=force_root_label)
processed_labels[HEADS] = heads
if return_trees:
TREE['tree'] = tree
if return_jsons:
tree_dict = tree2dict(tree)
if self.segmentation and not is_tokenized:
assert (nps is not None)
self._add_nps(tree_dict, nps)
if self.tagging and self.lemmatisation:
self._add_oov(tree_dict)
if self.named_entities:
tree_dict['named_entities'] = get_nes(tkns, processed_labels[NE_IOBS])
if verbose:
tree_dict['info'] = info
TREE['json'] = tree_dict
if return_conllu:
TREE['conllu'] = get_conllu(tkns, processed_labels, nps)
# do this last as make_tree might slightly change the the tree
if return_labels:
if not is_tokenized:
assert (nps is not None)
processed_labels['nps'] = nps
TREE['labels'] = processed_labels
trees.append(TREE)
idx += 1
t4 = time.perf_counter_ns()
if return_times:
return trees, { 'total' : t4 - t1, 'preprocessing': t2 - t1, 'prediction' : t3 - t2, 'postprocessing' : t4 - t3 }
else:
return trees