Source code for hydra.parser

"""The main class for the Hydra parser is defined here."""

from __future__ import annotations

import json
import time

from collections import defaultdict
from itertools import chain
from math import floor
from typing import Any, Callable

import logging
logger = logging.getLogger(__name__)
logger.debug('Hydra says hi!')

import morfeusz2

import torch
from transformers import AutoTokenizer, PreTrainedTokenizerBase

from datasets.features import Features

from .utils.hydra import load_config, category_names, get_labels, get_param
from .utils.dataset import  masked_word_ids, morf_tokenize
from .utils.hybrid_tree import make_lemma, correct_lemma, EXC, drop_rozpodabniacz, get_heads, get_graph_edges, make_tree, tree2dict, get_tree_dict_yield
from .utils.conllu import get_conllu
from .utils.ner import get_nes
from .utils.model import get_model_class, MultiTargetTokenClassification
from .utils.constants import (
    SEG_BEGIN,
    SEGS,
    LEMMAS,
    LEMMA_CASES,
    LEMMA_RULES,
    TAGS,
    HEADS,
    ADJACENCY_MATRIX,
    DEPRELS,
    GRAPH_HEADS,
    GRAPH_ADJACENCY_MATRIX,
    GRAPH_DEPRELS,
    SPINES,
    NE_IOBS,
    MASK_VALUE,
    EAGER,
    MST,
)
from .utils.types import TokenList, IntLabelList, StrLabelList, AnyLabelList, LabelDict, TreeDict, HydraAnalysis

[docs] class Hydra(object): """A Hydra parser. Use of :meth:`~hydra.training_and_evaluation.Trainer.train` recommended for new instance creation.""" def __init__( self, pretrained_path: str, model_type: str, model: MultiTargetTokenClassification, labels: dict[str, list[str]], segmentation: bool = False, lemmatisation: bool = False, tagging: bool = False, dependency: bool = False, graph: bool = False, hybrid: bool = False, nonterminal_features: list[str] | None = None, named_entities: bool = False, morfeusz_dict_path: str | None ='polimorf', morfeusz_dict_name: str | None = None, pretrained_tokenizer: PreTrainedTokenizerBase | None = None, tokenizer_kwargs: dict[str, Any] | None = None, ) -> None: self.pretrained_path = pretrained_path self.model_type = model_type self.model = model self.segmentation = segmentation self.lemmatisation = lemmatisation self.tagging = tagging self.dependency = dependency self.graph=graph self.hybrid = hybrid self.nonterminal_features = nonterminal_features if nonterminal_features is not None else [] self.named_entities = named_entities self.morfeusz_dict_path = morfeusz_dict_path self.morfeusz_dict_name = morfeusz_dict_name self.categories = category_names( segmentation, lemmatisation, tagging, dependency, graph, hybrid, nonterminal_features, named_entities) self.labels = labels self.tokenizer_kwargs = tokenizer_kwargs if tokenizer_kwargs is not None else {} if pretrained_tokenizer is not None: self.pretrained_tokenizer = pretrained_tokenizer else: self.pretrained_tokenizer = AutoTokenizer.from_pretrained(pretrained_path, **self.tokenizer_kwargs) if self.segmentation or self.lemmatisation or self.tagging: self.morfeusz = morfeusz2.Morfeusz( generate=False, dict_path=self.morfeusz_dict_path, dict_name=self.morfeusz_dict_name, expand_tags=True, aggl='isolated' ) else: self.morfeusz = None self.morfeusz_dict_path = None self.morfeusz_dict_name = None
[docs] def save(self, path: str) -> None: """Save a trained Hydra model to disk.""" self.model.save_pretrained(f'{path}/model') config = { 'segmentation' : self.segmentation, 'lemmatisation' : self.lemmatisation, 'tagging' : self.tagging, 'dependency' : self.dependency, 'graph' : self.graph, 'hybrid' : self.hybrid, 'nonterminal_features' : self.nonterminal_features, 'named_entities' : self.named_entities, 'morfeusz_dict_path' : self.morfeusz_dict_path, 'morfeusz_dict_name' : self.morfeusz_dict_name, 'pretrained_path' : self.pretrained_path, 'tokenizer_kwargs' : self.tokenizer_kwargs, 'model_type' : self.model_type, 'labels' : self.labels, } with open(f'{path}/config.json', 'w') as f: json.dump(config, f, ensure_ascii=False)
[docs] @classmethod def create( cls, pretrained_path: str, model_type: str, features: Features, segmentation: bool = False, lemmatisation: bool = False, tagging: bool = False, dependency: bool = False, graph: bool = False, hybrid: bool = False, nonterminal_features: list[str] | None = None, named_entities: bool = False, morfeusz_dict_path: str | None = None, morfeusz_dict_name: str | None = 'sgjp', pretrained_tokenizer: PreTrainedTokenizerBase | None = None, tokenizer_kwargs: dict[str, Any] | None = None, ) -> Hydra: """Create a new Hydra instance. Args: pretrained_path: pretrained language model to be fine-tuned. model_type: pretrained language model type. features: the features signature of the dataset to be used for training. segmentation: enable segmentation module. lemmatisation: enable lemmatisation module. tagging: enable tagging module. dependency: enable dependency parsing module. graph: enable graph parsing module (extended/additional edges). hybrid: enable hybrid syntactic parsing module. nonterminal_features: nonterminal features to predict (experimental). named_entities: enable NER module. morfeusz_dict_path: path to directory containing Morfeusz dictionary (if not using the default one). morfeusz_dict_name: name of the Morfeusz dictionary (if not using the default one). pretrained_tokenizer: tokenizer for the language model. tokenizer_kwargs: addditonal kwargs for the language model tokenizer. Returns: a Hydra instance. """ if hybrid and not dependency: logger.warning('Hybrid parsing requires dependency parsing, setting `dependency` to True.') dependency = True categories = category_names( segmentation, lemmatisation, tagging, dependency, graph, hybrid, nonterminal_features, named_entities) labels = get_labels(features, categories) model = get_model_class(model_type).from_pretrained( pretrained_path, categories=categories, labels=labels, ) return cls( pretrained_path, model_type, model, labels, segmentation=segmentation, lemmatisation=lemmatisation, tagging=tagging, dependency=dependency, graph=graph, hybrid=hybrid, nonterminal_features=nonterminal_features, named_entities=named_entities, morfeusz_dict_path=morfeusz_dict_path, morfeusz_dict_name=morfeusz_dict_name, pretrained_tokenizer=pretrained_tokenizer, tokenizer_kwargs=tokenizer_kwargs if tokenizer_kwargs is not None else {}, )
[docs] @classmethod def load(cls, path: str, morfeusz_dict_path: str | None = None) -> Hydra: """Load a trained Hydra model. Returns: a Hydra instance. """ config = load_config(path) labels = get_param(config, 'labels') segmentation = get_param(config, 'segmentation') lemmatisation = get_param(config, 'lemmatisation') tagging = get_param(config, 'tagging') dependency = get_param(config, 'dependency') graph = get_param(config, 'graph', default=False) hybrid = get_param(config, 'hybrid', alt_keys=['spines']) nonterminal_features = get_param(config, 'nonterminal_features') named_entities = get_param(config, 'named_entities') if not morfeusz_dict_path: morfeusz_dict_path = get_param(config, 'morfeusz_dict_path', allow_none=True) morfeusz_dict_name = get_param(config, 'morfeusz_dict_name', allow_none=True) logger.debug(f'morfeusz_dict_path: {morfeusz_dict_path}, morfeusz_dict_name: {morfeusz_dict_name}') # for backwards compatibility when only BERT was possible pretrained_path = get_param(config, 'pretrained_path', alt_keys=['bert_path']) model_type = get_param(config, 'model_type', default='bert') tokenizer_kwargs = get_param(config, 'tokenizer_kwargs', default={}) categories = category_names( segmentation, lemmatisation, tagging, dependency, graph, hybrid, nonterminal_features, named_entities) if lemmatisation and len(labels[LEMMA_CASES]) == 2: logger.warning('This model uses legacy rules for lemma casing, consider training a new model.') model = get_model_class(model_type).from_pretrained( f'{path}/model', # use gpu if available device_map='auto', categories=categories, labels=labels, ) return cls( pretrained_path, model_type, model, labels, segmentation=segmentation, lemmatisation=lemmatisation, tagging=tagging, dependency=dependency, graph=graph, hybrid=hybrid, nonterminal_features=nonterminal_features, named_entities=named_entities, morfeusz_dict_path=morfeusz_dict_path, morfeusz_dict_name=morfeusz_dict_name, tokenizer_kwargs=tokenizer_kwargs, )
def _retokenize_mask(self, tokens: TokenList, seg: StrLabelList, min_tokens: TokenList) -> tuple[TokenList, IntLabelList]: """Extract model-predicted sentence segmentation. Args: tokens: tokens produced by the underlying language model’s tokeniser. seg: model-predicted segmentation labels (SEG_BEGIN/SEG_INSIDE). min_tokens: minimally fine segmentation required. Returns: model-predicted tokens (segments) and 1/None mask marking segment-initial tokens. """ tok2 = '' index = 0 assert(len(tokens) == len(seg)) tokens2 = [] mask: list[int | None] = [] for token, seglabel in zip(tokens, seg): if seglabel == SEG_BEGIN or tok2 == min_tokens[index]: tokens2.append(token) mask.append(1) else: tokens2[-1] += token mask.append(None) if tok2 == min_tokens[index]: tok2 = '' index += 1 tok2 += token return tokens2, mask def _select_unmasked(self, labels: AnyLabelList, mask: IntLabelList, category: str) -> AnyLabelList: """Filter out any masked labels. Returns: only the non-masked values. """ matrix = category in (ADJACENCY_MATRIX, GRAPH_ADJACENCY_MATRIX, GRAPH_DEPRELS) return [ # type: ignore [return-value] lbl if not matrix else self._select_unmasked(lbl, mask, '') # type: ignore[arg-type] # make matrix == False in the nested call for lbl, m in zip(labels, mask) if m is not None ] def _get_nps(self, sentence: str, tokens: TokenList) -> list[bool]: """Generate no-preceding-space annotation for a sentence. Returns: no-preceding-space labels. """ nps = [] for i, token in enumerate(tokens): if i == 0: nps.append(False) else: nps.append(sentence.startswith(token)) while not sentence.startswith(token): sentence = sentence[1:] assert(sentence.startswith(token)) sentence = sentence[len(token):] assert (not sentence) return nps def _add_nps(self, tree_dict: TreeDict, nps: list[bool]) -> None: """Add no-preceding-space annotations to a tree json.""" terminals = sorted(get_tree_dict_yield(tree_dict), key=lambda x: x['span']['from']) assert len(terminals) == len(nps) for terminal, _nps in zip(terminals, nps): if 'features' not in terminal: terminal['features'] = {} terminal['features']['nps'] = _nps def _add_oov(self, tree_dict: TreeDict) -> None: """Add out-of-vocabulary annotations to a tree json.""" for terminal in get_tree_dict_yield(tree_dict): orth = terminal['orth'] base = terminal['base'] tag = terminal['tag'] oov = (orth, base, tag) not in EXC if oov: for i, j, interp in self.morfeusz.analyse(orth): o, b, t = interp[:3] b = drop_rozpodabniacz(b, t) if (o, b, t) == (orth, base, tag): oov = False if 'features' not in terminal: terminal['features'] = {} terminal['features']['oov'] = oov
[docs] def process_labels( self, labels: LabelDict, tokens: TokenList, correct_lemmata: bool = False, morph_postprocess: Callable[[str, str, str], tuple[str, str]] | None = None, mst: str = MST, allow_masked: bool = False, ) -> LabelDict: """Process model-predicted labels. Args: labels: model-predicted labels. tokens: the processed sentence. correct_lemmata: apply lemmata correction using Morfeusz. morph_postprocess: custom postprocessing function for tags and lemmata. mst: dependency tree decoding algorithm to use: :const:`~hydra.utils.constants.MST` or :const:`~hydra.utils.constants.EAGER`. allow_masked: accept masked labels. Returns: labels that can be returned as parsing result or used to construct a tree. """ if self.lemmatisation: rules = labels.pop(LEMMA_RULES) cases = labels.pop(LEMMA_CASES) legacy_casing = len(self.labels[LEMMA_CASES]) == 2 labels[LEMMAS] = [make_lemma(*x, legacy_casing=legacy_casing) for x in zip(tokens, cases, rules)] # type: ignore[arg-type] else: labels[LEMMAS] = ['_' for _ in tokens] if correct_lemmata: tags = labels[TAGS] lemmas = labels[LEMMAS] # may be an empty sentence... if tags: new_lemmas, new_tags = map(list, zip(*(correct_lemma(*x, self.morfeusz) for x in zip(tokens, lemmas, tags)))) # type: ignore[arg-type] labels[LEMMAS] = new_lemmas labels[TAGS] = new_tags if morph_postprocess: tags = labels[TAGS] lemmas = labels[LEMMAS] # may be an empty sentence... if tags: new_lemmas, new_tags = map(list, zip(*(morph_postprocess(*x) for x in zip(tokens, lemmas, tags)))) # type: ignore[arg-type] labels[LEMMAS] = new_lemmas labels[TAGS] = new_tags if self.dependency: matrix = labels.pop(ADJACENCY_MATRIX) if allow_masked and set(chain.from_iterable(matrix)) == {MASK_VALUE}: labels[HEADS] = [MASK_VALUE for _ in tokens] else: labels[HEADS] = get_heads(matrix, mst=mst) # type: ignore[arg-type] # ints/floats # !!! dodane 26.07.2025 if labels[HEADS]: # check in case of an empty input string labels[DEPRELS][labels[HEADS].index(None)] = 'root' if self.graph: graph = labels.pop(GRAPH_ADJACENCY_MATRIX) deprels = labels.pop(GRAPH_DEPRELS) if allow_masked and set(chain.from_iterable(graph)) == {MASK_VALUE}: labels[GRAPH_HEADS] = [[MASK_VALUE] for _ in tokens] labels[GRAPH_DEPRELS] = [[None] for _ in tokens] else: labels[GRAPH_HEADS], labels[GRAPH_DEPRELS] = get_graph_edges(graph, deprels) return labels
[docs] def parse( self, sentences: list[str] | str, correct_lemmata: bool = False, return_jsons: bool = False, return_conllu: bool = False, return_trees: bool = False, return_labels: bool = False, return_logits: bool = False, root_label: str | None = None, force_root_label: bool = False, skip_long: bool = False, is_tokenized: bool = False, morph_postprocess: Callable[[str, str, str], tuple[str, str]] | None = None, mst: str = MST, return_times: bool = False, mem_limit: int | None = None, verbose: bool = False, ) -> list[HydraAnalysis] | tuple[list[HydraAnalysis], dict[str, int]]: """Parse a single sentence or a list of sentences. Args: sentences: sentence(s) to process. correct_lemmata: apply lemmata correction using Morfeusz. return_jsons: include full json outputs in the results. Only available for models trained with `hybrid=True`. return_conllu: include ConLLu outputs in the results. Only available for models trained with `dependency=True`. return_trees: include tree objects in the results. Only available for models trained with `hybrid=True`. return_labels: include label lists in the results. return_logits: include raw predicted logits in the results. root_label: used by `force_root_label`. force_root_label: enforce a specific root label in resulting trees. skip_long: omit sentences with too many tokens (an error message will be logged in the sentences’ metadata). is_tokenized: don’t perform segmentation (treat spaces as segment delimiters). morph_postprocess: custom postprocessing function for tags and lemmata. mst: dependency tree decoding algorithm to use: :const:`~hydra.utils.constants.MST` or :const:`~hydra.utils.constants.EAGER`. return_times: include parsing time info in the results. mem_limit: memory limit (experimental). verbose: include additional info in the results. Returns: the parsing results. """ t1 = time.perf_counter_ns() if return_logits and sum((return_jsons, return_conllu, return_trees, return_labels)): logger.warning('`return_logits` can’t be used with other formats, disabling all other formats.') return_jsons = False return_conllu = False return_trees = False return_labels = False if not is_tokenized and not self.segmentation: raise RuntimeError('This model can’t tokenize, please use is_tokenized=True and pass a space-separated tokenized sentence, e.g ‘Miał em kota .’') if correct_lemmata and not (self.lemmatisation and self.tagging): logger.warning('This model can’t lemmatise and/or tag, setting `correct_lemmata` to False.') correct_lemmata = False if correct_lemmata and not (self.lemmatisation and self.tagging): logger.warning('This model can’t lemmatise and/or tag, ignoring the `morph_postprocess` argument.') morph_postprocess = None if return_jsons and not self.hybrid: logger.warning('This model can’t perform hybrid parsing and won’t return jsons, setting `return_jsons` to False.') return_jsons = False if return_trees and not self.hybrid: logger.warning('This model can’t perform hybrid parsing and won’t return trees, setting `return_trees` to False.') return_trees = False if force_root_label and not self.hybrid: logger.warning('This model can’t perform hybrid parsing, `force_root_label` will have no effect.') force_root_label = False if force_root_label and not root_label: logger.critical('`force_root_label` was set to True with no `root_label` provided.') raise RuntimeError if return_conllu and not self.dependency: logger.warning('This model can’t perform dependency parsing and won’t return conllu, setting `return_conllu` to False.') return_conllu = False if not return_jsons and not return_labels and self.named_entities: logger.info('This model can do NER. Use return_jsons=True or return_labels=True to retrieve them.') if verbose and not return_jsons: logger.warning('verbose=True has no effect with return_jsons=False. Use return_jsons=True to get verbose info.') if sum((return_jsons, return_conllu, return_trees, return_labels, return_logits)) == 0: logger.critical('At least one of the parameters must be set to True: ' + \ '`return_jsons`, `return_conllu`, `return_trees`, `return_labels`, `return_logits`.') raise RuntimeError if isinstance(sentences, str): sentences = [sentences] # some tokenizers replace spaces with '▁' sentences = [self.pretrained_tokenizer._tokenizer.normalizer.normalize_str(s).replace('▁', ' ').strip() for s in sentences] tokens = [s.split() for s in sentences] if self.segmentation and not is_tokenized: #tokens = [morf_tokenize(' '.join(toks), self.morfeusz) for toks in tokens] tokens = [list(chain.from_iterable(morf_tokenize(t, self.morfeusz) for t in toks)) for toks in tokens] num_lm_tokens = self.pretrained_tokenizer( tokens, is_split_into_words=True, truncation=False, padding=False, return_length=True )['length'] max_seq_len = max(num_lm_tokens) max_valid_len = self.pretrained_tokenizer.model_max_length if max_seq_len > max_valid_len and not skip_long: raise RuntimeError(f'LM tokenizer produced a sequence of {max_seq_len} tokens which exceeds the model’s limit ({self.pretrained_tokenizer.model_max_length}). Parse shorter sentences or call parse with skip_long=True.') valid_indices = [i for i, l in enumerate(num_lm_tokens) if l <= max_valid_len] invalid_indices = [i for i, l in enumerate(num_lm_tokens) if l > max_valid_len] sequence_is_valid = [l <= max_valid_len for l in num_lm_tokens] tokenized = self.pretrained_tokenizer( [tokens[i] for i in valid_indices], is_split_into_words=True, return_offsets_mapping=True, padding=True, return_tensors='pt', ) t2 = time.perf_counter_ns() self.model.eval() # type: ignore[no-untyped-call] device = self.model.device num_samples = tokenized['input_ids'].shape[0] if mem_limit is not None: k = 64#2 * 128 batch_size = min(num_samples, max(1, floor(k * mem_limit / (max_seq_len * max_seq_len)))) else: batch_size = num_samples logits_lists = defaultdict(list) with torch.no_grad(): for i in range(0, num_samples, batch_size): batch = {k: v[i:(i + batch_size)].to(device) for k, v in tokenized.items()} for k, v in self.model(**batch).logits.items(): logits_lists[k].append(v.cpu()) predicted = {k : torch.cat(v, dim=0) for k, v in logits_lists.items()} t3 = time.perf_counter_ns() # values are logits if return_logits == True, labels otherwise labels = dict() for cat, pred in predicted.items(): if return_logits and cat != SEGS: _lbls = pred.numpy() else: if cat == ADJACENCY_MATRIX: _lbls = torch.nn.functional.softmax(pred, dim=-1).numpy() elif cat == GRAPH_ADJACENCY_MATRIX: _lbls = torch.round(torch.nn.functional.sigmoid(pred)).numpy() elif cat == GRAPH_DEPRELS: label_ids = torch.argmax(pred, dim=-1).numpy() _lbls = [[[self.labels[cat][i] for i in row_ids] for row_ids in instance_ids] for instance_ids in label_ids] else: label_ids = torch.argmax(pred, dim=-1).numpy() _lbls = [[self.labels[cat][i] for i in l_ids] for l_ids in label_ids] labels[cat] = _lbls trees = [] idx = 0 for i, (is_valid, tkns, sentence, num_lm_toks) in enumerate(zip(sequence_is_valid, tokens, sentences, num_lm_tokens)): TREE: HydraAnalysis = { 'sentence' : sentence, 'tokens' : tkns, # leaving 'bert_tokens' key for backwards compatibility, 'lm_tokens' would be better 'metadata' : { 'bert_tokens' : num_lm_toks }, } if not is_valid: TREE['metadata']['error'] = 'too long' trees.append(TREE) continue mask = masked_word_ids(tokenized.word_ids(idx)) # values are logits if return_logits == True, labels otherwise, except for segments (always labels!) selected_labels = {cat : self._select_unmasked(lbls[idx], mask, cat) for cat, lbls in labels.items()} nps = None # empty sentence can happen for automatically sentenced text, eg. single '\u200b' bert-normalised to '' if self.segmentation and sentence: # remove the seg labels seg: StrLabelList = selected_labels.pop(SEGS) # type: ignore[assignment] # we know they’re labels if not is_tokenized: # can’t start with ‘I’nside segment seg[0] = SEG_BEGIN tkns, mask = self._retokenize_mask(tkns, seg, sentence.split()) selected_labels = {cat : self._select_unmasked(lbl, mask, cat) for cat, lbl in selected_labels.items()} nps = self._get_nps(sentence, tkns) TREE['tokens'] = tkns if return_logits: TREE['logits'] = selected_labels trees.append(TREE) continue processed_labels = self.process_labels( selected_labels, # type: ignore[arg-type] # dict/LabelDict, but at this point we know they’re labels (return_logits == False) tkns, correct_lemmata=correct_lemmata, morph_postprocess=morph_postprocess, mst=mst ) if return_trees or return_jsons: if not sentence: if return_trees: TREE['tree'] = None if return_jsons: TREE['json'] = None else: tree, info, heads = make_tree(tkns, processed_labels, self.nonterminal_features, root_label=root_label, force_root_label=force_root_label) processed_labels[HEADS] = heads if return_trees: TREE['tree'] = tree if return_jsons: tree_dict = tree2dict(tree) if self.segmentation and not is_tokenized: assert (nps is not None) self._add_nps(tree_dict, nps) if self.tagging and self.lemmatisation: self._add_oov(tree_dict) if self.named_entities: tree_dict['named_entities'] = get_nes(tkns, processed_labels[NE_IOBS]) if verbose: tree_dict['info'] = info TREE['json'] = tree_dict if return_conllu: TREE['conllu'] = get_conllu(tkns, processed_labels, nps) # do this last as make_tree might slightly change the the tree if return_labels: if not is_tokenized: assert (nps is not None) processed_labels['nps'] = nps TREE['labels'] = processed_labels trees.append(TREE) idx += 1 t4 = time.perf_counter_ns() if return_times: return trees, { 'total' : t4 - t1, 'preprocessing': t2 - t1, 'prediction' : t3 - t2, 'postprocessing' : t4 - t3 } else: return trees