Source code for hydra.training_and_evaluation

"""Implementation of the training and evaluation logic for Hydra parser."""

import os
import subprocess

from collections import Counter
from datetime import datetime
from itertools import chain
from typing import Any

import logging
logger = logging.getLogger(__name__)

import torch
from torch.utils.data import DataLoader

from datasets import Dataset, DatasetDict, concatenate_datasets
from datasets.features import Features
from transformers import (
    set_seed,
    AutoTokenizer,
    EarlyStoppingCallback,
    EvalPrediction,
    TrainingArguments,
    Trainer as HFTrainer,
)

from .parser import Hydra
from .utils.hydra import category_names, get_labels #get_feature_labels
from .utils.model import get_model_type, ROBERTA
from .utils.data_collator import DataCollatorForHydra
from .utils.dataset import (
    get_dataset_features,
    add_lemma_rules,
    add_spines_and_attachments,
    add_adjacency_matrix,
    add_graph_matrices,
    morfeusz_retokenize,
    lm_tokenize_and_align,
)
from .utils.hybrid_tree import make_tree
from .utils.evaluation import (
    EvaluationMetric,
    UAS,
    LAS,
    Deprels,
    DeprelsConfusionMatrix,
    DepArcs,
    DepArcsConfusionMatrix,
    EUAS,
    ELAS,
    GUAS,
    GLAS,
    NER,
    Tagging,
    TaggingCorr,
    Lemmatisation,
    LemmatisationCorr,
    HeadedSpans,
    NonHeadedSpans,
    UnlabeledSpans,
    HeadedSpansDiscontinuous,
    NonHeadedSpansDiscontinuous,
    UnlabeledSpansDiscontinuous,
    Edges,
)
from .utils.constants import (
    FIRST,
    MASK_VALUE,
    TOKENS,
    SEG_BEGIN,
    SEG_INSIDE,
    SEGS,
    LEMMAS,
    LEMMA_CASES,
    LEMMA_RULES,
    TAGS,
    NONTERMINALS,
    HEADS,
    DEPRELS,
    ADJACENCY_MATRIX,
    SPINES,
    ANCHORS,
    ANCHOR_HS,
    GRAPH_HEADS,
    GRAPH_DEPRELS,
    GRAPH_ADJACENCY_MATRIX,
    NE_IOBS,
    MST,
    EAGER,
)
from .utils.types import TokenList, StrLabelList, LabelDict, HydraAnalysis, DatasetInstance, EvalResults

SPLIT = 'split'
DROP = 'drop'

[docs] class ClassificationTask(object): """A Hydra training configuration.""" def __init__( self, name: str, dataset: DatasetDict, pretrained_path: str, #model_type, segmentation: bool = False, lemmatisation: bool = False, tagging: bool = False, dependency: bool = False, graph: bool = False, hybrid: bool = False, named_entities: bool = False, root_label: str = 'ROOT', force_root_label: bool = False, morfeusz_dict_path: str | None = None, morfeusz_dict_name: str | None = None, nonterminal_features: list[str] | None = None, tree_weight: float = 1.0, batch_size: int = 16, ) -> None: self.name = name self.dataset = dataset self.pretrained_path = pretrained_path self.model_type = get_model_type(pretrained_path) self.segmentation = segmentation self.lemmatisation = lemmatisation self.tagging = tagging self.dependency = dependency self.graph = graph self.hybrid = hybrid self.named_entities = named_entities self.root_label = root_label self.force_root_label = force_root_label self.morfeusz_dict_path = morfeusz_dict_path self.morfeusz_dict_name = morfeusz_dict_name self.nonterminal_features = nonterminal_features if nonterminal_features is not None else [] self.tree_weight = tree_weight self.batch_size = batch_size
[docs] class Trainer(object): """Used for creating, training and evaluating instances of Hydra parser.""" def __init__( self, task: ClassificationTask, ) -> None: self.segmentation = task.segmentation self.lemmatisation = task.lemmatisation self.tagging = task.tagging self.dependency = task.dependency self.graph = task.graph self.hybrid = task.hybrid self.nonterminal_features = task.nonterminal_features self.named_entities = task.named_entities self.root_label = task.root_label self.force_root_label = task.force_root_label self.pretrained_path = task.pretrained_path self.model_type = task.model_type self.masking_strategy = FIRST self.morfeusz_dict_path = task.morfeusz_dict_path self.morfeusz_dict_name = task.morfeusz_dict_name self.categories = category_names(self.segmentation, self.lemmatisation, self.tagging, self.dependency, self.graph, self.hybrid, self.nonterminal_features, self.named_entities) if not self.categories: raise RuntimeError(f'Can’t train without objective(s)!') #if long_sequences not in (SPLIT, DROP): # raise ValueError(f'<long_sequences> must be one of: {SPLIT}, {DROP}') dataset = task.dataset for part in ('train', 'validation'): if part not in task.dataset: raise RuntimeError(f'Can’t train without "{part}" part in the dataset!') dataset_features = get_dataset_features(dataset) if self.lemmatisation: if LEMMAS not in dataset_features: raise RuntimeError(f'Can’t train for lemmatisation without "{LEMMAS}" column in the dataset!') logger.info('Preprocessing the dataset for lemmatisation...') dataset = add_lemma_rules(dataset) if self.tagging: if TAGS not in dataset_features: raise RuntimeError(f'Can’t train for tagging without "{TAGS}" column in the dataset!') column: str if self.hybrid: for column in (NONTERMINALS, HEADS, DEPRELS): if column not in dataset_features: raise RuntimeError(f'Can’t train for hybrid parsing without "{column}" column in the dataset!') logger.info('Preprocessing the dataset for constituency...') dataset = add_spines_and_attachments(dataset) self.dependency = True if self.dependency: for column in (HEADS, DEPRELS): if column not in dataset_features: raise RuntimeError(f'Can’t train for dependency parsing without "{column}" column in the dataset!') logger.info('Preprocessing the dataset for dependency parsing...') dataset = add_adjacency_matrix(dataset) if self.graph: for column in (GRAPH_HEADS, GRAPH_DEPRELS): if column not in dataset_features: raise RuntimeError(f'Can’t train for graph parsing without "{column}" column in the dataset!') logger.info('Preprocessing the dataset for graph parsing...') dataset = add_graph_matrices(dataset) if self.nonterminal_features: if not self.hybrid: raise RuntimeError(f'Can’t train for nonterminal features without training for hybrid parsing!') for ntf in self.nonterminal_features: if ntf not in dataset_features: raise RuntimeError(f'Can’t train for "{ntf}" feature without "{ntf}" column in the dataset!') if self.named_entities: if NE_IOBS not in dataset_features: raise RuntimeError(f'Can’t train for NER without "{NE_IOBS}" column in the dataset!') cols_to_remove = [c for c in dataset_features if c not in self.categories + [TOKENS, LEMMAS, 'sent_id']] logger.info(f'Removing columns: {", ".join(cols_to_remove)}') dataset = dataset.remove_columns(cols_to_remove) if self.segmentation: logger.info('Preprocessing the dataset for segmentation...') dataset = morfeusz_retokenize( dataset, morfeusz_dict_path=self.morfeusz_dict_path, morfeusz_dict_name=self.morfeusz_dict_name) #the features have changed, i.e. spines added dataset_features = get_dataset_features(dataset) self.features = Features({cat : dataset_features[cat] for cat in self.categories}) logger.info('Loading pretrained tokenizer...') self.tokenizer_kwargs = {} if self.model_type == ROBERTA: self.tokenizer_kwargs['add_prefix_space'] = True self.pretrained_tokenizer = AutoTokenizer.from_pretrained(self.pretrained_path, **self.tokenizer_kwargs) logger.info('Preprocessing the dataset for LM...') dataset = self._remove_long_sequences(dataset) self.dataset = dataset.map(lambda x: lm_tokenize_and_align(x, self.pretrained_tokenizer, self.masking_strategy)) self.batch_size = task.batch_size self.tree_weight = task.tree_weight self.train_data = None self.dev_data = None self.eval_parts: list[str] = [] def _get_num_lm_tokens(self, instance : DatasetInstance) -> int: l = self.pretrained_tokenizer(instance['tokens'], is_split_into_words=True, truncation=False, return_length=True)['length'] assert(len(l) == 1) return l[0] # type: ignore[no-any-return] def _remove_long_sequences(self, dataset : DatasetDict) -> DatasetDict: logger.info(f'Removing instances with over {self.pretrained_tokenizer.model_max_length} LM tokens...') return dataset.filter(lambda x: self._get_num_lm_tokens(x) <= self.pretrained_tokenizer.model_max_length) def _data_from_dataset( self, dataset: DatasetDict ) -> None: if dataset is None: dataset = self.dataset self.train_data = None self.dev_data = None self.eval_parts = [] for part, ds in dataset.items(): if part == 'train': self.train_data = ds else: self.eval_parts.append(part) if part == 'validation': self.dev_data = ds # ------------------------------- TRAINING ------------------------------- def _prepare_output_dir(self, path: str) -> None: if not os.path.exists(path): subprocess.call(f'mkdir {path}', shell=True) else: if not os.path.isdir(path): raise ValueError(f'{path} is not a directory') elif os.listdir(path): logger.info(f'emptying {path}') subprocess.call(f'rm -r {path}/*', shell=True)
[docs] def train( self, epochs: int = 10, patience: int = 4, lr: float = 0.00001, log_dir: str | None = None, model_dir: str | None = None, dataset: DatasetDict | None = None ) -> Hydra: """Create and train a parser. Args: epochs: number of epochs. patience: how many epochs are allowed with no training progress. The best model across all epochs is then retrieved. lr: learning rate. log_dir: where to save training logs. model_dir: where to save the final model. dataset: training data. Must contain `'train'` and `'validation'` parts (used for training), with any number of `'test*'` parts for later evaluation. Returns: a parser instance. """ self._data_from_dataset(self.dataset) assert (self.dev_data is not None) if log_dir is not None: self._prepare_output_dir(log_dir) if model_dir is not None: self._prepare_output_dir(model_dir) set_seed(136) parser = Hydra.create( self.pretrained_path, self.model_type, self.features, segmentation=self.segmentation, lemmatisation=self.lemmatisation, tagging=self.tagging, dependency=self.dependency, graph=self.graph, hybrid=self.hybrid, nonterminal_features=self.nonterminal_features, named_entities=self.named_entities, morfeusz_dict_path=self.morfeusz_dict_path, morfeusz_dict_name=self.morfeusz_dict_name, pretrained_tokenizer=self.pretrained_tokenizer, tokenizer_kwargs=self.tokenizer_kwargs, ) if self.tree_weight != 1: loss_weights = { cat : (self.tree_weight if cat in (SPINES, ANCHORS, ANCHOR_HS, DEPRELS, ADJACENCY_MATRIX, GRAPH_DEPRELS, GRAPH_ADJACENCY_MATRIX) else 1.0) for cat in self.categories } parser.model.set_loss_weights(loss_weights) early_stopping = EarlyStoppingCallback( early_stopping_patience=patience, early_stopping_threshold=0.0001, ) training_args = TrainingArguments( output_dir=log_dir, label_names=self.categories, num_train_epochs=epochs, learning_rate=lr, lr_scheduler_type='constant', # Replicates Keras constant LR behavior bf16=True, save_only_model=True, logging_strategy='epoch', eval_strategy='epoch', save_strategy='best', # only keep the best model after each epoch save_total_limit=1, # don’t clog the memory by accumulating eval tensors eval_accumulation_steps=16, dataloader_num_workers=0, metric_for_best_model='avg', load_best_model_at_end=True, ) training_args.set_dataloader(train_batch_size=self.batch_size, eval_batch_size=self.batch_size) data_collator = DataCollatorForHydra(tokenizer=self.pretrained_tokenizer, features=self.features) TRAINER = self class HydraTrainer(HFTrainer): def __init__( self, *args: Any, parser: Hydra, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self.parser = parser def evaluate(self, eval_dataset=None, ignore_keys=None, metric_key_prefix: str = 'eval'): eval_dataset = eval_dataset if eval_dataset is not None else self.eval_dataset GOLD_DATA = TRAINER._get_gold_data(eval_dataset, self.parser, []) PRED_DATA = TRAINER._get_predicted_data(GOLD_DATA[0], self.parser, mst=MST) tokens, tags_true, tags_pred = GOLD_DATA[0], GOLD_DATA[1], PRED_DATA[1] results = TRAINER._evaluate(self.parser, tokens, tags_true, tags_pred, allow_masked=True) metrics = {} if TRAINER.dependency: metrics['uas'] = results['UAS']['result'] metrics['las'] = results['LAS']['result'] if TRAINER.graph: #metrics['euas'] = results['EUAS']['result']['total']['F1'] #metrics['elas'] = results['ELAS']['result']['total']['F1'] metrics['guas'] = results['GUAS']['result']['total']['F1'] metrics['glas'] = results['GLAS']['result']['total']['F1'] if TRAINER.tagging: metrics['tags_acc'] = results['tagging']['result'] if TRAINER.lemmatisation: metrics['lemmas_acc'] = results['lemmatisation']['result'] if TRAINER.hybrid: metrics['constit_f1'] = results['headed constit.']['result']['total']['F1'] metrics['edges_f1'] = results['edges']['result']['total']['F1'] if TRAINER.named_entities: metrics['NER_f1'] = results['NER']['result']['total']['F1'] metrics = {k: 0.0 if v == 'N/A' else v for k, v in metrics.items()} metrics['avg'] = sum(metrics.values()) / len(metrics) metrics = {f'{metric_key_prefix}_{k}': v for k, v in metrics.items()} self.log(metrics) # store in state history (crucial for tracking/callbacks) self.control = self.callback_handler.on_evaluate( self.args, self.state, self.control, metrics ) return metrics hf_trainer = HydraTrainer( model=parser.model, parser=parser, args=training_args, train_dataset=self.train_data, eval_dataset=self.dev_data, callbacks=[early_stopping], data_collator=data_collator, processing_class=self.pretrained_tokenizer, ) hf_trainer.train() if model_dir is not None: parser.save(model_dir) with open(f'{model_dir}/trainer.info', 'w') as f: print(f'dataset: {self.dataset}', file=f) print(f'tree_weight: {self.tree_weight}', file=f) print(f'batch_size: {self.batch_size}', file=f) print(f'epochs: {epochs}', file=f) print(f'lr: {lr}', file=f) return parser
# ------------------------------- EVALUATION ------------------------------- def _desegmentize( self, tokens: TokenList, segs: StrLabelList ) -> TokenList: new_tokens = [] for token, seg in zip(tokens, segs): if seg == SEG_BEGIN: new_tokens.append(token) else: new_tokens[-1] += token return new_tokens def _get_gold_data( self, dataset: Dataset, parser: Hydra, ) -> tuple[list[TokenList], list[LabelDict]]: HDR: list[str] = [] if self.lemmatisation: HDR.append(LEMMAS) if self.tagging: HDR.append(TAGS) if self.dependency: HDR += [HEADS, DEPRELS] if self.graph: HDR += [GRAPH_HEADS, GRAPH_DEPRELS] if self.hybrid: HDR += [SPINES, ANCHORS, ANCHOR_HS] if self.named_entities: HDR += [NE_IOBS] values = [ADJACENCY_MATRIX, GRAPH_ADJACENCY_MATRIX] gold = [] #feature_dict = { # c : { i : v for i, v in enumerate(get_feature_labels(dataset.features[c])) } # type: ignore[arg-type] # for c in self.categories if c not in values #} feature_dict = get_labels(dataset.features, self.categories) for sent in dataset: tokens = sent[TOKENS] if self.segmentation: tokens = self._desegmentize(tokens, [ feature_dict[SEGS][v] for v in sent[SEGS] if v != MASK_VALUE ]) try: assert(len(tokens) == len(sent[LEMMAS])) except: if len(sent[SEGS]) == 512: logger.warning('Sentence probably truncated due to > 512 BERT tokens – skipping:') logger.warning(f' ... {tokens[-16:]}') logger.warning(f' missing: ... {sent[LEMMAS][len(tokens):]}') continue else: logger.critical('Segmentation catastrophe!') logger.critical(tokens) logger.critical(sent[LEMMAS]) raise # take any key that corresponds to a masked column MASK_KEY = [h for h in HDR if h not in (LEMMAS, HEADS, GRAPH_HEADS, GRAPH_DEPRELS) and set(sent[h]) != {MASK_VALUE}][0] mask = [1 if t != MASK_VALUE else None for t in sent[MASK_KEY]] _lbls = {c : parser.select_unmasked(sent[c], mask, c) for c in self.categories} lbls: LabelDict = {} for c in self.categories: if c in values: lbls[c] = _lbls[c] # type: ignore[literal-required] elif c == GRAPH_DEPRELS: lbls[c] = [[feature_dict[c][v] if v != MASK_VALUE else None for v in row] for row in _lbls[c]] # type: ignore[index, union-attr] else: lbls[c] = [feature_dict[c][v] if v != MASK_VALUE else None for v in _lbls[c]] # type: ignore[literal-required, index] # mst choice doesn’t matter since adjacency matrix is well-formed 0-1 lbls = parser.process_labels(lbls, tokens, correct_lemmata=False, mst=MST, allow_masked=True) gold.append([tokens, lbls]) gold = sorted(gold, key=lambda x: (len(x[0]), ''.join(x[0]))) gold_tokens, gold_tags = map(list, zip(*gold)) return gold_tokens, gold_tags def _get_predicted_data_batch( self, gold_tokens: list[TokenList], parser: Hydra, mst: str = MST ) -> tuple[list[TokenList], list[LabelDict]]: pred: list[HydraAnalysis] = parser.parse([' '.join(toks) for toks in gold_tokens], return_labels=True, is_tokenized=True, mst=mst) # type: ignore[assignment] pred_tokens, pred_tags = [p['tokens'] for p in pred], [p['labels'] for p in pred] try: assert(pred_tokens == gold_tokens) except: for tp, tt in zip(pred_tokens, gold_tokens): if tp != tt: if len(tp) != len(tt): logger.critical('Tokens length mismatch!') logger.critical(tp) logger.critical(tt) raise logger.error('Tokens mismatch! (will evaluate anyway)') logger.error(f'gold: {tp}') logger.error(f'pred: {tt}') return pred_tokens, pred_tags def _get_predicted_data( self, gold_tokens: list[TokenList], parser: Hydra, mst: str = MST ) -> tuple[list[TokenList], list[LabelDict]]: #i, step = 0, 1024 processed = 0 pred_tokens, pred_tags = [], [] #while i * step <= len(gold_tokens): while processed < len(gold_tokens): #j, k = i * step, (i + 1) * step L = len(gold_tokens[processed]) batch_size = max(1, min(1024, round(16 * 1024 / L))) batch_size = min(batch_size, len(gold_tokens) - processed) j, k = processed, processed + batch_size logger.info(f'batch size: {batch_size} (instances {j} to {k})') tokens, tags = self._get_predicted_data_batch(gold_tokens[j:k], parser, mst=mst) pred_tokens += tokens pred_tags += tags processed = k #i += 1 return pred_tokens, pred_tags def _evaluate( self, parser: Hydra, tokens: list[TokenList], tags_true: list[LabelDict], tags_pred: list[LabelDict], continuous_only: bool = False, allow_masked: bool = False, ) -> dict[str, Any]: k = 0 PROBLEM_TREES = [] measures: list[EvaluationMetric] = [] raw_measures: list[EvaluationMetric] = [] set_seed(136) if self.dependency: measures += [UAS(), LAS(), Deprels(), DeprelsConfusionMatrix(), DepArcs(), DepArcsConfusionMatrix()] if self.graph: measures += [EUAS(), ELAS(), GUAS(), GLAS()] if self.tagging: measures.append(Tagging()) if self.lemmatisation: measures.append(Lemmatisation()) if self.lemmatisation and self.tagging: measures.append(TaggingCorr(parser.morfeusz)) measures.append(LemmatisationCorr(parser.morfeusz)) if self.hybrid: measures += [ HeadedSpans(), NonHeadedSpans(), UnlabeledSpans(), HeadedSpansDiscontinuous(), NonHeadedSpansDiscontinuous(), UnlabeledSpansDiscontinuous(), Edges()] raw_uas, raw_las = UAS(), LAS() raw_uas.name = 'raw UAS' raw_las.name = 'raw LAS' raw_measures = [raw_uas, raw_las] if self.named_entities: measures.append(NER()) for toks, true, pred in zip(tokens, tags_true, tags_pred): for measure in raw_measures: measure.update(toks, true, pred, None, None) if self.hybrid: #if allow_masked and MASK_VALUE in set(true[HEADS] + true[DEPRELS] + true[SPINES] + true[ANCHORS] + true[ANCHOR_HS]): if allow_masked and (MASK_VALUE in true[HEADS] or None in true[SPINES] + true[ANCHORS] + true[ANCHOR_HS]): tree_true, tree_pred = None, None else: tree_true, _, heads = make_tree( toks, true, self.nonterminal_features, root_label=self.root_label, force_root_label=self.force_root_label, decompress=True) assert heads == true[HEADS] tree_pred, info, heads = make_tree( toks, pred, self.nonterminal_features, root_label=self.root_label, force_root_label=self.force_root_label, decompress=True) # !!! pred[HEADS] = heads PROBLEM_TREES.append(info) else: tree_true, tree_pred = None, None if continuous_only and tree_true and not tree_true.is_continuous(): continue for measure in measures: measure.update(toks, true, pred, tree_true, tree_pred) result = {measure.name : measure.calculate() for measure in measures} for measure in raw_measures: result[measure.name] = measure.calculate() if PROBLEM_TREES: result['heuristics_used'] = { 'trees' : len(list(filter(None, PROBLEM_TREES))) / len(PROBLEM_TREES), 'stats' : Counter(chain.from_iterable(PROBLEM_TREES)) } return result
[docs] def evaluate( self, parser: Hydra, mst: str = MST, allow_masked: bool = False, dataset: DatasetDict | None = None ) -> EvalResults: """Perform evaluation of a trained Hydra model. Args: parser: the parser instance to be evaluated. mst: dependency tree decoding algoriths to be used. allow_masked: accept masked labels in gold data (but exclude them from metric computation). dataset: source of gold instances. Evaluation will be run on `'validation'` and any `'test*'` parts. Training dataset will be used as default if this argument is not provided. Returns: evaluation results. """ if dataset is None: dataset = self.dataset self._data_from_dataset(dataset) logger.info('Preparing gold data for evaluation...') GOLD_DATA = { part : self._get_gold_data(dataset[part], parser) for part in self.eval_parts } logger.info('Preparing predicted data for evaluation...') PRED_DATA = { part : self._get_predicted_data(GOLD_DATA[part][0], parser, mst=mst) for part in self.eval_parts } results = [] for part in self.eval_parts: logger.info(f'Evaluating on {part}...') tokens, tags_true, tags_pred = GOLD_DATA[part][0], GOLD_DATA[part][1], PRED_DATA[part][1] r = self._evaluate(parser, tokens, tags_true, tags_pred, allow_masked=allow_masked) results.append((part, r)) return results
[docs] def cross_validate( self, nfolds: int = 10, epochs: int = 10, patience: int = 4, lr: float = 0.00001, mst: str = MST, allow_masked: bool = False ) -> list[EvalResults]: """Perform 10-fold cross validation. Returns: results of each fold run. """ # TODO dataset as argument logger.info('running cross-validation...') cv_dataset = concatenate_datasets([instances for part, instances in sorted(self.dataset.items())]) N = len(cv_dataset) folds = [cv_dataset.shard(num_shards=nfolds, index=i, contiguous=True) for i in range(nfolds)] results = [] for i in range(nfolds): logger.info(f'FOLD #{i + 1}') train_ids = [j for j in range(nfolds) if j not in (i, (i + 1) % nfolds)] dev_id = i test_id = (i + 1) % nfolds dataset = DatasetDict({ 'train': concatenate_datasets([folds[j] for j in train_ids]), 'validation': folds[dev_id], 'test': folds[test_id], }) parser = self.train(epochs=epochs, patience=patience, lr=lr, log_dir=None, model_dir=None, dataset=dataset) results.append(self.evaluate(parser, mst=mst, allow_masked=allow_masked, dataset=dataset)) return results