"""Implementation of the training and evaluation logic for Hydra parser."""
import os
import subprocess
from collections import Counter
from datetime import datetime
from itertools import chain
from typing import Any
import logging
logger = logging.getLogger(__name__)
import torch
from torch.utils.data import DataLoader
from datasets import Dataset, DatasetDict, concatenate_datasets
from datasets.features import Features
from transformers import (
set_seed,
AutoTokenizer,
EarlyStoppingCallback,
EvalPrediction,
TrainingArguments,
Trainer as HFTrainer,
)
from .parser import Hydra
from .utils.hydra import category_names, get_labels #get_feature_labels
from .utils.model import get_model_type, ROBERTA
from .utils.data_collator import DataCollatorForHydra
from .utils.dataset import (
get_dataset_features,
add_lemma_rules,
add_spines_and_attachments,
add_adjacency_matrix,
add_graph_matrices,
morfeusz_retokenize,
lm_tokenize_and_align,
)
from .utils.hybrid_tree import make_tree
from .utils.evaluation import (
EvaluationMetric,
UAS,
LAS,
Deprels,
DeprelsConfusionMatrix,
DepArcs,
DepArcsConfusionMatrix,
EUAS,
ELAS,
GUAS,
GLAS,
NER,
Tagging,
TaggingCorr,
Lemmatisation,
LemmatisationCorr,
HeadedSpans,
NonHeadedSpans,
UnlabeledSpans,
HeadedSpansDiscontinuous,
NonHeadedSpansDiscontinuous,
UnlabeledSpansDiscontinuous,
Edges,
)
from .utils.constants import (
FIRST,
MASK_VALUE,
TOKENS,
SEG_BEGIN,
SEG_INSIDE,
SEGS,
LEMMAS,
LEMMA_CASES,
LEMMA_RULES,
TAGS,
NONTERMINALS,
HEADS,
DEPRELS,
ADJACENCY_MATRIX,
SPINES,
ANCHORS,
ANCHOR_HS,
GRAPH_HEADS,
GRAPH_DEPRELS,
GRAPH_ADJACENCY_MATRIX,
NE_IOBS,
MST,
EAGER,
)
from .utils.types import TokenList, StrLabelList, LabelDict, HydraAnalysis, DatasetInstance, EvalResults
SPLIT = 'split'
DROP = 'drop'
[docs]
class ClassificationTask(object):
"""A Hydra training configuration."""
def __init__(
self,
name: str,
dataset: DatasetDict,
pretrained_path: str,
#model_type,
segmentation: bool = False,
lemmatisation: bool = False,
tagging: bool = False,
dependency: bool = False,
graph: bool = False,
hybrid: bool = False,
named_entities: bool = False,
root_label: str = 'ROOT',
force_root_label: bool = False,
morfeusz_dict_path: str | None = None,
morfeusz_dict_name: str | None = None,
nonterminal_features: list[str] | None = None,
tree_weight: float = 1.0,
batch_size: int = 16,
) -> None:
self.name = name
self.dataset = dataset
self.pretrained_path = pretrained_path
self.model_type = get_model_type(pretrained_path)
self.segmentation = segmentation
self.lemmatisation = lemmatisation
self.tagging = tagging
self.dependency = dependency
self.graph = graph
self.hybrid = hybrid
self.named_entities = named_entities
self.root_label = root_label
self.force_root_label = force_root_label
self.morfeusz_dict_path = morfeusz_dict_path
self.morfeusz_dict_name = morfeusz_dict_name
self.nonterminal_features = nonterminal_features if nonterminal_features is not None else []
self.tree_weight = tree_weight
self.batch_size = batch_size
[docs]
class Trainer(object):
"""Used for creating, training and evaluating instances of Hydra parser."""
def __init__(
self,
task: ClassificationTask,
) -> None:
self.segmentation = task.segmentation
self.lemmatisation = task.lemmatisation
self.tagging = task.tagging
self.dependency = task.dependency
self.graph = task.graph
self.hybrid = task.hybrid
self.nonterminal_features = task.nonterminal_features
self.named_entities = task.named_entities
self.root_label = task.root_label
self.force_root_label = task.force_root_label
self.pretrained_path = task.pretrained_path
self.model_type = task.model_type
self.masking_strategy = FIRST
self.morfeusz_dict_path = task.morfeusz_dict_path
self.morfeusz_dict_name = task.morfeusz_dict_name
self.categories = category_names(self.segmentation, self.lemmatisation, self.tagging, self.dependency, self.graph, self.hybrid, self.nonterminal_features, self.named_entities)
if not self.categories:
raise RuntimeError(f'Can’t train without objective(s)!')
#if long_sequences not in (SPLIT, DROP):
# raise ValueError(f'<long_sequences> must be one of: {SPLIT}, {DROP}')
dataset = task.dataset
for part in ('train', 'validation'):
if part not in task.dataset:
raise RuntimeError(f'Can’t train without "{part}" part in the dataset!')
dataset_features = get_dataset_features(dataset)
if self.lemmatisation:
if LEMMAS not in dataset_features:
raise RuntimeError(f'Can’t train for lemmatisation without "{LEMMAS}" column in the dataset!')
logger.info('Preprocessing the dataset for lemmatisation...')
dataset = add_lemma_rules(dataset)
if self.tagging:
if TAGS not in dataset_features:
raise RuntimeError(f'Can’t train for tagging without "{TAGS}" column in the dataset!')
column: str
if self.hybrid:
for column in (NONTERMINALS, HEADS, DEPRELS):
if column not in dataset_features:
raise RuntimeError(f'Can’t train for hybrid parsing without "{column}" column in the dataset!')
logger.info('Preprocessing the dataset for constituency...')
dataset = add_spines_and_attachments(dataset)
self.dependency = True
if self.dependency:
for column in (HEADS, DEPRELS):
if column not in dataset_features:
raise RuntimeError(f'Can’t train for dependency parsing without "{column}" column in the dataset!')
logger.info('Preprocessing the dataset for dependency parsing...')
dataset = add_adjacency_matrix(dataset)
if self.graph:
for column in (GRAPH_HEADS, GRAPH_DEPRELS):
if column not in dataset_features:
raise RuntimeError(f'Can’t train for graph parsing without "{column}" column in the dataset!')
logger.info('Preprocessing the dataset for graph parsing...')
dataset = add_graph_matrices(dataset)
if self.nonterminal_features:
if not self.hybrid:
raise RuntimeError(f'Can’t train for nonterminal features without training for hybrid parsing!')
for ntf in self.nonterminal_features:
if ntf not in dataset_features:
raise RuntimeError(f'Can’t train for "{ntf}" feature without "{ntf}" column in the dataset!')
if self.named_entities:
if NE_IOBS not in dataset_features:
raise RuntimeError(f'Can’t train for NER without "{NE_IOBS}" column in the dataset!')
cols_to_remove = [c for c in dataset_features if c not in self.categories + [TOKENS, LEMMAS, 'sent_id']]
logger.info(f'Removing columns: {", ".join(cols_to_remove)}')
dataset = dataset.remove_columns(cols_to_remove)
if self.segmentation:
logger.info('Preprocessing the dataset for segmentation...')
dataset = morfeusz_retokenize(
dataset,
morfeusz_dict_path=self.morfeusz_dict_path,
morfeusz_dict_name=self.morfeusz_dict_name)
#the features have changed, i.e. spines added
dataset_features = get_dataset_features(dataset)
self.features = Features({cat : dataset_features[cat] for cat in self.categories})
logger.info('Loading pretrained tokenizer...')
self.tokenizer_kwargs = {}
if self.model_type == ROBERTA:
self.tokenizer_kwargs['add_prefix_space'] = True
self.pretrained_tokenizer = AutoTokenizer.from_pretrained(self.pretrained_path, **self.tokenizer_kwargs)
logger.info('Preprocessing the dataset for LM...')
dataset = self._remove_long_sequences(dataset)
self.dataset = dataset.map(lambda x: lm_tokenize_and_align(x, self.pretrained_tokenizer, self.masking_strategy))
self.batch_size = task.batch_size
self.tree_weight = task.tree_weight
self.train_data = None
self.dev_data = None
self.eval_parts: list[str] = []
def _get_num_lm_tokens(self, instance : DatasetInstance) -> int:
l = self.pretrained_tokenizer(instance['tokens'], is_split_into_words=True, truncation=False, return_length=True)['length']
assert(len(l) == 1)
return l[0] # type: ignore[no-any-return]
def _remove_long_sequences(self, dataset : DatasetDict) -> DatasetDict:
logger.info(f'Removing instances with over {self.pretrained_tokenizer.model_max_length} LM tokens...')
return dataset.filter(lambda x: self._get_num_lm_tokens(x) <= self.pretrained_tokenizer.model_max_length)
def _data_from_dataset(
self,
dataset: DatasetDict
) -> None:
if dataset is None:
dataset = self.dataset
self.train_data = None
self.dev_data = None
self.eval_parts = []
for part, ds in dataset.items():
if part == 'train':
self.train_data = ds
else:
self.eval_parts.append(part)
if part == 'validation':
self.dev_data = ds
# ------------------------------- TRAINING -------------------------------
def _prepare_output_dir(self, path: str) -> None:
if not os.path.exists(path):
subprocess.call(f'mkdir {path}', shell=True)
else:
if not os.path.isdir(path):
raise ValueError(f'{path} is not a directory')
elif os.listdir(path):
logger.info(f'emptying {path}')
subprocess.call(f'rm -r {path}/*', shell=True)
[docs]
def train(
self,
epochs: int = 10,
patience: int = 4,
lr: float = 0.00001,
log_dir: str | None = None,
model_dir: str | None = None,
dataset: DatasetDict | None = None
) -> Hydra:
"""Create and train a parser.
Args:
epochs: number of epochs.
patience: how many epochs are allowed with no training progress. The best model across all epochs is then retrieved.
lr: learning rate.
log_dir: where to save training logs.
model_dir: where to save the final model.
dataset: training data. Must contain `'train'` and `'validation'` parts (used for training), with any number of `'test*'` parts for later evaluation.
Returns:
a parser instance.
"""
self._data_from_dataset(self.dataset)
assert (self.dev_data is not None)
if log_dir is not None:
self._prepare_output_dir(log_dir)
if model_dir is not None:
self._prepare_output_dir(model_dir)
set_seed(136)
parser = Hydra.create(
self.pretrained_path,
self.model_type,
self.features,
segmentation=self.segmentation,
lemmatisation=self.lemmatisation,
tagging=self.tagging,
dependency=self.dependency,
graph=self.graph,
hybrid=self.hybrid,
nonterminal_features=self.nonterminal_features,
named_entities=self.named_entities,
morfeusz_dict_path=self.morfeusz_dict_path,
morfeusz_dict_name=self.morfeusz_dict_name,
pretrained_tokenizer=self.pretrained_tokenizer,
tokenizer_kwargs=self.tokenizer_kwargs,
)
if self.tree_weight != 1:
loss_weights = {
cat : (self.tree_weight if cat in (SPINES, ANCHORS, ANCHOR_HS, DEPRELS, ADJACENCY_MATRIX, GRAPH_DEPRELS, GRAPH_ADJACENCY_MATRIX) else 1.0)
for cat in self.categories
}
parser.model.set_loss_weights(loss_weights)
early_stopping = EarlyStoppingCallback(
early_stopping_patience=patience,
early_stopping_threshold=0.0001,
)
training_args = TrainingArguments(
output_dir=log_dir,
label_names=self.categories,
num_train_epochs=epochs,
learning_rate=lr,
lr_scheduler_type='constant', # Replicates Keras constant LR behavior
bf16=True,
save_only_model=True,
logging_strategy='epoch',
eval_strategy='epoch',
save_strategy='best',
# only keep the best model after each epoch
save_total_limit=1,
# don’t clog the memory by accumulating eval tensors
eval_accumulation_steps=16,
dataloader_num_workers=0,
metric_for_best_model='avg',
load_best_model_at_end=True,
)
training_args.set_dataloader(train_batch_size=self.batch_size, eval_batch_size=self.batch_size)
data_collator = DataCollatorForHydra(tokenizer=self.pretrained_tokenizer, features=self.features)
TRAINER = self
class HydraTrainer(HFTrainer):
def __init__(
self,
*args: Any,
parser: Hydra,
**kwargs: Any,
) -> None:
super().__init__(*args, **kwargs)
self.parser = parser
def evaluate(self, eval_dataset=None, ignore_keys=None, metric_key_prefix: str = 'eval'):
eval_dataset = eval_dataset if eval_dataset is not None else self.eval_dataset
GOLD_DATA = TRAINER._get_gold_data(eval_dataset, self.parser, [])
PRED_DATA = TRAINER._get_predicted_data(GOLD_DATA[0], self.parser, mst=MST)
tokens, tags_true, tags_pred = GOLD_DATA[0], GOLD_DATA[1], PRED_DATA[1]
results = TRAINER._evaluate(self.parser, tokens, tags_true, tags_pred, allow_masked=True)
metrics = {}
if TRAINER.dependency:
metrics['uas'] = results['UAS']['result']
metrics['las'] = results['LAS']['result']
if TRAINER.graph:
#metrics['euas'] = results['EUAS']['result']['total']['F1']
#metrics['elas'] = results['ELAS']['result']['total']['F1']
metrics['guas'] = results['GUAS']['result']['total']['F1']
metrics['glas'] = results['GLAS']['result']['total']['F1']
if TRAINER.tagging:
metrics['tags_acc'] = results['tagging']['result']
if TRAINER.lemmatisation:
metrics['lemmas_acc'] = results['lemmatisation']['result']
if TRAINER.hybrid:
metrics['constit_f1'] = results['headed constit.']['result']['total']['F1']
metrics['edges_f1'] = results['edges']['result']['total']['F1']
if TRAINER.named_entities:
metrics['NER_f1'] = results['NER']['result']['total']['F1']
metrics = {k: 0.0 if v == 'N/A' else v for k, v in metrics.items()}
metrics['avg'] = sum(metrics.values()) / len(metrics)
metrics = {f'{metric_key_prefix}_{k}': v for k, v in metrics.items()}
self.log(metrics)
# store in state history (crucial for tracking/callbacks)
self.control = self.callback_handler.on_evaluate(
self.args, self.state, self.control, metrics
)
return metrics
hf_trainer = HydraTrainer(
model=parser.model,
parser=parser,
args=training_args,
train_dataset=self.train_data,
eval_dataset=self.dev_data,
callbacks=[early_stopping],
data_collator=data_collator,
processing_class=self.pretrained_tokenizer,
)
hf_trainer.train()
if model_dir is not None:
parser.save(model_dir)
with open(f'{model_dir}/trainer.info', 'w') as f:
print(f'dataset: {self.dataset}', file=f)
print(f'tree_weight: {self.tree_weight}', file=f)
print(f'batch_size: {self.batch_size}', file=f)
print(f'epochs: {epochs}', file=f)
print(f'lr: {lr}', file=f)
return parser
# ------------------------------- EVALUATION -------------------------------
def _desegmentize(
self,
tokens: TokenList,
segs: StrLabelList
) -> TokenList:
new_tokens = []
for token, seg in zip(tokens, segs):
if seg == SEG_BEGIN:
new_tokens.append(token)
else:
new_tokens[-1] += token
return new_tokens
def _get_gold_data(
self,
dataset: Dataset,
parser: Hydra,
) -> tuple[list[TokenList], list[LabelDict]]:
HDR: list[str] = []
if self.lemmatisation:
HDR.append(LEMMAS)
if self.tagging:
HDR.append(TAGS)
if self.dependency:
HDR += [HEADS, DEPRELS]
if self.graph:
HDR += [GRAPH_HEADS, GRAPH_DEPRELS]
if self.hybrid:
HDR += [SPINES, ANCHORS, ANCHOR_HS]
if self.named_entities:
HDR += [NE_IOBS]
values = [ADJACENCY_MATRIX, GRAPH_ADJACENCY_MATRIX]
gold = []
#feature_dict = {
# c : { i : v for i, v in enumerate(get_feature_labels(dataset.features[c])) } # type: ignore[arg-type]
# for c in self.categories if c not in values
#}
feature_dict = get_labels(dataset.features, self.categories)
for sent in dataset:
tokens = sent[TOKENS]
if self.segmentation:
tokens = self._desegmentize(tokens, [
feature_dict[SEGS][v]
for v in sent[SEGS] if v != MASK_VALUE
])
try:
assert(len(tokens) == len(sent[LEMMAS]))
except:
if len(sent[SEGS]) == 512:
logger.warning('Sentence probably truncated due to > 512 BERT tokens – skipping:')
logger.warning(f' ... {tokens[-16:]}')
logger.warning(f' missing: ... {sent[LEMMAS][len(tokens):]}')
continue
else:
logger.critical('Segmentation catastrophe!')
logger.critical(tokens)
logger.critical(sent[LEMMAS])
raise
# take any key that corresponds to a masked column
MASK_KEY = [h for h in HDR if h not in (LEMMAS, HEADS, GRAPH_HEADS, GRAPH_DEPRELS) and set(sent[h]) != {MASK_VALUE}][0]
mask = [1 if t != MASK_VALUE else None for t in sent[MASK_KEY]]
_lbls = {c : parser.select_unmasked(sent[c], mask, c) for c in self.categories}
lbls: LabelDict = {}
for c in self.categories:
if c in values:
lbls[c] = _lbls[c] # type: ignore[literal-required]
elif c == GRAPH_DEPRELS:
lbls[c] = [[feature_dict[c][v] if v != MASK_VALUE else None for v in row] for row in _lbls[c]] # type: ignore[index, union-attr]
else:
lbls[c] = [feature_dict[c][v] if v != MASK_VALUE else None for v in _lbls[c]] # type: ignore[literal-required, index]
# mst choice doesn’t matter since adjacency matrix is well-formed 0-1
lbls = parser.process_labels(lbls, tokens, correct_lemmata=False, mst=MST, allow_masked=True)
gold.append([tokens, lbls])
gold = sorted(gold, key=lambda x: (len(x[0]), ''.join(x[0])))
gold_tokens, gold_tags = map(list, zip(*gold))
return gold_tokens, gold_tags
def _get_predicted_data_batch(
self,
gold_tokens: list[TokenList],
parser: Hydra,
mst: str = MST
) -> tuple[list[TokenList], list[LabelDict]]:
pred: list[HydraAnalysis] = parser.parse([' '.join(toks) for toks in gold_tokens], return_labels=True, is_tokenized=True, mst=mst) # type: ignore[assignment]
pred_tokens, pred_tags = [p['tokens'] for p in pred], [p['labels'] for p in pred]
try:
assert(pred_tokens == gold_tokens)
except:
for tp, tt in zip(pred_tokens, gold_tokens):
if tp != tt:
if len(tp) != len(tt):
logger.critical('Tokens length mismatch!')
logger.critical(tp)
logger.critical(tt)
raise
logger.error('Tokens mismatch! (will evaluate anyway)')
logger.error(f'gold: {tp}')
logger.error(f'pred: {tt}')
return pred_tokens, pred_tags
def _get_predicted_data(
self,
gold_tokens: list[TokenList],
parser: Hydra,
mst: str = MST
) -> tuple[list[TokenList], list[LabelDict]]:
#i, step = 0, 1024
processed = 0
pred_tokens, pred_tags = [], []
#while i * step <= len(gold_tokens):
while processed < len(gold_tokens):
#j, k = i * step, (i + 1) * step
L = len(gold_tokens[processed])
batch_size = max(1, min(1024, round(16 * 1024 / L)))
batch_size = min(batch_size, len(gold_tokens) - processed)
j, k = processed, processed + batch_size
logger.info(f'batch size: {batch_size} (instances {j} to {k})')
tokens, tags = self._get_predicted_data_batch(gold_tokens[j:k], parser, mst=mst)
pred_tokens += tokens
pred_tags += tags
processed = k
#i += 1
return pred_tokens, pred_tags
def _evaluate(
self,
parser: Hydra,
tokens: list[TokenList],
tags_true: list[LabelDict],
tags_pred: list[LabelDict],
continuous_only: bool = False,
allow_masked: bool = False,
) -> dict[str, Any]:
k = 0
PROBLEM_TREES = []
measures: list[EvaluationMetric] = []
raw_measures: list[EvaluationMetric] = []
set_seed(136)
if self.dependency:
measures += [UAS(), LAS(), Deprels(), DeprelsConfusionMatrix(), DepArcs(), DepArcsConfusionMatrix()]
if self.graph:
measures += [EUAS(), ELAS(), GUAS(), GLAS()]
if self.tagging:
measures.append(Tagging())
if self.lemmatisation:
measures.append(Lemmatisation())
if self.lemmatisation and self.tagging:
measures.append(TaggingCorr(parser.morfeusz))
measures.append(LemmatisationCorr(parser.morfeusz))
if self.hybrid:
measures += [
HeadedSpans(), NonHeadedSpans(), UnlabeledSpans(),
HeadedSpansDiscontinuous(), NonHeadedSpansDiscontinuous(), UnlabeledSpansDiscontinuous(),
Edges()]
raw_uas, raw_las = UAS(), LAS()
raw_uas.name = 'raw UAS'
raw_las.name = 'raw LAS'
raw_measures = [raw_uas, raw_las]
if self.named_entities:
measures.append(NER())
for toks, true, pred in zip(tokens, tags_true, tags_pred):
for measure in raw_measures:
measure.update(toks, true, pred, None, None)
if self.hybrid:
#if allow_masked and MASK_VALUE in set(true[HEADS] + true[DEPRELS] + true[SPINES] + true[ANCHORS] + true[ANCHOR_HS]):
if allow_masked and (MASK_VALUE in true[HEADS] or None in true[SPINES] + true[ANCHORS] + true[ANCHOR_HS]):
tree_true, tree_pred = None, None
else:
tree_true, _, heads = make_tree(
toks, true, self.nonterminal_features,
root_label=self.root_label,
force_root_label=self.force_root_label,
decompress=True)
assert heads == true[HEADS]
tree_pred, info, heads = make_tree(
toks, pred, self.nonterminal_features,
root_label=self.root_label,
force_root_label=self.force_root_label,
decompress=True)
# !!!
pred[HEADS] = heads
PROBLEM_TREES.append(info)
else:
tree_true, tree_pred = None, None
if continuous_only and tree_true and not tree_true.is_continuous():
continue
for measure in measures:
measure.update(toks, true, pred, tree_true, tree_pred)
result = {measure.name : measure.calculate() for measure in measures}
for measure in raw_measures:
result[measure.name] = measure.calculate()
if PROBLEM_TREES:
result['heuristics_used'] = {
'trees' : len(list(filter(None, PROBLEM_TREES))) / len(PROBLEM_TREES),
'stats' : Counter(chain.from_iterable(PROBLEM_TREES))
}
return result
[docs]
def evaluate(
self,
parser: Hydra,
mst: str = MST,
allow_masked: bool = False,
dataset: DatasetDict | None = None
) -> EvalResults:
"""Perform evaluation of a trained Hydra model.
Args:
parser: the parser instance to be evaluated.
mst: dependency tree decoding algoriths to be used.
allow_masked: accept masked labels in gold data (but exclude them from metric computation).
dataset: source of gold instances. Evaluation will be run on `'validation'` and any `'test*'` parts. Training dataset will be used as default if this argument is not provided.
Returns:
evaluation results.
"""
if dataset is None:
dataset = self.dataset
self._data_from_dataset(dataset)
logger.info('Preparing gold data for evaluation...')
GOLD_DATA = { part : self._get_gold_data(dataset[part], parser) for part in self.eval_parts }
logger.info('Preparing predicted data for evaluation...')
PRED_DATA = { part : self._get_predicted_data(GOLD_DATA[part][0], parser, mst=mst) for part in self.eval_parts }
results = []
for part in self.eval_parts:
logger.info(f'Evaluating on {part}...')
tokens, tags_true, tags_pred = GOLD_DATA[part][0], GOLD_DATA[part][1], PRED_DATA[part][1]
r = self._evaluate(parser, tokens, tags_true, tags_pred, allow_masked=allow_masked)
results.append((part, r))
return results
[docs]
def cross_validate(
self,
nfolds: int = 10,
epochs: int = 10,
patience: int = 4,
lr: float = 0.00001,
mst: str = MST,
allow_masked: bool = False
) -> list[EvalResults]:
"""Perform 10-fold cross validation.
Returns:
results of each fold run.
"""
# TODO dataset as argument
logger.info('running cross-validation...')
cv_dataset = concatenate_datasets([instances for part, instances in sorted(self.dataset.items())])
N = len(cv_dataset)
folds = [cv_dataset.shard(num_shards=nfolds, index=i, contiguous=True) for i in range(nfolds)]
results = []
for i in range(nfolds):
logger.info(f'FOLD #{i + 1}')
train_ids = [j for j in range(nfolds) if j not in (i, (i + 1) % nfolds)]
dev_id = i
test_id = (i + 1) % nfolds
dataset = DatasetDict({
'train': concatenate_datasets([folds[j] for j in train_ids]),
'validation': folds[dev_id],
'test': folds[test_id],
})
parser = self.train(epochs=epochs, patience=patience, lr=lr, log_dir=None, model_dir=None, dataset=dataset)
results.append(self.evaluate(parser, mst=mst, allow_masked=allow_masked, dataset=dataset))
return results