Source code for hydra.utils.data_collator

"""Data collation utilities for models."""

from collections import defaultdict
from dataclasses import dataclass
from itertools import chain

from typing import override

import numpy as np

from datasets.features import ClassLabel, Features, List, Sequence

from transformers.data.data_collator import DataCollator, DataCollatorMixin, pad_without_fast_tokenizer_warning
from transformers.tokenization_utils_base import PreTrainedTokenizerBase
from transformers.utils.generic import PaddingStrategy

from .constants import MASK_VALUE

# based on transformers.data.data_collator.DataCollatorForTokenClassification
[docs] @dataclass class DataCollatorForHydra(DataCollatorMixin): """Handles padding for label sequences and matrices.""" tokenizer: PreTrainedTokenizerBase features: Features padding: bool | str | PaddingStrategy = True max_length: int | None = None pad_to_multiple_of: int | None = None label_pad_token_id: int = MASK_VALUE return_tensors: str = 'pt' def _pad_labels(self, labels: list[int], sequence_length: int) -> list[int]: if self.tokenizer.padding_side == 'right': return list(labels) + [MASK_VALUE] * (sequence_length - len(labels)) else: return [MASK_VALUE] * (sequence_length - len(labels)) + list(labels)
[docs] @override def torch_call(self, batch): # type: ignore[no-untyped-def] """Prepare the batch for passing to a torch model. Returns: a padded batch. """ import torch input_features = [{k: v for k, v in instance.items() if k in self.tokenizer.model_input_names} for instance in batch] padded_batch = pad_without_fast_tokenizer_warning( # type: ignore[no-untyped-call] self.tokenizer, input_features, padding=self.padding, max_length=self.max_length, pad_to_multiple_of=self.pad_to_multiple_of, return_tensors='pt', ) sequence_length = padded_batch['input_ids'].shape[1] for category, feat in self.features.items(): if isinstance(feat.feature, ClassLabel): padded_labels = [self._pad_labels(instance[category], sequence_length) for instance in batch] padded_batch[category] = torch.tensor(padded_labels, dtype=torch.long) elif isinstance(feat.feature, (Sequence, List)): # pad the matrix rows to sequence_length # and add empty rows to pad to obtain a sequence_length x sequence_length matrix nested_padded_labels = [ [ self._pad_labels(row, sequence_length) for row in instance[category] ] + [ self._pad_labels([], sequence_length) for _ in range(sequence_length - len(instance[category])) ] for instance in batch ] padded_batch[category] = torch.tensor(nested_padded_labels, dtype=torch.long) else: raise ValueError(f'Unsupported feature type {type(feat.feature)} for "{category}".') return padded_batch
[docs] @override def numpy_call(self, batch): # type: ignore[no-untyped-def] """Not to be used. :raises NotImplementedError: Always. """ raise NotImplementedError