Source code for hydra.utils.model

"""PyTorch models for Hydra are defined here."""

import logging
logger = logging.getLogger(__name__)

import torch
from torch import nn
from typing import Any, override

from transformers import AutoConfig, PretrainedConfig
from transformers.modeling_outputs import TokenClassifierOutput
from transformers.modeling_utils import PreTrainedModel
from transformers.utils.generic import can_return_tuple, TransformersKwargs

from .training import get_loss
from .constants import (
    ADJACENCY_MATRIX,
    GRAPH_DEPRELS,
    GRAPH_ADJACENCY_MATRIX,
)

[docs] class MultiTargetTokenClassification(PreTrainedModel): """Base class for Hydra PyTorch models.""" lm_layer_input_keys = ( 'input_ids', 'attention_mask', 'token_type_ids', 'position_ids', 'head_mask', 'inputs_embeds', 'output_attentions', 'output_hidden_states', 'return_dict', 'training', ) linear_size = 128 @property def lm_layer(self) -> nn.Module: """The underlying language model. Subclasses must implement this method. :raises NotImplementedError: Always. """ raise NotImplementedError
[docs] def set_loss_weights(self, loss_weights: dict[str, float]) -> None: """Assign weights to losses from particular model outputs.""" self.loss_weights = loss_weights
def __init__(self, config: PretrainedConfig, **kwargs: Any) -> None: logger.info(f'config.model_type: {config.model_type}') logger.info(f'self.config_class: {self.config_class}') categories = kwargs.pop('categories') labels = kwargs.pop('labels') logger.info(f'categories: {categories}') logger.info(f'labels: {self.config_class}') super().__init__(config) self.loss_weights = {} self.categories, self.categories2d, self.categories3d = [], [], [] for cat in categories: if cat == GRAPH_DEPRELS: self.categories3d.append(cat) elif cat in (ADJACENCY_MATRIX, GRAPH_ADJACENCY_MATRIX): self.categories2d.append(cat) else: self.categories.append(cat) classifier_dropout = ( config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob ) self.dropout = nn.Dropout(classifier_dropout) self.classifiers = nn.ModuleList([ nn.Linear(config.hidden_size, len(labels[cat])) for cat in self.categories ]) logger.info(f'created {len(self.classifiers)} classifier(s): {", ".join(self.categories)}') self.mappings2d = nn.ModuleList([ nn.ModuleDict({ 'head' : nn.Linear(config.hidden_size, self.linear_size), 'dependent' : nn.Linear(config.hidden_size, self.linear_size), }) for cat in self.categories2d ]) logger.info(f'created {len(self.mappings2d)} head/dependent mapping(s): {", ".join(self.categories2d)}') self.mappings3d = nn.ModuleList([ nn.ModuleDict({ 'head' : nn.Linear(config.hidden_size, self.linear_size), 'dependent' : nn.Linear(config.hidden_size, self.linear_size), 'classifier' : nn.Linear(2 * self.linear_size, len(labels[cat])), }) for cat in self.categories3d ]) logger.info(f'created {len(self.mappings3d)} head/dependent mapping(s) with classifier: {", ".join(self.categories3d)}') # Initialize weights and apply final processing self.post_init() # type: ignore[no-untyped-call] def _init_weights(self, module: nn.Module) -> None: super()._init_weights(module) # type: ignore[no-untyped-call] if isinstance(module, nn.Linear): is_custom_layer = ( any(module in c.modules() for c in self.classifiers) or any(module in m.modules() for m in self.mappings2d) or any(module in m.modules() for m in self.mappings3d) ) if is_custom_layer: torch.nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) if module.bias is not None: torch.nn.init.zeros_(module.bias)
[docs] def process_lm_output(self, sequence_output: torch.Tensor) -> torch.Tensor: """Perform any additional operations on the underlying language model’s output. Override in subclasses for specific behaviour. Returns: its input. """ # do nothing by default return sequence_output
#@can_return_tuple
[docs] def forward( self, input_ids: torch.Tensor | None = None, attention_mask: torch.Tensor | None = None, token_type_ids: torch.Tensor | None = None, position_ids: torch.Tensor | None = None, inputs_embeds: torch.Tensor | None = None, labels: torch.Tensor | None = None, **kwargs: Any, ) -> tuple[torch.Tensor] | TokenClassifierOutput: """Run a forward pass of the model. Returns: model outputs. """ outputs = self.lm_layer( input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, inputs_embeds=inputs_embeds, return_dict=True, **kwargs, ) sequence_output = outputs[0] sequence_output = self.process_lm_output(sequence_output) if torch.isnan(sequence_output).any(): 1/0 sequence_output = self.dropout(sequence_output) logits = [classifier(sequence_output) for classifier in self.classifiers] for mapping in self.mappings2d: heads = mapping['head'](sequence_output) # type: ignore[index] dependents = mapping['dependent'](sequence_output) # type: ignore[index] product = torch.einsum('bin, bjn -> bij', heads, dependents) logits.append(product) for mapping in self.mappings3d: seq_len = sequence_output.shape[1] hds = mapping['head'](sequence_output) # type: ignore[index] # batch x seq_len x linear_size dps = mapping['dependent'](sequence_output) # type: ignore[index] # batch x seq_len x linear_size h_expanded = hds.unsqueeze(dim=2).expand(-1, -1, seq_len, -1) # batch x seq_len x 1 x linear_size -> batch x seq_len x seq_len x linear_size d_expanded = dps.unsqueeze(dim=1).expand(-1, seq_len, -1, -1) # batch x 1 x seq_len x linear_size -> batch x seq_len x seq_len x linear_size concatenated = torch.cat([h_expanded, d_expanded], dim=-1) # batch x seq_len x seq_len x (linear_size*2) result = mapping['classifier'](concatenated) # type: ignore[index] logits.append(result) keys = self.categories + self.categories2d + self.categories3d losses = [] for cat, _logits in zip(keys, logits): if cat in kwargs: # ground truth labels were passed, so compute loss _labels = kwargs[cat] loss_fn = get_loss(cat) _loss = loss_fn(_logits, _labels) if torch.isnan(_loss).any(): print(cat, _loss) print(_logits) print(_labels) 1/0 weight = self.loss_weights.get(cat, 1.0) losses.append(weight * _loss) #1/0 return TokenClassifierOutput( loss=sum(losses), # type: ignore[arg-type] # expected FloatTensor, got Tensor logits=dict(zip(keys, logits)), # type: ignore[arg-type] # expected FloatTensor, got dict hidden_states=outputs.hidden_states, attentions=outputs.attentions, )
BERT = 'bert' ROBERTA = 'roberta' XLM_ROBERTA = 'xlm-roberta' MODERN_BERT = 'modernbert' T5 = 't5' MODEL_CLASSES: dict[str, type[MultiTargetTokenClassification]] = {} try: from transformers.models.bert.modeling_bert import BertPreTrainedModel, BertModel
[docs] class BertForMultiTargetTokenClassification(MultiTargetTokenClassification, BertPreTrainedModel): """A Hydra PyTorch model using a language model of BERT architecture.""" _keys_to_ignore_on_load_unexpected = [ r'bert\.pooler\..*', r'cls\.predictions\..*', r'cls\.sso\..*', ] @property @override def lm_layer(self) -> nn.Module: return self.bert def __init__( self, config: PretrainedConfig, *args: Any, **kwargs: Any, ) -> None: super().__init__(config, *args, **kwargs) # type: ignore[no-untyped-call] # this is the exact attribute name expected by the Hugging Face weights self.bert = BertModel(config, add_pooling_layer=False) # type: ignore[no-untyped-call] self.post_init() # type: ignore[no-untyped-call]
MODEL_CLASSES[BERT] = BertForMultiTargetTokenClassification except ImportError: logging.warning('Warning: couldn’t register BERT model type handler.') try: from transformers.models.roberta.modeling_roberta import RobertaPreTrainedModel, RobertaModel
[docs] class RobertaForMultiTargetTokenClassification(MultiTargetTokenClassification, RobertaPreTrainedModel): """A Hydra PyTorch model using a lanugage model of RoBERTa architecture.""" #_keys_to_ignore_on_load_unexpected = [ # r'pooler', # r'lm_head', # r'embeddings.position_ids', # r'dropout', # r'dot', #] @property @override def lm_layer(self) -> nn.Module: return self.roberta def __init__( self, config: PretrainedConfig, *args: Any, **kwargs: Any, ) -> None: super().__init__(config, *args, **kwargs) # type: ignore[no-untyped-call] # this is the exact attribute name expected by the Hugging Face weights self.roberta = RobertaModel(config, add_pooling_layer=False) # type: ignore[no-untyped-call] self.post_init() # type: ignore[no-untyped-call]
MODEL_CLASSES[ROBERTA] = RobertaForMultiTargetTokenClassification except ImportError: logger.warning('Warning: couldn’t register RoBERTa model type handler.') raise try: from transformers.models.xlm_roberta.modeling_xlm_roberta import XLMRobertaPreTrainedModel, XLMRobertaModel
[docs] class XLMRobertaForMultiTargetTokenClassification(MultiTargetTokenClassification, XLMRobertaPreTrainedModel): """A Hydra PyTorch model using a language model of XMLRoBERTa architecture.""" #_keys_to_ignore_on_load_unexpected = [ # r'pooler', # r'lm_head', # r'dropout', # r'dot', #] @property @override def lm_layer(self) -> nn.Module: return self.roberta # yup! it should be 'roberta'! def __init__( self, config: PretrainedConfig, *args: Any, **kwargs: Any, ) -> None: super().__init__(config, *args, **kwargs) # type: ignore[no-untyped-call] # this is the exact attribute name expected by the Hugging Face weights self.roberta = XLMRobertaModel(config, add_pooling_layer=False) # type: ignore[no-untyped-call] self.post_init() # type: ignore[no-untyped-call]
MODEL_CLASSES[XLM_ROBERTA] = XLMRobertaForMultiTargetTokenClassification except ImportError: logger.warning('Warning: couldn’t register XLM-RoBERTa model type handler.') raise try: from transformers.models.modernbert.modeling_modernbert import ModernBertPreTrainedModel, ModernBertModel, ModernBertPredictionHead from transformers.models.modernbert.configuration_modernbert import ModernBertConfig
[docs] class ModernBertForMultiTargetTokenClassification(MultiTargetTokenClassification, ModernBertPreTrainedModel): """A Hydra PyTorch model using a language model of ModernBert architecture.""" #_keys_to_ignore_on_load_unexpected = [ # r'pooler', # r'lm_head', # r'dropout', # r'dot', #] @property @override def lm_layer(self) -> nn.Module: return self.model def __init__( self, config: ModernBertConfig, *args: Any, **kwargs: Any, ) -> None: super().__init__(config, *args, **kwargs) # type: ignore[no-untyped-call] # this is the exact attribute name expected by the Hugging Face weights self.model: ModernBertModel = ModernBertModel(config) self.head = ModernBertPredictionHead(config) self.post_init() # type: ignore[no-untyped-call]
[docs] @override def process_lm_output(self, sequence_output: torch.Tensor) -> torch.Tensor: return self.head(sequence_output) # type: ignore[no-any-return]
MODEL_CLASSES[MODERN_BERT] = ModernBertForMultiTargetTokenClassification except ImportError: logger.warning('Warning: couldn’t register XLM-RoBERTa model type handler.') raise try: from transformers.models.t5.modeling_t5 import T5PreTrainedModel, T5EncoderModel from transformers.models.t5.configuration_t5 import T5Config
[docs] class T5ForMultiTargetTokenClassification(MultiTargetTokenClassification, T5PreTrainedModel): """A Hydra PyTorch model using a language model of T5 architecture.""" #_keys_to_ignore_on_load_unexpected = [ # r'decoder', # r'lm_head', # r'dropout', # r'dot' #] #lm_layer_input_keys = ( # 'input_ids', # 'attention_mask', # 'head_mask', # 'inputs_embeds', # 'output_attentions', # 'output_hidden_states', # 'return_dict', # 'training', #) @property @override def lm_layer(self) -> nn.Module: return self.transformer def __init__( self, config: T5Config, *args: Any, **kwargs: Any, ) -> None: super().__init__(config, *args, **kwargs) # type: ignore[no-untyped-call] # this is the exact attribute name expected by the Hugging Face weights self.transformer = T5EncoderModel(config) self.post_init() # type: ignore[no-untyped-call]
MODEL_CLASSES[T5] = T5ForMultiTargetTokenClassification except ImportError: logger.warning('Warning: couldn’t register T5 model type handler.') raise
[docs] def get_model_type(pretrained_path: str) -> str: """Retrieve the type of pretrained language model from its config. Returns: model type identifier (e.g. `'bert'`). """ return AutoConfig.from_pretrained(pretrained_path).model_type # type: ignore[no-any-return]
[docs] def get_model_class(model_type: str) -> type[MultiTargetTokenClassification]: """Provide a class for Hydra PyTorch model based on language model type. Returns: the class. :raises KeyError: for unsupported model types. """ if model_type in MODEL_CLASSES: return MODEL_CLASSES[model_type] raise KeyError(f'unsupported language model type: {model_type}')