"""PyTorch models for Hydra are defined here."""
import logging
logger = logging.getLogger(__name__)
import torch
from torch import nn
from typing import Any, override
from transformers import AutoConfig, PretrainedConfig
from transformers.modeling_outputs import TokenClassifierOutput
from transformers.modeling_utils import PreTrainedModel
from transformers.utils.generic import can_return_tuple, TransformersKwargs
from .training import get_loss
from .constants import (
ADJACENCY_MATRIX,
GRAPH_DEPRELS,
GRAPH_ADJACENCY_MATRIX,
)
[docs]
class MultiTargetTokenClassification(PreTrainedModel):
"""Base class for Hydra PyTorch models."""
lm_layer_input_keys = (
'input_ids',
'attention_mask',
'token_type_ids',
'position_ids',
'head_mask',
'inputs_embeds',
'output_attentions',
'output_hidden_states',
'return_dict',
'training',
)
linear_size = 128
@property
def lm_layer(self) -> nn.Module:
"""The underlying language model.
Subclasses must implement this method.
:raises NotImplementedError: Always.
"""
raise NotImplementedError
[docs]
def set_loss_weights(self, loss_weights: dict[str, float]) -> None:
"""Assign weights to losses from particular model outputs."""
self.loss_weights = loss_weights
def __init__(self, config: PretrainedConfig, **kwargs: Any) -> None:
logger.info(f'config.model_type: {config.model_type}')
logger.info(f'self.config_class: {self.config_class}')
categories = kwargs.pop('categories')
labels = kwargs.pop('labels')
logger.info(f'categories: {categories}')
logger.info(f'labels: {self.config_class}')
super().__init__(config)
self.loss_weights = {}
self.categories, self.categories2d, self.categories3d = [], [], []
for cat in categories:
if cat == GRAPH_DEPRELS:
self.categories3d.append(cat)
elif cat in (ADJACENCY_MATRIX, GRAPH_ADJACENCY_MATRIX):
self.categories2d.append(cat)
else:
self.categories.append(cat)
classifier_dropout = (
config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
)
self.dropout = nn.Dropout(classifier_dropout)
self.classifiers = nn.ModuleList([
nn.Linear(config.hidden_size, len(labels[cat])) for cat in self.categories
])
logger.info(f'created {len(self.classifiers)} classifier(s): {", ".join(self.categories)}')
self.mappings2d = nn.ModuleList([
nn.ModuleDict({
'head' : nn.Linear(config.hidden_size, self.linear_size),
'dependent' : nn.Linear(config.hidden_size, self.linear_size),
}) for cat in self.categories2d
])
logger.info(f'created {len(self.mappings2d)} head/dependent mapping(s): {", ".join(self.categories2d)}')
self.mappings3d = nn.ModuleList([
nn.ModuleDict({
'head' : nn.Linear(config.hidden_size, self.linear_size),
'dependent' : nn.Linear(config.hidden_size, self.linear_size),
'classifier' : nn.Linear(2 * self.linear_size, len(labels[cat])),
}) for cat in self.categories3d
])
logger.info(f'created {len(self.mappings3d)} head/dependent mapping(s) with classifier: {", ".join(self.categories3d)}')
# Initialize weights and apply final processing
self.post_init() # type: ignore[no-untyped-call]
def _init_weights(self, module: nn.Module) -> None:
super()._init_weights(module) # type: ignore[no-untyped-call]
if isinstance(module, nn.Linear):
is_custom_layer = (
any(module in c.modules() for c in self.classifiers) or
any(module in m.modules() for m in self.mappings2d) or
any(module in m.modules() for m in self.mappings3d)
)
if is_custom_layer:
torch.nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
if module.bias is not None:
torch.nn.init.zeros_(module.bias)
[docs]
def process_lm_output(self, sequence_output: torch.Tensor) -> torch.Tensor:
"""Perform any additional operations on the underlying language model’s output.
Override in subclasses for specific behaviour.
Returns:
its input.
"""
# do nothing by default
return sequence_output
#@can_return_tuple
[docs]
def forward(
self,
input_ids: torch.Tensor | None = None,
attention_mask: torch.Tensor | None = None,
token_type_ids: torch.Tensor | None = None,
position_ids: torch.Tensor | None = None,
inputs_embeds: torch.Tensor | None = None,
labels: torch.Tensor | None = None,
**kwargs: Any,
) -> tuple[torch.Tensor] | TokenClassifierOutput:
"""Run a forward pass of the model.
Returns:
model outputs.
"""
outputs = self.lm_layer(
input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
inputs_embeds=inputs_embeds,
return_dict=True,
**kwargs,
)
sequence_output = outputs[0]
sequence_output = self.process_lm_output(sequence_output)
if torch.isnan(sequence_output).any():
1/0
sequence_output = self.dropout(sequence_output)
logits = [classifier(sequence_output) for classifier in self.classifiers]
for mapping in self.mappings2d:
heads = mapping['head'](sequence_output) # type: ignore[index]
dependents = mapping['dependent'](sequence_output) # type: ignore[index]
product = torch.einsum('bin, bjn -> bij', heads, dependents)
logits.append(product)
for mapping in self.mappings3d:
seq_len = sequence_output.shape[1]
hds = mapping['head'](sequence_output) # type: ignore[index] # batch x seq_len x linear_size
dps = mapping['dependent'](sequence_output) # type: ignore[index] # batch x seq_len x linear_size
h_expanded = hds.unsqueeze(dim=2).expand(-1, -1, seq_len, -1) # batch x seq_len x 1 x linear_size -> batch x seq_len x seq_len x linear_size
d_expanded = dps.unsqueeze(dim=1).expand(-1, seq_len, -1, -1) # batch x 1 x seq_len x linear_size -> batch x seq_len x seq_len x linear_size
concatenated = torch.cat([h_expanded, d_expanded], dim=-1) # batch x seq_len x seq_len x (linear_size*2)
result = mapping['classifier'](concatenated) # type: ignore[index]
logits.append(result)
keys = self.categories + self.categories2d + self.categories3d
losses = []
for cat, _logits in zip(keys, logits):
if cat in kwargs:
# ground truth labels were passed, so compute loss
_labels = kwargs[cat]
loss_fn = get_loss(cat)
_loss = loss_fn(_logits, _labels)
if torch.isnan(_loss).any():
print(cat, _loss)
print(_logits)
print(_labels)
1/0
weight = self.loss_weights.get(cat, 1.0)
losses.append(weight * _loss)
#1/0
return TokenClassifierOutput(
loss=sum(losses), # type: ignore[arg-type] # expected FloatTensor, got Tensor
logits=dict(zip(keys, logits)), # type: ignore[arg-type] # expected FloatTensor, got dict
hidden_states=outputs.hidden_states,
attentions=outputs.attentions,
)
BERT = 'bert'
ROBERTA = 'roberta'
XLM_ROBERTA = 'xlm-roberta'
MODERN_BERT = 'modernbert'
T5 = 't5'
MODEL_CLASSES: dict[str, type[MultiTargetTokenClassification]] = {}
try:
from transformers.models.bert.modeling_bert import BertPreTrainedModel, BertModel
MODEL_CLASSES[BERT] = BertForMultiTargetTokenClassification
except ImportError:
logging.warning('Warning: couldn’t register BERT model type handler.')
try:
from transformers.models.roberta.modeling_roberta import RobertaPreTrainedModel, RobertaModel
MODEL_CLASSES[ROBERTA] = RobertaForMultiTargetTokenClassification
except ImportError:
logger.warning('Warning: couldn’t register RoBERTa model type handler.')
raise
try:
from transformers.models.xlm_roberta.modeling_xlm_roberta import XLMRobertaPreTrainedModel, XLMRobertaModel
MODEL_CLASSES[XLM_ROBERTA] = XLMRobertaForMultiTargetTokenClassification
except ImportError:
logger.warning('Warning: couldn’t register XLM-RoBERTa model type handler.')
raise
try:
from transformers.models.modernbert.modeling_modernbert import ModernBertPreTrainedModel, ModernBertModel, ModernBertPredictionHead
from transformers.models.modernbert.configuration_modernbert import ModernBertConfig
MODEL_CLASSES[MODERN_BERT] = ModernBertForMultiTargetTokenClassification
except ImportError:
logger.warning('Warning: couldn’t register XLM-RoBERTa model type handler.')
raise
try:
from transformers.models.t5.modeling_t5 import T5PreTrainedModel, T5EncoderModel
from transformers.models.t5.configuration_t5 import T5Config
MODEL_CLASSES[T5] = T5ForMultiTargetTokenClassification
except ImportError:
logger.warning('Warning: couldn’t register T5 model type handler.')
raise
[docs]
def get_model_type(pretrained_path: str) -> str:
"""Retrieve the type of pretrained language model from its config.
Returns:
model type identifier (e.g. `'bert'`).
"""
return AutoConfig.from_pretrained(pretrained_path).model_type # type: ignore[no-any-return]
[docs]
def get_model_class(model_type: str) -> type[MultiTargetTokenClassification]:
"""Provide a class for Hydra PyTorch model based on language model type.
Returns:
the class.
:raises KeyError: for unsupported model types.
"""
if model_type in MODEL_CLASSES:
return MODEL_CLASSES[model_type]
raise KeyError(f'unsupported language model type: {model_type}')