"""Training utilities: loss functions."""
import torch
from numpy import ndarray
from typing import Callable
from .constants import (
MASK_VALUE,
ADJACENCY_MATRIX,
GRAPH_DEPRELS,
GRAPH_ADJACENCY_MATRIX,
)
def _loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
"""Calculate the loss function for any per-token labels.
Returns:
the loss.
"""
num_labels = logits.shape[-1]
# logits: batch x seq_len x num_labels -> batch*seq_len x num_labels
# labels: batch x seq_len -> batch*seq_len
logits_reshaped = logits.view(-1, num_labels)
labels_reshaped = labels.view(-1)
if (labels_reshaped != -100).any():
# cross_entropy will deal with all -100 labels (both all-row and token-masking)
return torch.nn.functional.cross_entropy(logits_reshaped, labels_reshaped)
else:
# the whole batch was masked
return torch.tensor(0.0, device=logits.device, requires_grad=True)
def _matrix_loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
"""Calculate the loss function for dependency tree adjacency matrix.
Returns:
the loss.
"""
num_labels = logits.shape[-1]
logits_reshaped = logits.view(-1, num_labels)
labels_reshaped = labels.view(-1, num_labels)
# we need to apply the mask here as CrossEntropyLoss from ground truth
# probabilities doesn’t handle -100 mask
row_mask = ~(labels_reshaped == MASK_VALUE).all(dim=1)
if row_mask.any():
logits_masked = logits_reshaped[row_mask]
labels_masked = labels_reshaped[row_mask]
# add -MASK_VALUE where labels == MASK_VALUE to replace it with zero
# this will not affect the loss since its only non-zero component is for the true label
labels_masked = (labels_masked + (labels_masked == MASK_VALUE) * -MASK_VALUE).float()
return torch.nn.functional.cross_entropy(logits_masked, labels_masked)
else:
# the whole batch was masked
return torch.tensor(0.0, device=logits.device, requires_grad=True)
def _graph_loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
"""Calculate the loss function for graph adjacency matrix.
Returns:
the loss.
"""
# each value is an individual edge’s probability, so reshape them into vectors
# to easily deal with all -100’s at once (both all-row and token-masking)
logits_reshaped = logits.view(-1)
labels_reshaped = labels.view(-1)
# we need to apply the mask here as BCEWithLogitsLoss doesn’t handle -100 mask
mask = ~(labels_reshaped == MASK_VALUE)
logits_masked = logits_reshaped[mask]
labels_masked = labels_reshaped[mask].float()
return torch.nn.functional.binary_cross_entropy_with_logits(logits_masked, labels_masked)
def _graph_deprels_loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
"""Calculate the loss function for graph edge labels.
Returns:
the loss.
"""
# the same loss logic, just one more dimension will be flattened by view()!
return _loss(logits, labels)
[docs]
def get_loss(category: str) -> Callable[[torch.Tensor, torch.Tensor], torch.Tensor]:
"""Select a loss function according to category being predicted.
Returns:
a loss function.
"""
if category == ADJACENCY_MATRIX:
return _matrix_loss
if category == GRAPH_ADJACENCY_MATRIX:
return _graph_loss
if category == GRAPH_DEPRELS:
return _graph_deprels_loss
return _loss