Source code for hydra.utils.training

"""Training utilities: loss functions."""

import torch

from numpy import ndarray
from typing import Callable

from .constants import (
    MASK_VALUE,
    ADJACENCY_MATRIX,
    GRAPH_DEPRELS,
    GRAPH_ADJACENCY_MATRIX,
)

def _loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
    """Calculate the loss function for any per-token labels.
    
    Returns:
        the loss.
    
    """
    num_labels = logits.shape[-1]
    # logits: batch x seq_len x num_labels -> batch*seq_len x num_labels
    # labels: batch x seq_len -> batch*seq_len
    logits_reshaped = logits.view(-1, num_labels)
    labels_reshaped = labels.view(-1)
    if (labels_reshaped != -100).any():
        # cross_entropy will deal with all -100 labels (both all-row and token-masking)
        return torch.nn.functional.cross_entropy(logits_reshaped, labels_reshaped)
    else:
        # the whole batch was masked
        return torch.tensor(0.0, device=logits.device, requires_grad=True)

def _matrix_loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
    """Calculate the loss function for dependency tree adjacency matrix.
    
    Returns:
        the loss.
    
    """
    num_labels = logits.shape[-1]
    logits_reshaped = logits.view(-1, num_labels)
    labels_reshaped = labels.view(-1, num_labels)
    # we need to apply the mask here as CrossEntropyLoss from ground truth
    # probabilities doesn’t handle -100 mask
    row_mask = ~(labels_reshaped == MASK_VALUE).all(dim=1)
    if row_mask.any():
        logits_masked = logits_reshaped[row_mask]
        labels_masked = labels_reshaped[row_mask]
        # add -MASK_VALUE where labels == MASK_VALUE to replace it with zero
        # this will not affect the loss since its only non-zero component is for the true label
        labels_masked = (labels_masked + (labels_masked == MASK_VALUE) * -MASK_VALUE).float()
        return torch.nn.functional.cross_entropy(logits_masked, labels_masked)
    else:
        # the whole batch was masked
        return torch.tensor(0.0, device=logits.device, requires_grad=True)

def _graph_loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
    """Calculate the loss function for graph adjacency matrix.
    
    Returns:
        the loss.
    
    """
    # each value is an individual edge’s probability, so reshape them into vectors
    # to easily deal with all -100’s at once (both all-row and token-masking)
    logits_reshaped = logits.view(-1)
    labels_reshaped = labels.view(-1)
    # we need to apply the mask here as BCEWithLogitsLoss doesn’t handle -100 mask
    mask = ~(labels_reshaped == MASK_VALUE)
    logits_masked = logits_reshaped[mask]
    labels_masked = labels_reshaped[mask].float()
    return torch.nn.functional.binary_cross_entropy_with_logits(logits_masked, labels_masked)

def _graph_deprels_loss(logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
    """Calculate the loss function for graph edge labels.
    
    Returns:
        the loss.
    
    """
    # the same loss logic, just one more dimension will be flattened by view()!
    return _loss(logits, labels)

[docs] def get_loss(category: str) -> Callable[[torch.Tensor, torch.Tensor], torch.Tensor]: """Select a loss function according to category being predicted. Returns: a loss function. """ if category == ADJACENCY_MATRIX: return _matrix_loss if category == GRAPH_ADJACENCY_MATRIX: return _graph_loss if category == GRAPH_DEPRELS: return _graph_deprels_loss return _loss