"""Dataset manipulation utilities."""
from collections import Counter, defaultdict
from itertools import chain
import logging
logger = logging.getLogger(__name__)
from datasets import ClassLabel, DatasetDict, Features, Sequence
from transformers import BatchEncoding, PreTrainedTokenizerBase
from morfeusz2 import Morfeusz
from .hybrid_tree import tree_from_dataset_instance, TreeNode
from .constants import (
FIRST,
LAST,
MASK_VALUE,
EMPTY,
ALL_UPPER,
ALL_LOWER,
FIRST_UPPER,
KEEP,
SEG_BEGIN,
SEG_INSIDE,
TOKENS,
SEGS,
LEMMAS,
LEMMA_CASES,
LEMMA_RULES,
TAGS,
SPINES,
ANCHORS,
ANCHOR_HS,
HEADS,
ADJACENCY_MATRIX,
GRAPH_HEADS,
GRAPH_DEPRELS,
GRAPH_ADJACENCY_MATRIX,
)
from .types import TokenList, StrLabelList, SimpleLabelList, AnyLabelList, DatasetInstance
from IPython.display import display
# assuming all parts have identical features!
[docs]
def get_dataset_features(dataset: DatasetDict) -> Features:
"""Get the dataset’s features signature.
Returns:
the features.
"""
any_key = list(dataset.keys())[0]
return dataset[any_key].features
def _none_to_mask(instance: DatasetInstance, columns: list[str]) -> DatasetInstance:
ret: DatasetInstance = {}
for column in columns:
ret[column] = [v if v is not None else MASK_VALUE for v in instance[column]] # type: ignore[literal-required]
return ret
[docs]
def cast_labels(dataset: DatasetDict, columns: list[str]) -> DatasetDict:
"""Cast the provided columns from `datasets.Value` into `datasets.ClassLabel`, updating the features signature.
Returns:
the dataset with cast columns.
"""
vals: dict[str, set[str]] = defaultdict(set)
for d in dataset.values():
for column in columns:
vals[column].update(chain.from_iterable(s[column] for s in d))
new_features = get_dataset_features(dataset).copy()
for column in columns:
vals[column].discard(None)
new_features[column] = Sequence(ClassLabel(names=sorted(vals[column])))
# labels to indices
cast_dataset = dataset.cast(new_features)
# replace None with MASK_VALUE
return cast_dataset.map(lambda instance: _none_to_mask(instance, columns))
def _get_case_pattern(word: str) -> str:
"""Generate an upper/lowercase pattern ot the given string. The pattern truncates any trailing same-case characters.
e.g.: `kotek` → `xxxxx` → `x`, `Anna` → `Xxxx` → `Xx`, `ZOMO` → `XXXX` → `X`, `UOKiK` → `XXXxX` → `XXXxX`.
Returns:
the pattern.
"""
c = [char.isupper() for char in word]
while len(c) > 1 and c[-1] == c[-2]:
c = c[:-1]
return ''.join('X' if x else 'x' for x in c)
def _get_lemma_case_pattern(form: str, lemma: str) -> str:
"""Generate a case pattern for the lemma: all lowercase, all uppercase, first uppercase or follow the form.
Returns:
the pattern.
"""
cf = _get_case_pattern(form)
cl = _get_case_pattern(lemma)
if cl in (ALL_UPPER, ALL_LOWER, FIRST_UPPER):
return cl
elif cl == cf or form.startswith(lemma):
return KEEP
# fallbacks
elif cl[0] == 'x':
return ALL_LOWER
else:
return FIRST_UPPER
def _make_lemma_rule(token: str, lemma: str, tag: str) -> tuple[str, str]:
"""Generate a rule for transforming the token form into a lemma in the form of `<prefix_cut>_<cut>_<suffix>`.
e.g. `'3_7_y'` (remove first 3 characters, remove last 2 characters, append `y`) for `'najpiękniejszej'` → `'piękny'`.
Returns:
the pattern.
"""
case = _get_lemma_case_pattern(token, lemma)
prefix_cut = 0
token, lemma = token.lower(), lemma.lower()
#if lemma.startswith('naj') and or
if (token.startswith('nie') and 'neg' in tag) or (token.startswith('naj') and 'sup' in tag):
prefix_cut = 3
token = token[3:]
cut = 0
while token[:(cut + 1)] == lemma[:(cut + 1)] and cut < len(token):
cut += 1
suffix = lemma[cut:]
cut = len(token) - cut
return case, f'{prefix_cut}_{cut}_{suffix}'
def _add_lemma_rules(instance: DatasetInstance, tag_dict: dict[int, str]) -> DatasetInstance:
tokens = instance[TOKENS]
lemmas = instance[LEMMAS]
tags = [tag_dict[v] for v in instance[TAGS]] # type: ignore [index] # all v’s are not None
cases, rules = map(list, zip(*(_make_lemma_rule(*x) for x in zip(tokens, lemmas, tags)))) # type: ignore [arg-type]
return {
LEMMA_CASES : cases,
LEMMA_RULES : rules,
}
[docs]
def add_lemma_rules(dataset: DatasetDict) -> DatasetDict:
"""Add dataset columns used to train the lemmatisation module.
Returns:
a dataset with columns added.
"""
tag_dict = get_dataset_features(dataset)[TAGS].feature.names
new_dataset = dataset.map(lambda instance: _add_lemma_rules(instance, tag_dict))
return cast_labels(new_dataset, [LEMMA_CASES, LEMMA_RULES])
def _do_collect_spines(tree: TreeNode) -> tuple[list[TreeNode], list[tuple[str, int, list[TreeNode]]]]:
if not tree.children:
return [tree], []
heads = [child for child in tree.children if child.is_head]
assert(len(heads) == 1)
head = heads[0]
paths = []
my_path = [tree]
non_heads = []
for child in tree.children:
child_path, grandchildren_paths = _do_collect_spines(child)
paths += grandchildren_paths
if child == head:
my_path += child_path
else:
non_heads.append(child_path)
for child_path in non_heads:
# h == which <tree.category> counting from the bottom is the anchor
assert(tree.category is not None)
h = [n.category for n in my_path].count(tree.category)
paths.append((tree.category, h, child_path))
return my_path, paths
def _collect_spines(tree: TreeNode) -> dict[TreeNode, tuple[str, int, list[TreeNode]]]:
try:
path, paths = _do_collect_spines(tree)
except:
logger.critical('Couldn’t colllect spines:')
logger.critical(tree.to_brackets())
raise
return {p[-1] : (anchor, h, p[:-1]) for anchor, h, p in [('<ROOT>', -1, path)] + paths}
def _compress_spine(spine: list[str]) -> list[str]:
compressed: list[str] = []
for category in spine:
if category in compressed:
assert(category == compressed[-1])
else:
compressed.append(category)
return compressed
def _add_spines_and_attachments(instance: DatasetInstance, dataset_features: Features, compress: bool) -> DatasetInstance:
tree = tree_from_dataset_instance(instance, dataset_features)
if tree is None:
return {
SPINES : [None for _ in instance[TOKENS]], # type: ignore[typeddict-item]
ANCHORS : [None for _ in instance[TOKENS]], # type: ignore[typeddict-item]
ANCHOR_HS : [None for _ in instance[TOKENS]], # type: ignore[typeddict-item]
}
spines = _collect_spines(tree)
leafs_linear = sorted(tree.get_yield(), key=lambda leaf: leaf.from_index) # type: ignore[arg-type, return-value] # all from_index’s are not None
rows: list[tuple[str, str, str]] = []
for leaf in leafs_linear:
anchor, anchor_h, spine = spines[leaf]
spine_cats: list[str] = [node.category for node in spine] # type: ignore[misc] # all category’s are not None
if compress:
spine_cats = _compress_spine(spine_cats)
spine_label = '_'.join(spine_cats) if spine_cats else EMPTY
rows.append((spine_label, anchor, str(anchor_h) if anchor_h != -1 else '<ROOT>'))
spine_labels, anchors, anchor_hs = map(list, zip(*rows))
return {
SPINES : spine_labels,
ANCHORS : anchors,
ANCHOR_HS : anchor_hs,
}
[docs]
def add_spines_and_attachments(dataset: DatasetDict, compress: bool = False) -> DatasetDict:
"""Add dataset columns used to train the hybrid syntactic parsing module.
Returns:
a dataset istance with columns added.
"""
dataset_features = get_dataset_features(dataset)
new_dataset = dataset.map(
lambda instance: _add_spines_and_attachments(instance, dataset_features, compress=compress))
return cast_labels(new_dataset, [SPINES, ANCHORS, ANCHOR_HS])
EDGE, NO_EDGE = 1, 0
def _add_adjacency_matrix(instance: DatasetInstance) -> DatasetInstance:
heads = instance[HEADS]
if set(heads) == {MASK_VALUE}:
return {ADJACENCY_MATRIX : [[MASK_VALUE for j in range(len(heads))] for i in range(len(heads))]}
# ROOT is ‘it’s own’ head
int_heads: list[int] = [x if x is not None else i for i, x in enumerate(heads)]
am = [[NO_EDGE for j in range(len(heads))] for i in range(len(heads))]
for i, head in enumerate(int_heads):
am[i][head] = EDGE
return {ADJACENCY_MATRIX : am} # type: ignore[typeddict-item]
[docs]
def add_adjacency_matrix(dataset: DatasetDict) -> DatasetDict:
"""Add dataset column used to train the dependency parsing module.
Returns:
a dataset istance with columns added.
"""
return dataset.map(lambda instance: _add_adjacency_matrix(instance))
def _add_graph_matrices(instance: DatasetInstance) -> DatasetInstance:
heads = instance[GRAPH_HEADS]
deprels = instance[GRAPH_DEPRELS]
N = len(heads)
if set(chain.from_iterable(heads)) == {MASK_VALUE}:
return {
GRAPH_ADJACENCY_MATRIX : [[MASK_VALUE for j in range(N)] for i in range(N)],
GRAPH_DEPRELS : [[MASK_VALUE for j in range(N)] for i in range(N)],
}
# ROOT is ‘it’s own’ head
am = [[NO_EDGE for j in range(N)] for i in range(N)]
# deprels: mask unless there is an edge
dm = [[MASK_VALUE for j in range(N)] for i in range(N)]
for i, (graph_heads, graph_deprels) in enumerate(zip(heads, deprels)):
for head, deprel in zip(graph_heads, graph_deprels):
if head is None:
head = i
am[i][head] = EDGE
dm[i][head] = deprel # type: ignore[assignment]
return {GRAPH_ADJACENCY_MATRIX : am, GRAPH_DEPRELS : dm} # type: ignore[typeddict-item]
[docs]
def add_graph_matrices(dataset: DatasetDict) -> DatasetDict:
"""Add dataset columns used to train the graph parsing module.
Returns:
a dataset istance with columns added.
"""
return dataset.map(lambda instance: _add_graph_matrices(instance))
# https://huggingface.co/docs/transformers/v4.23.1/en/tasks/token_classification
[docs]
def masked_word_ids(word_ids: list[int | None], masking_strategy: str = FIRST) -> list[int | None]:
"""Mask positions of multiple language model tokens corresponding to single input segment (word).
This is needed to properly align training labels (one per word) with tokens passed to the language model.
Args:
word_ids: indices of input words from which the language model tokens come.
masking_strategy: :const:`~hydra.utils.constants.FIRST` (only leave the first token unmasked for each word) or :const:`~hydra.utils.constants.LAST` (only leave the last token unmasked for each word).
Returns:
a list of the same length as `word_ids`, with masked postions repleced with `None`.
"""
masked: list[int | None] = []
for i, word_idx in enumerate(word_ids):
# Set the label for the first/last token of each word.
# Mask the label for:
# * special tokens (word id = None)
# * other tokens in a word
if word_idx is None:
masked.append(None)
else:
if masking_strategy == FIRST:
masked.append(word_idx if word_idx != word_ids[i - 1] else None)
elif masking_strategy == LAST:
masked.append(word_idx if word_idx != word_ids[i + 1] else None)
return masked
def _align_row(values: SimpleLabelList, masked_word_ids: list[int | None]) -> SimpleLabelList: # in fact, it’s an IntLabelList
try:
return [MASK_VALUE if idx is None else values[idx] for idx in masked_word_ids] # type: ignore[return-value]
except:
logger.critical('Couldn’t align row:')
logger.critical(f'values: {values}')
logger.critical(f'masked_word_ids: {masked_word_ids}')
raise
def _align_example(example: DatasetInstance, masked_ids: list[int | None]) -> DatasetInstance:
labels: DatasetInstance = {}
masked_row = [MASK_VALUE for x in masked_ids]
try:
for column_name, values in example.items():
if column_name in (TOKENS, LEMMAS):
continue
if type(values) == str:
continue
matrix = hasattr(values[0], '__iter__') # type: ignore[index] # [*] because of the items() iteration, mypy complains about values’ type
aligned_labels: AnyLabelList
if matrix:
aligned_labels = [_align_row(values[idx], masked_ids) if idx is not None else masked_row for idx in masked_ids] # type: ignore[index, assignment] # [*]
else:
aligned_labels = _align_row(values, masked_ids) # type: ignore[arg-type] # [*]
labels[column_name] = aligned_labels # type: ignore[literal-required]
except:
logger.critical('Couldn’t align example:')
logger.critical(f'example: {example}')
logger.critical(f'column_name: {column_name}')
logger.critical(f'example[column_name]: {values}')
logger.critical(f'masked_ids: masked_ids')
raise
return labels
[docs]
def morf_tokenize(text: str, m: Morfeusz) -> TokenList:
"""Tokenize the text using Morfeusz. The most fine-grained tokenisation is retrieved from the DAG.
Returns:
the tokenised text.
"""
segs: dict[tuple[int, int], str] = {}
max_j = 0
for i, j, interp in m.analyse(text):
orth = interp[0]
if (i, j) in segs:
assert (orth == segs[(i, j)])
else:
segs[(i, j)] = orth
max_j = max(max_j, j)
# Morfeusz DAG vertices <-> positions in the text
dag_to_text_idx = {0 : 0}
for (i, j), seg in sorted(segs.items()):
assert(i in dag_to_text_idx)
jdx = dag_to_text_idx[i] + len(seg)
if j in dag_to_text_idx:
assert(dag_to_text_idx[j] == jdx)
else:
dag_to_text_idx[j] = jdx
for (i, j), seg in sorted(segs.items()):
try:
assert(seg == text[dag_to_text_idx[i]:dag_to_text_idx[j]])
except:
logger.error(f'Couldn’t get Morfeusz tokenisation: for ‘{text}’')
logger.error(m.analyse(text))
logger.error(segs)
logger.error(dag_to_text_idx)
logger.error('Leaving this segment untokenised')
return [text]
try:
# return [segs[(i, i + 1)] for i in range(max_j)]
return [text[dag_to_text_idx[i]:dag_to_text_idx[i + 1]] for i in range(max_j)]
except:
logger.error(f'Couldn’t get Morfeusz tokenisation: for ‘{text}’')
logger.error(m.analyse(text))
logger.error(segs)
logger.error(dag_to_text_idx)
logger.error('Leaving this segment untokenised')
return [text]
def _morf_tokenize_and_align(example: DatasetInstance, morfeusz: Morfeusz, masking_strategy: str = FIRST) -> DatasetInstance:
if masking_strategy not in (FIRST, LAST):
raise RuntimeError(f'Uknown masking strategy: {masking_strategy}')
if masking_strategy == LAST:
raise RuntimeError(f'Can’t use {masking_strategy} masking strategy with retokenize')
labels: DatasetInstance = {}
tokens: TokenList = []
segs: StrLabelList = []
mask = []
for i, token in enumerate(example[TOKENS]):
for j, morf_token in enumerate(morf_tokenize(token, morfeusz)):
tokens.append(morf_token)
segs.append(SEG_BEGIN if j == 0 else SEG_INSIDE)
mask.append(i if j == 0 else None)
labels[TOKENS] = tokens
labels[SEGS] = segs
labels.update(_align_example(example, mask))
return labels
[docs]
def morfeusz_retokenize(
dataset: DatasetDict,
masking_strategy: str = FIRST,
morfeusz_dict_path: str | None = None,
morfeusz_dict_name: str | None = None
) -> DatasetDict:
"""Retokenise the dataset using Morfeusz, possibly splitting tokens into smaller parts.
This is to be done before language model tokenisation, which can futrther split the Morfeusz tokens.
Returns:
the retokenised dataset.
"""
morfeusz = Morfeusz(generate=False, dict_path=morfeusz_dict_path, dict_name=morfeusz_dict_name)
logger.info(f'retokenizing using {morfeusz.dict_id()}')
new_dataset = dataset.map(lambda x: _morf_tokenize_and_align(x, morfeusz, masking_strategy=masking_strategy))
return cast_labels(new_dataset, [SEGS])
[docs]
def lm_tokenize_and_align(
example: DatasetInstance,
tokenizer: PreTrainedTokenizerBase,
masking_strategy: str = FIRST
) -> BatchEncoding:
"""Tokenise the dataset instance using a language model-specific tokeniser.
Returns:
the tokenised instance, encoded as model input.
"""
if masking_strategy not in (FIRST, LAST):
raise RuntimeError(f'Uknown masking strategy: {masking_strategy}')
tokenized_inputs: BatchEncoding = tokenizer(example[TOKENS], truncation=True, is_split_into_words=True)
word_ids = tokenized_inputs.word_ids()
mask = masked_word_ids(word_ids, masking_strategy)
labels = _align_example(example, mask)
tokenized_inputs.update(labels)
return tokenized_inputs