Source code for hydra.utils.dataset

"""Dataset manipulation utilities."""

from collections import Counter, defaultdict
from itertools import chain

import logging
logger = logging.getLogger(__name__)

from datasets import ClassLabel, DatasetDict, Features, Sequence
from transformers import BatchEncoding, PreTrainedTokenizerBase

from morfeusz2 import Morfeusz

from .hybrid_tree import tree_from_dataset_instance, TreeNode
from .constants import (
    FIRST,
    LAST,
    MASK_VALUE,
    EMPTY,
    ALL_UPPER,
    ALL_LOWER,
    FIRST_UPPER,
    KEEP,
    SEG_BEGIN,
    SEG_INSIDE,
    TOKENS,
    SEGS,
    LEMMAS,
    LEMMA_CASES,
    LEMMA_RULES,
    TAGS,
    SPINES,
    ANCHORS,
    ANCHOR_HS,
    HEADS,
    ADJACENCY_MATRIX,
    GRAPH_HEADS,
    GRAPH_DEPRELS,
    GRAPH_ADJACENCY_MATRIX,
)
from .types import TokenList, StrLabelList, SimpleLabelList, AnyLabelList, DatasetInstance

from IPython.display import display

# assuming all parts have identical features!
[docs] def get_dataset_features(dataset: DatasetDict) -> Features: """Get the dataset’s features signature. Returns: the features. """ any_key = list(dataset.keys())[0] return dataset[any_key].features
def _none_to_mask(instance: DatasetInstance, columns: list[str]) -> DatasetInstance: ret: DatasetInstance = {} for column in columns: ret[column] = [v if v is not None else MASK_VALUE for v in instance[column]] # type: ignore[literal-required] return ret
[docs] def cast_labels(dataset: DatasetDict, columns: list[str]) -> DatasetDict: """Cast the provided columns from `datasets.Value` into `datasets.ClassLabel`, updating the features signature. Returns: the dataset with cast columns. """ vals: dict[str, set[str]] = defaultdict(set) for d in dataset.values(): for column in columns: vals[column].update(chain.from_iterable(s[column] for s in d)) new_features = get_dataset_features(dataset).copy() for column in columns: vals[column].discard(None) new_features[column] = Sequence(ClassLabel(names=sorted(vals[column]))) # labels to indices cast_dataset = dataset.cast(new_features) # replace None with MASK_VALUE return cast_dataset.map(lambda instance: _none_to_mask(instance, columns))
def _get_case_pattern(word: str) -> str: """Generate an upper/lowercase pattern ot the given string. The pattern truncates any trailing same-case characters. e.g.: `kotek` → `xxxxx` → `x`, `Anna` → `Xxxx` → `Xx`, `ZOMO` → `XXXX` → `X`, `UOKiK` → `XXXxX` → `XXXxX`. Returns: the pattern. """ c = [char.isupper() for char in word] while len(c) > 1 and c[-1] == c[-2]: c = c[:-1] return ''.join('X' if x else 'x' for x in c) def _get_lemma_case_pattern(form: str, lemma: str) -> str: """Generate a case pattern for the lemma: all lowercase, all uppercase, first uppercase or follow the form. Returns: the pattern. """ cf = _get_case_pattern(form) cl = _get_case_pattern(lemma) if cl in (ALL_UPPER, ALL_LOWER, FIRST_UPPER): return cl elif cl == cf or form.startswith(lemma): return KEEP # fallbacks elif cl[0] == 'x': return ALL_LOWER else: return FIRST_UPPER def _make_lemma_rule(token: str, lemma: str, tag: str) -> tuple[str, str]: """Generate a rule for transforming the token form into a lemma in the form of `<prefix_cut>_<cut>_<suffix>`. e.g. `'3_7_y'` (remove first 3 characters, remove last 2 characters, append `y`) for `'najpiękniejszej'` → `'piękny'`. Returns: the pattern. """ case = _get_lemma_case_pattern(token, lemma) prefix_cut = 0 token, lemma = token.lower(), lemma.lower() #if lemma.startswith('naj') and or if (token.startswith('nie') and 'neg' in tag) or (token.startswith('naj') and 'sup' in tag): prefix_cut = 3 token = token[3:] cut = 0 while token[:(cut + 1)] == lemma[:(cut + 1)] and cut < len(token): cut += 1 suffix = lemma[cut:] cut = len(token) - cut return case, f'{prefix_cut}_{cut}_{suffix}' def _add_lemma_rules(instance: DatasetInstance, tag_dict: dict[int, str]) -> DatasetInstance: tokens = instance[TOKENS] lemmas = instance[LEMMAS] tags = [tag_dict[v] for v in instance[TAGS]] # type: ignore [index] # all v’s are not None cases, rules = map(list, zip(*(_make_lemma_rule(*x) for x in zip(tokens, lemmas, tags)))) # type: ignore [arg-type] return { LEMMA_CASES : cases, LEMMA_RULES : rules, }
[docs] def add_lemma_rules(dataset: DatasetDict) -> DatasetDict: """Add dataset columns used to train the lemmatisation module. Returns: a dataset with columns added. """ tag_dict = get_dataset_features(dataset)[TAGS].feature.names new_dataset = dataset.map(lambda instance: _add_lemma_rules(instance, tag_dict)) return cast_labels(new_dataset, [LEMMA_CASES, LEMMA_RULES])
def _do_collect_spines(tree: TreeNode) -> tuple[list[TreeNode], list[tuple[str, int, list[TreeNode]]]]: if not tree.children: return [tree], [] heads = [child for child in tree.children if child.is_head] assert(len(heads) == 1) head = heads[0] paths = [] my_path = [tree] non_heads = [] for child in tree.children: child_path, grandchildren_paths = _do_collect_spines(child) paths += grandchildren_paths if child == head: my_path += child_path else: non_heads.append(child_path) for child_path in non_heads: # h == which <tree.category> counting from the bottom is the anchor assert(tree.category is not None) h = [n.category for n in my_path].count(tree.category) paths.append((tree.category, h, child_path)) return my_path, paths def _collect_spines(tree: TreeNode) -> dict[TreeNode, tuple[str, int, list[TreeNode]]]: try: path, paths = _do_collect_spines(tree) except: logger.critical('Couldn’t colllect spines:') logger.critical(tree.to_brackets()) raise return {p[-1] : (anchor, h, p[:-1]) for anchor, h, p in [('<ROOT>', -1, path)] + paths} def _compress_spine(spine: list[str]) -> list[str]: compressed: list[str] = [] for category in spine: if category in compressed: assert(category == compressed[-1]) else: compressed.append(category) return compressed def _add_spines_and_attachments(instance: DatasetInstance, dataset_features: Features, compress: bool) -> DatasetInstance: tree = tree_from_dataset_instance(instance, dataset_features) if tree is None: return { SPINES : [None for _ in instance[TOKENS]], # type: ignore[typeddict-item] ANCHORS : [None for _ in instance[TOKENS]], # type: ignore[typeddict-item] ANCHOR_HS : [None for _ in instance[TOKENS]], # type: ignore[typeddict-item] } spines = _collect_spines(tree) leafs_linear = sorted(tree.get_yield(), key=lambda leaf: leaf.from_index) # type: ignore[arg-type, return-value] # all from_index’s are not None rows: list[tuple[str, str, str]] = [] for leaf in leafs_linear: anchor, anchor_h, spine = spines[leaf] spine_cats: list[str] = [node.category for node in spine] # type: ignore[misc] # all category’s are not None if compress: spine_cats = _compress_spine(spine_cats) spine_label = '_'.join(spine_cats) if spine_cats else EMPTY rows.append((spine_label, anchor, str(anchor_h) if anchor_h != -1 else '<ROOT>')) spine_labels, anchors, anchor_hs = map(list, zip(*rows)) return { SPINES : spine_labels, ANCHORS : anchors, ANCHOR_HS : anchor_hs, }
[docs] def add_spines_and_attachments(dataset: DatasetDict, compress: bool = False) -> DatasetDict: """Add dataset columns used to train the hybrid syntactic parsing module. Returns: a dataset istance with columns added. """ dataset_features = get_dataset_features(dataset) new_dataset = dataset.map( lambda instance: _add_spines_and_attachments(instance, dataset_features, compress=compress)) return cast_labels(new_dataset, [SPINES, ANCHORS, ANCHOR_HS])
EDGE, NO_EDGE = 1, 0 def _add_adjacency_matrix(instance: DatasetInstance) -> DatasetInstance: heads = instance[HEADS] if set(heads) == {MASK_VALUE}: return {ADJACENCY_MATRIX : [[MASK_VALUE for j in range(len(heads))] for i in range(len(heads))]} # ROOT is ‘it’s own’ head int_heads: list[int] = [x if x is not None else i for i, x in enumerate(heads)] am = [[NO_EDGE for j in range(len(heads))] for i in range(len(heads))] for i, head in enumerate(int_heads): am[i][head] = EDGE return {ADJACENCY_MATRIX : am} # type: ignore[typeddict-item]
[docs] def add_adjacency_matrix(dataset: DatasetDict) -> DatasetDict: """Add dataset column used to train the dependency parsing module. Returns: a dataset istance with columns added. """ return dataset.map(lambda instance: _add_adjacency_matrix(instance))
def _add_graph_matrices(instance: DatasetInstance) -> DatasetInstance: heads = instance[GRAPH_HEADS] deprels = instance[GRAPH_DEPRELS] N = len(heads) if set(chain.from_iterable(heads)) == {MASK_VALUE}: return { GRAPH_ADJACENCY_MATRIX : [[MASK_VALUE for j in range(N)] for i in range(N)], GRAPH_DEPRELS : [[MASK_VALUE for j in range(N)] for i in range(N)], } # ROOT is ‘it’s own’ head am = [[NO_EDGE for j in range(N)] for i in range(N)] # deprels: mask unless there is an edge dm = [[MASK_VALUE for j in range(N)] for i in range(N)] for i, (graph_heads, graph_deprels) in enumerate(zip(heads, deprels)): for head, deprel in zip(graph_heads, graph_deprels): if head is None: head = i am[i][head] = EDGE dm[i][head] = deprel # type: ignore[assignment] return {GRAPH_ADJACENCY_MATRIX : am, GRAPH_DEPRELS : dm} # type: ignore[typeddict-item]
[docs] def add_graph_matrices(dataset: DatasetDict) -> DatasetDict: """Add dataset columns used to train the graph parsing module. Returns: a dataset istance with columns added. """ return dataset.map(lambda instance: _add_graph_matrices(instance))
# https://huggingface.co/docs/transformers/v4.23.1/en/tasks/token_classification
[docs] def masked_word_ids(word_ids: list[int | None], masking_strategy: str = FIRST) -> list[int | None]: """Mask positions of multiple language model tokens corresponding to single input segment (word). This is needed to properly align training labels (one per word) with tokens passed to the language model. Args: word_ids: indices of input words from which the language model tokens come. masking_strategy: :const:`~hydra.utils.constants.FIRST` (only leave the first token unmasked for each word) or :const:`~hydra.utils.constants.LAST` (only leave the last token unmasked for each word). Returns: a list of the same length as `word_ids`, with masked postions repleced with `None`. """ masked: list[int | None] = [] for i, word_idx in enumerate(word_ids): # Set the label for the first/last token of each word. # Mask the label for: # * special tokens (word id = None) # * other tokens in a word if word_idx is None: masked.append(None) else: if masking_strategy == FIRST: masked.append(word_idx if word_idx != word_ids[i - 1] else None) elif masking_strategy == LAST: masked.append(word_idx if word_idx != word_ids[i + 1] else None) return masked
def _align_row(values: SimpleLabelList, masked_word_ids: list[int | None]) -> SimpleLabelList: # in fact, it’s an IntLabelList try: return [MASK_VALUE if idx is None else values[idx] for idx in masked_word_ids] # type: ignore[return-value] except: logger.critical('Couldn’t align row:') logger.critical(f'values: {values}') logger.critical(f'masked_word_ids: {masked_word_ids}') raise def _align_example(example: DatasetInstance, masked_ids: list[int | None]) -> DatasetInstance: labels: DatasetInstance = {} masked_row = [MASK_VALUE for x in masked_ids] try: for column_name, values in example.items(): if column_name in (TOKENS, LEMMAS): continue if type(values) == str: continue matrix = hasattr(values[0], '__iter__') # type: ignore[index] # [*] because of the items() iteration, mypy complains about values’ type aligned_labels: AnyLabelList if matrix: aligned_labels = [_align_row(values[idx], masked_ids) if idx is not None else masked_row for idx in masked_ids] # type: ignore[index, assignment] # [*] else: aligned_labels = _align_row(values, masked_ids) # type: ignore[arg-type] # [*] labels[column_name] = aligned_labels # type: ignore[literal-required] except: logger.critical('Couldn’t align example:') logger.critical(f'example: {example}') logger.critical(f'column_name: {column_name}') logger.critical(f'example[column_name]: {values}') logger.critical(f'masked_ids: masked_ids') raise return labels
[docs] def morf_tokenize(text: str, m: Morfeusz) -> TokenList: """Tokenize the text using Morfeusz. The most fine-grained tokenisation is retrieved from the DAG. Returns: the tokenised text. """ segs: dict[tuple[int, int], str] = {} max_j = 0 for i, j, interp in m.analyse(text): orth = interp[0] if (i, j) in segs: assert (orth == segs[(i, j)]) else: segs[(i, j)] = orth max_j = max(max_j, j) # Morfeusz DAG vertices <-> positions in the text dag_to_text_idx = {0 : 0} for (i, j), seg in sorted(segs.items()): assert(i in dag_to_text_idx) jdx = dag_to_text_idx[i] + len(seg) if j in dag_to_text_idx: assert(dag_to_text_idx[j] == jdx) else: dag_to_text_idx[j] = jdx for (i, j), seg in sorted(segs.items()): try: assert(seg == text[dag_to_text_idx[i]:dag_to_text_idx[j]]) except: logger.error(f'Couldn’t get Morfeusz tokenisation: for ‘{text}’') logger.error(m.analyse(text)) logger.error(segs) logger.error(dag_to_text_idx) logger.error('Leaving this segment untokenised') return [text] try: # return [segs[(i, i + 1)] for i in range(max_j)] return [text[dag_to_text_idx[i]:dag_to_text_idx[i + 1]] for i in range(max_j)] except: logger.error(f'Couldn’t get Morfeusz tokenisation: for ‘{text}’') logger.error(m.analyse(text)) logger.error(segs) logger.error(dag_to_text_idx) logger.error('Leaving this segment untokenised') return [text]
def _morf_tokenize_and_align(example: DatasetInstance, morfeusz: Morfeusz, masking_strategy: str = FIRST) -> DatasetInstance: if masking_strategy not in (FIRST, LAST): raise RuntimeError(f'Uknown masking strategy: {masking_strategy}') if masking_strategy == LAST: raise RuntimeError(f'Can’t use {masking_strategy} masking strategy with retokenize') labels: DatasetInstance = {} tokens: TokenList = [] segs: StrLabelList = [] mask = [] for i, token in enumerate(example[TOKENS]): for j, morf_token in enumerate(morf_tokenize(token, morfeusz)): tokens.append(morf_token) segs.append(SEG_BEGIN if j == 0 else SEG_INSIDE) mask.append(i if j == 0 else None) labels[TOKENS] = tokens labels[SEGS] = segs labels.update(_align_example(example, mask)) return labels
[docs] def morfeusz_retokenize( dataset: DatasetDict, masking_strategy: str = FIRST, morfeusz_dict_path: str | None = None, morfeusz_dict_name: str | None = None ) -> DatasetDict: """Retokenise the dataset using Morfeusz, possibly splitting tokens into smaller parts. This is to be done before language model tokenisation, which can futrther split the Morfeusz tokens. Returns: the retokenised dataset. """ morfeusz = Morfeusz(generate=False, dict_path=morfeusz_dict_path, dict_name=morfeusz_dict_name) logger.info(f'retokenizing using {morfeusz.dict_id()}') new_dataset = dataset.map(lambda x: _morf_tokenize_and_align(x, morfeusz, masking_strategy=masking_strategy)) return cast_labels(new_dataset, [SEGS])
[docs] def lm_tokenize_and_align( example: DatasetInstance, tokenizer: PreTrainedTokenizerBase, masking_strategy: str = FIRST ) -> BatchEncoding: """Tokenise the dataset instance using a language model-specific tokeniser. Returns: the tokenised instance, encoded as model input. """ if masking_strategy not in (FIRST, LAST): raise RuntimeError(f'Uknown masking strategy: {masking_strategy}') tokenized_inputs: BatchEncoding = tokenizer(example[TOKENS], truncation=True, is_split_into_words=True) word_ids = tokenized_inputs.word_ids() mask = masked_word_ids(word_ids, masking_strategy) labels = _align_example(example, mask) tokenized_inputs.update(labels) return tokenized_inputs