from __future__ import annotations from functools import lru_cache @lru_cache(maxsize=16_384) def damerau_levenshtein_distance(left: str, right: str) -> int: """Return unrestricted Damerau-Levenshtein distance. In addition to insertions, deletions and substitutions, adjacent character transpositions cost one operation. The implementation is the unrestricted variant, not the more limited optimal-string-alignment shortcut. """ if left == right: return 0 if not left: return len(right) if not right: return len(left) left_length = len(left) right_length = len(right) maximum_distance = left_length + right_length matrix = [ [0 for _ in range(right_length + 2)] for _ in range(left_length + 2) ] matrix[0][0] = maximum_distance for left_index in range(left_length + 1): matrix[left_index + 1][0] = maximum_distance matrix[left_index + 1][1] = left_index for right_index in range(right_length + 1): matrix[0][right_index + 1] = maximum_distance matrix[1][right_index + 1] = right_index last_row_by_character: dict[str, int] = {} for left_index in range(1, left_length + 1): last_matching_column = 0 for right_index in range(1, right_length + 1): matching_row = last_row_by_character.get(right[right_index - 1], 0) matching_column = last_matching_column substitution_cost = 1 if left[left_index - 1] == right[right_index - 1]: substitution_cost = 0 last_matching_column = right_index matrix[left_index + 1][right_index + 1] = min( matrix[left_index][right_index] + substitution_cost, matrix[left_index + 1][right_index] + 1, matrix[left_index][right_index + 1] + 1, matrix[matching_row][matching_column] + (left_index - matching_row - 1) + 1 + (right_index - matching_column - 1), ) last_row_by_character[left[left_index - 1]] = left_index return matrix[left_length + 1][right_length + 1] def damerau_similarity(left: str, right: str) -> float: if left == right: return 1.0 maximum_length = max(len(left), len(right)) if maximum_length == 0: return 1.0 return 1.0 - damerau_levenshtein_distance(left, right) / maximum_length @lru_cache(maxsize=32_768) def character_trigrams(value: str) -> frozenset[str]: padded = f"^{value}$" if len(padded) <= 3: return frozenset({padded}) return frozenset(padded[index : index + 3] for index in range(len(padded) - 2)) def trigram_dice(left: str, right: str) -> float: left_trigrams = character_trigrams(left) right_trigrams = character_trigrams(right) denominator = len(left_trigrams) + len(right_trigrams) if denominator == 0: return 1.0 return 2.0 * len(left_trigrams & right_trigrams) / denominator @lru_cache(maxsize=16_384) def lcs_similarity(left: str, right: str) -> float: maximum_length = max(len(left), len(right)) if maximum_length == 0: return 1.0 previous = [0] * (len(right) + 1) for left_character in left: current = [0] for right_index, right_character in enumerate(right, start=1): if left_character == right_character: current.append(previous[right_index - 1] + 1) else: current.append(max(previous[right_index], current[-1])) previous = current return previous[-1] / maximum_length def word_similarity(left: str, right: str) -> float: if left == right: return 1.0 if not left or not right: return 0.0 shorter_length = min(len(left), len(right)) edit = damerau_similarity(left, right) if shorter_length <= 4: # Trigrams are unstable for very short words; edit distance is the main # signal and exact/prefix matching is handled before this function. return edit trigram = trigram_dice(left, right) lcs = lcs_similarity(left, right) return edit * 0.60 + trigram * 0.30 + lcs * 0.10 def fuzzy_threshold(token_length: int) -> float: if token_length <= 4: return 0.84 if token_length <= 7: return 0.72 if token_length <= 10: return 0.61 return 0.53