134 lines
4.3 KiB
Python
134 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
from functools import lru_cache
|
|
|
|
|
|
@lru_cache(maxsize=16_384)
|
|
def damerau_levenshtein_distance(left: str, right: str) -> int:
|
|
"""Return unrestricted Damerau-Levenshtein distance.
|
|
|
|
In addition to insertions, deletions and substitutions, adjacent character
|
|
transpositions cost one operation. The implementation is the unrestricted
|
|
variant, not the more limited optimal-string-alignment shortcut.
|
|
"""
|
|
|
|
if left == right:
|
|
return 0
|
|
if not left:
|
|
return len(right)
|
|
if not right:
|
|
return len(left)
|
|
|
|
left_length = len(left)
|
|
right_length = len(right)
|
|
maximum_distance = left_length + right_length
|
|
matrix = [
|
|
[0 for _ in range(right_length + 2)] for _ in range(left_length + 2)
|
|
]
|
|
matrix[0][0] = maximum_distance
|
|
|
|
for left_index in range(left_length + 1):
|
|
matrix[left_index + 1][0] = maximum_distance
|
|
matrix[left_index + 1][1] = left_index
|
|
for right_index in range(right_length + 1):
|
|
matrix[0][right_index + 1] = maximum_distance
|
|
matrix[1][right_index + 1] = right_index
|
|
|
|
last_row_by_character: dict[str, int] = {}
|
|
for left_index in range(1, left_length + 1):
|
|
last_matching_column = 0
|
|
for right_index in range(1, right_length + 1):
|
|
matching_row = last_row_by_character.get(right[right_index - 1], 0)
|
|
matching_column = last_matching_column
|
|
substitution_cost = 1
|
|
|
|
if left[left_index - 1] == right[right_index - 1]:
|
|
substitution_cost = 0
|
|
last_matching_column = right_index
|
|
|
|
matrix[left_index + 1][right_index + 1] = min(
|
|
matrix[left_index][right_index] + substitution_cost,
|
|
matrix[left_index + 1][right_index] + 1,
|
|
matrix[left_index][right_index + 1] + 1,
|
|
matrix[matching_row][matching_column]
|
|
+ (left_index - matching_row - 1)
|
|
+ 1
|
|
+ (right_index - matching_column - 1),
|
|
)
|
|
last_row_by_character[left[left_index - 1]] = left_index
|
|
|
|
return matrix[left_length + 1][right_length + 1]
|
|
|
|
|
|
def damerau_similarity(left: str, right: str) -> float:
|
|
if left == right:
|
|
return 1.0
|
|
maximum_length = max(len(left), len(right))
|
|
if maximum_length == 0:
|
|
return 1.0
|
|
return 1.0 - damerau_levenshtein_distance(left, right) / maximum_length
|
|
|
|
|
|
@lru_cache(maxsize=32_768)
|
|
def character_trigrams(value: str) -> frozenset[str]:
|
|
padded = f"^{value}$"
|
|
if len(padded) <= 3:
|
|
return frozenset({padded})
|
|
return frozenset(padded[index : index + 3] for index in range(len(padded) - 2))
|
|
|
|
|
|
def trigram_dice(left: str, right: str) -> float:
|
|
left_trigrams = character_trigrams(left)
|
|
right_trigrams = character_trigrams(right)
|
|
denominator = len(left_trigrams) + len(right_trigrams)
|
|
if denominator == 0:
|
|
return 1.0
|
|
return 2.0 * len(left_trigrams & right_trigrams) / denominator
|
|
|
|
|
|
@lru_cache(maxsize=16_384)
|
|
def lcs_similarity(left: str, right: str) -> float:
|
|
maximum_length = max(len(left), len(right))
|
|
if maximum_length == 0:
|
|
return 1.0
|
|
|
|
previous = [0] * (len(right) + 1)
|
|
for left_character in left:
|
|
current = [0]
|
|
for right_index, right_character in enumerate(right, start=1):
|
|
if left_character == right_character:
|
|
current.append(previous[right_index - 1] + 1)
|
|
else:
|
|
current.append(max(previous[right_index], current[-1]))
|
|
previous = current
|
|
return previous[-1] / maximum_length
|
|
|
|
|
|
def word_similarity(left: str, right: str) -> float:
|
|
if left == right:
|
|
return 1.0
|
|
if not left or not right:
|
|
return 0.0
|
|
|
|
shorter_length = min(len(left), len(right))
|
|
edit = damerau_similarity(left, right)
|
|
|
|
if shorter_length <= 4:
|
|
# Trigrams are unstable for very short words; edit distance is the main
|
|
# signal and exact/prefix matching is handled before this function.
|
|
return edit
|
|
|
|
trigram = trigram_dice(left, right)
|
|
lcs = lcs_similarity(left, right)
|
|
return edit * 0.60 + trigram * 0.30 + lcs * 0.10
|
|
|
|
|
|
def fuzzy_threshold(token_length: int) -> float:
|
|
if token_length <= 4:
|
|
return 0.84
|
|
if token_length <= 7:
|
|
return 0.72
|
|
if token_length <= 10:
|
|
return 0.61
|
|
return 0.53
|