Fiden/test-search-engine
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add normalization and product alias indexes

Browse Source
This commit is contained in:
Fiden
2026-08-07 13:05:58 +00:00
parent daf107feed
commit 8e1e0f82a2
3 changed files with 592 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/aliases.py
+325
View File
@@ -0,0 +1,325 @@
from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from app.normalization import meaningful_tokens, normalize_text, tokenize, word_tokens
from app.typo import character_trigrams, fuzzy_threshold, word_similarity
# Canonical product concepts are intentionally separate from SKU aliases. The
# list contains domain vocabulary only; it never maps a phrase directly to an
# individual product.
PRODUCT_ALIASES = [
{
"canonical": "ушм",
"aliases": [
"ушм",
"болгарка",
"углошлифовальная машина",
"угловая шлифмашина",
"угловая шлифовальная машина",
],
},
{
"canonical": "шуруповерт",
"aliases": [
"шуруповерт",
"шуруповёрт",
"шурик",
"дрель шуруповерт",
"дрель-шуруповерт",
],
},
{
"canonical": "перфоратор",
"aliases": ["перфоратор", "перф", "перфоратор sds"],
},
{"canonical": "саморез", "aliases": ["саморез", "саморезы", "саморезов"]},
{
"canonical": "труба",
"aliases": [
"труба",
"трубы",
"проф труба",
"профтруба",
"труба профильная",
"профильная труба",
],
},
{
"canonical": "кабель",
"aliases": [
"кабель",
"кабеля",
"кабелей",
"ввг",
"ввгнг",
"ввгнг ls",
"ввгнг лс",
"шввп",
"пвс",
],
},
{"canonical": "провод", "aliases": ["провод", "провода", "проводов", "пугв"]},
{"canonical": "бита", "aliases": ["бита", "биты", "биту", "биты для шуруповерта"]},
{
"canonical": "держатель бит",
"aliases": ["держатель бит", "битодержатель", "магнитный держатель"],
},
{"canonical": "адаптер бит", "aliases": ["адаптер для бит", "адаптер бит"]},
{"canonical": "сверло", "aliases": ["сверло", "сверла", "сверел", "сверлить"]},
{"canonical": "бур", "aliases": ["бур", "буры", "сдс бур", "sds бур"]},
{
"canonical": "диск",
"aliases": [
"диск",
"диски",
"дисков",
"круг отрезной",
"отрезной круг",
"круг зачистной",
"зачистной круг",
"пильный круг",
],
},
{
"canonical": "круг",
"aliases": ["круг шлифовальный", "шлифовальный круг", "круг на липучке"],
},
{"canonical": "дрель", "aliases": ["дрель", "дрели", "ударная дрель"]},
{"canonical": "лобзик", "aliases": ["лобзик", "электролобзик"]},
{"canonical": "фен", "aliases": ["строительный фен", "термофен", "фен"]},
{"canonical": "перчатки", "aliases": ["перчатки", "перчаток"]},
{"canonical": "изолента", "aliases": ["изолента", "изоляционная лента"]},
{"canonical": "скотч", "aliases": ["скотч", "малярный скотч"]},
{
"canonical": "стяжка",
"aliases": [
"стяжка",
"стяжки",
"кабельная стяжка",
"нейлоновая стяжка",
"хомут пластиковый",
"хомуты пластиковые",
"пластиковый хомут",
"пластиковые хомуты",
],
},
{"canonical": "пена", "aliases": ["пена", "монтажная пена"]},
{"canonical": "герметик", "aliases": ["герметик", "герметики"]},
{"canonical": "нож", "aliases": ["нож", "строительный нож"]},
{"canonical": "лезвия", "aliases": ["лезвие", "лезвия", "сменные лезвия"]},
{"canonical": "рулетка", "aliases": ["рулетка", "рулетки"]},
{"canonical": "карандаш", "aliases": ["карандаш", "строительный карандаш"]},
{"canonical": "маркер", "aliases": ["маркер", "маркеры"]},
{"canonical": "респиратор", "aliases": ["респиратор", "респираторы"]},
{"canonical": "очки", "aliases": ["защитные очки", "очки"]},
{"canonical": "мешки", "aliases": ["мешок", "мешки", "мешки для мусора"]},
{"canonical": "кисть", "aliases": ["кисть", "кисти", "малярная кисть"]},
{"canonical": "валик", "aliases": ["валик", "валики", "малярный валик"]},
{"canonical": "ванночка", "aliases": ["ванночка", "малярная ванночка"]},
{"canonical": "удлинитель", "aliases": ["удлинитель", "удлинители"]},
{
"canonical": "шкурка",
"aliases": [
"шкурка",
"шлифовальная шкурка",
"наждачка",
"наждачная бумага",
],
},
{"canonical": "лента фум", "aliases": ["лента фум", "фум лента", "фумка"]},
{
"canonical": "хомут",
"aliases": ["хомут", "хомуты", "червячный хомут", "металлический хомут"],
},
{
"canonical": "гипсокартон",
"aliases": [
"гкл",
"гипсокартон",
"гипсокартонный лист",
"лист гипсокартонный",
],
},
{"canonical": "профиль", "aliases": ["профиль", "профили", "профиль для гкл"]},
{"canonical": "подвес", "aliases": ["подвес", "прямой подвес"]},
{"canonical": "соединитель", "aliases": ["соединитель", "краб", "соединитель краб"]},
{"canonical": "уголок", "aliases": ["уголок", "перфорированный уголок"]},
{"canonical": "лента серпянка", "aliases": ["серпянка", "лента серпянка"]},
{"canonical": "лента демпферная", "aliases": ["демпферная лента", "лента демпферная"]},
{"canonical": "болт", "aliases": ["болт", "болты", "болтов"]},
{"canonical": "гайка", "aliases": ["гайка", "гайки", "гаек"]},
{"canonical": "шайба", "aliases": ["шайба", "шайбы", "шайб"]},
{"canonical": "шпилька", "aliases": ["шпилька", "шпильки", "резьбовая шпилька"]},
{"canonical": "анкер", "aliases": ["анкер", "анкеры", "анкеров"]},
{"canonical": "гвозди", "aliases": ["гвоздь", "гвозди", "гвоздей"]},
{
"canonical": "дюбель",
"aliases": [
"дюбель",
"дюбели",
"дюбелей",
"дюбель гвоздь",
"дюбель-гвоздь",
],
},
{"canonical": "ключ", "aliases": ["ключ", "ключи", "рожковый ключ", "разводной ключ"]},
{"canonical": "отвертка", "aliases": ["отвертка", "отвёртка", "отвертки"]},
{"canonical": "молоток", "aliases": ["молоток", "молотки"]},
{"canonical": "ножовка", "aliases": ["ножовка", "ножовки"]},
{"canonical": "плоскогубцы", "aliases": ["плоскогубцы"]},
{"canonical": "бокорезы", "aliases": ["бокорезы", "бокорез"]},
{"canonical": "пассатижи", "aliases": ["пассатижи", "клещи переставные"]},
{"canonical": "уровень", "aliases": ["уровень", "уровни"]},
{"canonical": "угольник", "aliases": ["угольник", "угольники"]},
{"canonical": "степлер", "aliases": ["степлер", "мебельный степлер"]},
{"canonical": "скобы", "aliases": ["скоба", "скобы", "скобы для степлера"]},
]
@dataclass(frozen=True, slots=True)
class ProductTypeMatch:
canonical: str
score: float
exact: bool
alias: str
token_start: int
@dataclass(frozen=True, slots=True)
class _AliasRecord:
canonical: str
normalized: str
tokens: tuple[str, ...]
class ProductAliasIndex:
def __init__(self) -> None:
alias_to_canonical: dict[str, str] = {}
canonical_to_aliases: dict[str, list[str]] = defaultdict(list)
records: list[_AliasRecord] = []
single_token_records: list[_AliasRecord] = []
trigram_postings: dict[str, set[int]] = defaultdict(set)
for item in PRODUCT_ALIASES:
canonical = normalize_text(item["canonical"])
for raw_alias in item["aliases"]:
alias = normalize_text(raw_alias)
previous = alias_to_canonical.get(alias)
if previous is not None and previous != canonical:
raise ValueError(
f"Alias {raw_alias!r} points to both {previous!r} and {canonical!r}"
)
alias_to_canonical[alias] = canonical
canonical_to_aliases[canonical].append(alias)
record = _AliasRecord(canonical, alias, tokenize(alias, already_normalized=True))
records.append(record)
if len(record.tokens) == 1 and record.tokens[0].isalpha():
record_index = len(single_token_records)
single_token_records.append(record)
for trigram in character_trigrams(record.tokens[0]):
trigram_postings[trigram].add(record_index)
self.alias_to_canonical = alias_to_canonical
self.canonical_to_aliases = {
canonical: tuple(dict.fromkeys(aliases))
for canonical, aliases in canonical_to_aliases.items()
}
self._records = tuple(records)
self._single_token_records = tuple(single_token_records)
self._trigram_postings = {
trigram: frozenset(indexes) for trigram, indexes in trigram_postings.items()
}
def detect(self, text: str) -> ProductTypeMatch | None:
normalized = normalize_text(text)
tokens = tokenize(normalized, already_normalized=True)
exact = self._find_exact(tokens, prefix_only=False)
if exact is not None:
return exact
return self._find_fuzzy(tokens)
def detect_catalog_type(self, text: str) -> str:
normalized = normalize_text(text)
tokens = tokenize(normalized, already_normalized=True)
exact = self._find_exact(tokens, prefix_only=True)
if exact is None:
raise ValueError(f"No product type alias matches catalog name: {text!r}")
return exact.canonical
def aliases_for(self, canonical: str) -> tuple[str, ...]:
return self.canonical_to_aliases.get(canonical, ())
def _find_exact(
self, tokens: tuple[str, ...], *, prefix_only: bool
) -> ProductTypeMatch | None:
matches: list[ProductTypeMatch] = []
for record in self._records:
alias_length = len(record.tokens)
if alias_length == 0 or alias_length > len(tokens):
continue
starts = (0,) if prefix_only else range(len(tokens) - alias_length + 1)
for start in starts:
if tokens[start : start + alias_length] == record.tokens:
matches.append(
ProductTypeMatch(
canonical=record.canonical,
score=1.0,
exact=True,
alias=record.normalized,
token_start=start,
)
)
break
if not matches:
return None
matches.sort(
key=lambda match: (
-len(tokenize(match.alias, already_normalized=True)),
match.token_start,
-len(match.alias),
)
)
return matches[0]
def _find_fuzzy(self, tokens: tuple[str, ...]) -> ProductTypeMatch | None:
query_words = word_tokens(meaningful_tokens(tokens))
best: ProductTypeMatch | None = None
for token_start, query_token in enumerate(query_words):
if len(query_token) < 4:
continue
candidate_indexes: set[int] = set()
for trigram in character_trigrams(query_token):
candidate_indexes.update(self._trigram_postings.get(trigram, ()))
if not candidate_indexes:
continue
for candidate_index in candidate_indexes:
record = self._single_token_records[candidate_index]
alias_token = record.tokens[0]
if abs(len(query_token) - len(alias_token)) > max(4, len(alias_token) // 2):
continue
score = word_similarity(query_token, alias_token)
required = fuzzy_threshold(max(len(query_token), len(alias_token)))
if score < required:
continue
match = ProductTypeMatch(
canonical=record.canonical,
score=score,
exact=False,
alias=record.normalized,
token_start=token_start,
)
if best is None or (match.score, -match.token_start) > (
best.score,
-best.token_start,
):
best = match
return best
app/normalization.py
+134
View File
@@ -0,0 +1,134 @@
from __future__ import annotations
import re
import unicodedata
from collections.abc import Iterable
_DECIMAL_COMMA_RE = re.compile(r"(?<=\d),(?=\d)")
_DIMENSION_SEPARATOR_RE = re.compile(r"(?<=\d)\s*[xх×*]\s*(?=\d)", re.IGNORECASE)
_CYRILLIC_THREAD_RE = re.compile(r"(?<![a-zа-я0-9])м(?=\d)", re.IGNORECASE)
_CYRILLIC_GRIT_RE = re.compile(r"(?<![a-zа-я0-9])р(?=\d)", re.IGNORECASE)
_WHITESPACE_RE = re.compile(r"\s+")
_PUNCTUATION_RE = re.compile(r"[^0-9a-zа-я.\-+x\s]", re.IGNORECASE)
_TOKEN_RE = re.compile(
r"\d+(?:\.\d+)?(?:x\d+(?:\.\d+)?)+"
r"|[a-zа-я]+(?:\d+(?:\.\d+)?[a-zа-я]*)?"
r"|\d+(?:\.\d+)?",
re.IGNORECASE,
)
# These tokens stay in the normalized message but do not affect product scoring.
ZERO_WEIGHT_TOKENS = frozenset(
{
"а",
"без",
"бы",
"в",
"вам",
"ваш",
"вот",
"где",
"дайте",
"для",
"до",
"есть",
"еще",
"за",
"здравствуйте",
"и",
"из",
"как",
"какие",
"какой",
"какая",
"который",
"ли",
"мне",
"можно",
"на",
"надо",
"нужен",
"нужна",
"нужны",
"нужно",
"по",
"подскажите",
"посоветуйте",
"при",
"сколько",
"только",
"у",
"хочу",
"что",
"шт",
"штук",
"вопрос",
}
)
UNIT_TOKENS = frozenset(
{
"в",
"вольт",
"вольта",
"вольтов",
"вт",
"ватт",
"ватта",
"ваттов",
"мм",
"миллиметр",
"миллиметра",
"миллиметров",
"м",
"метр",
"метра",
"метров",
"см",
"г",
"кг",
"л",
"мл",
"дж",
}
)
def normalize_text(text: str) -> str:
"""Create one deterministic representation for aliases and catalog text.
The Cyrillic letter ``х`` is converted only when it is a multiplication
separator between numbers. A global replacement would corrupt normal words
such as ``хомут`` and ``находится``.
"""
value = unicodedata.normalize("NFKC", text or "").lower().replace("ё", "е")
value = value.replace("–", "-").replace("—", "-").replace("−", "-")
value = _DECIMAL_COMMA_RE.sub(".", value)
value = _DIMENSION_SEPARATOR_RE.sub("x", value)
value = _CYRILLIC_THREAD_RE.sub("m", value)
value = _CYRILLIC_GRIT_RE.sub("p", value)
value = _PUNCTUATION_RE.sub(" ", value)
return _WHITESPACE_RE.sub(" ", value).strip()
def tokenize(text: str, *, already_normalized: bool = False) -> tuple[str, ...]:
value = text if already_normalized else normalize_text(text)
return tuple(match.group(0) for match in _TOKEN_RE.finditer(value))
def meaningful_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
return tuple(
token
for token in tokens
if token not in ZERO_WEIGHT_TOKENS and token not in UNIT_TOKENS
)
def word_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
return tuple(token for token in tokens if any(character.isalpha() for character in token))
def normalize_code(value: str) -> str:
normalized = normalize_text(value)
return "".join(character for character in normalized if character.isalnum())
app/typo.py
+133
View File
@@ -0,0 +1,133 @@
from __future__ import annotations
from functools import lru_cache
@lru_cache(maxsize=16_384)
def damerau_levenshtein_distance(left: str, right: str) -> int:
"""Return unrestricted Damerau-Levenshtein distance.
In addition to insertions, deletions and substitutions, adjacent character
transpositions cost one operation. The implementation is the unrestricted
variant, not the more limited optimal-string-alignment shortcut.
"""
if left == right:
return 0
if not left:
return len(right)
if not right:
return len(left)
left_length = len(left)
right_length = len(right)
maximum_distance = left_length + right_length
matrix = [
[0 for _ in range(right_length + 2)] for _ in range(left_length + 2)
]
matrix[0][0] = maximum_distance
for left_index in range(left_length + 1):
matrix[left_index + 1][0] = maximum_distance
matrix[left_index + 1][1] = left_index
for right_index in range(right_length + 1):
matrix[0][right_index + 1] = maximum_distance
matrix[1][right_index + 1] = right_index
last_row_by_character: dict[str, int] = {}
for left_index in range(1, left_length + 1):
last_matching_column = 0
for right_index in range(1, right_length + 1):
matching_row = last_row_by_character.get(right[right_index - 1], 0)
matching_column = last_matching_column
substitution_cost = 1
if left[left_index - 1] == right[right_index - 1]:
substitution_cost = 0
last_matching_column = right_index
matrix[left_index + 1][right_index + 1] = min(
matrix[left_index][right_index] + substitution_cost,
matrix[left_index + 1][right_index] + 1,
matrix[left_index][right_index + 1] + 1,
matrix[matching_row][matching_column]
+ (left_index - matching_row - 1)
+ 1
+ (right_index - matching_column - 1),
)
last_row_by_character[left[left_index - 1]] = left_index
return matrix[left_length + 1][right_length + 1]
def damerau_similarity(left: str, right: str) -> float:
if left == right:
return 1.0
maximum_length = max(len(left), len(right))
if maximum_length == 0:
return 1.0
return 1.0 - damerau_levenshtein_distance(left, right) / maximum_length
@lru_cache(maxsize=32_768)
def character_trigrams(value: str) -> frozenset[str]:
padded = f"^{value}$"
if len(padded) <= 3:
return frozenset({padded})
return frozenset(padded[index : index + 3] for index in range(len(padded) - 2))
def trigram_dice(left: str, right: str) -> float:
left_trigrams = character_trigrams(left)
right_trigrams = character_trigrams(right)
denominator = len(left_trigrams) + len(right_trigrams)
if denominator == 0:
return 1.0
return 2.0 * len(left_trigrams & right_trigrams) / denominator
@lru_cache(maxsize=16_384)
def lcs_similarity(left: str, right: str) -> float:
maximum_length = max(len(left), len(right))
if maximum_length == 0:
return 1.0
previous = [0] * (len(right) + 1)
for left_character in left:
current = [0]
for right_index, right_character in enumerate(right, start=1):
if left_character == right_character:
current.append(previous[right_index - 1] + 1)
else:
current.append(max(previous[right_index], current[-1]))
previous = current
return previous[-1] / maximum_length
def word_similarity(left: str, right: str) -> float:
if left == right:
return 1.0
if not left or not right:
return 0.0
shorter_length = min(len(left), len(right))
edit = damerau_similarity(left, right)
if shorter_length <= 4:
# Trigrams are unstable for very short words; edit distance is the main
# signal and exact/prefix matching is handled before this function.
return edit
trigram = trigram_dice(left, right)
lcs = lcs_similarity(left, right)
return edit * 0.60 + trigram * 0.30 + lcs * 0.10
def fuzzy_threshold(token_length: int) -> float:
if token_length <= 4:
return 0.84
if token_length <= 7:
return 0.72
if token_length <= 10:
return 0.61
return 0.53