Add normalization and product alias indexes

app/aliases.py

@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+
+from app.normalization import meaningful_tokens, normalize_text, tokenize, word_tokens
+from app.typo import character_trigrams, fuzzy_threshold, word_similarity
+
+# Canonical product concepts are intentionally separate from SKU aliases. The
+# list contains domain vocabulary only; it never maps a phrase directly to an
+# individual product.
+PRODUCT_ALIASES = [
+    {
+        "canonical": "ушм",
+        "aliases": [
+            "ушм",
+            "болгарка",
+            "углошлифовальная машина",
+            "угловая шлифмашина",
+            "угловая шлифовальная машина",
+        ],
+    },
+    {
+        "canonical": "шуруповерт",
+        "aliases": [
+            "шуруповерт",
+            "шуруповёрт",
+            "шурик",
+            "дрель шуруповерт",
+            "дрель-шуруповерт",
+        ],
+    },
+    {
+        "canonical": "перфоратор",
+        "aliases": ["перфоратор", "перф", "перфоратор sds"],
+    },
+    {"canonical": "саморез", "aliases": ["саморез", "саморезы", "саморезов"]},
+    {
+        "canonical": "труба",
+        "aliases": [
+            "труба",
+            "трубы",
+            "проф труба",
+            "профтруба",
+            "труба профильная",
+            "профильная труба",
+        ],
+    },
+    {
+        "canonical": "кабель",
+        "aliases": [
+            "кабель",
+            "кабеля",
+            "кабелей",
+            "ввг",
+            "ввгнг",
+            "ввгнг ls",
+            "ввгнг лс",
+            "шввп",
+            "пвс",
+        ],
+    },
+    {"canonical": "провод", "aliases": ["провод", "провода", "проводов", "пугв"]},
+    {"canonical": "бита", "aliases": ["бита", "биты", "биту", "биты для шуруповерта"]},
+    {
+        "canonical": "держатель бит",
+        "aliases": ["держатель бит", "битодержатель", "магнитный держатель"],
+    },
+    {"canonical": "адаптер бит", "aliases": ["адаптер для бит", "адаптер бит"]},
+    {"canonical": "сверло", "aliases": ["сверло", "сверла", "сверел", "сверлить"]},
+    {"canonical": "бур", "aliases": ["бур", "буры", "сдс бур", "sds бур"]},
+    {
+        "canonical": "диск",
+        "aliases": [
+            "диск",
+            "диски",
+            "дисков",
+            "круг отрезной",
+            "отрезной круг",
+            "круг зачистной",
+            "зачистной круг",
+            "пильный круг",
+        ],
+    },
+    {
+        "canonical": "круг",
+        "aliases": ["круг шлифовальный", "шлифовальный круг", "круг на липучке"],
+    },
+    {"canonical": "дрель", "aliases": ["дрель", "дрели", "ударная дрель"]},
+    {"canonical": "лобзик", "aliases": ["лобзик", "электролобзик"]},
+    {"canonical": "фен", "aliases": ["строительный фен", "термофен", "фен"]},
+    {"canonical": "перчатки", "aliases": ["перчатки", "перчаток"]},
+    {"canonical": "изолента", "aliases": ["изолента", "изоляционная лента"]},
+    {"canonical": "скотч", "aliases": ["скотч", "малярный скотч"]},
+    {
+        "canonical": "стяжка",
+        "aliases": [
+            "стяжка",
+            "стяжки",
+            "кабельная стяжка",
+            "нейлоновая стяжка",
+            "хомут пластиковый",
+            "хомуты пластиковые",
+            "пластиковый хомут",
+            "пластиковые хомуты",
+        ],
+    },
+    {"canonical": "пена", "aliases": ["пена", "монтажная пена"]},
+    {"canonical": "герметик", "aliases": ["герметик", "герметики"]},
+    {"canonical": "нож", "aliases": ["нож", "строительный нож"]},
+    {"canonical": "лезвия", "aliases": ["лезвие", "лезвия", "сменные лезвия"]},
+    {"canonical": "рулетка", "aliases": ["рулетка", "рулетки"]},
+    {"canonical": "карандаш", "aliases": ["карандаш", "строительный карандаш"]},
+    {"canonical": "маркер", "aliases": ["маркер", "маркеры"]},
+    {"canonical": "респиратор", "aliases": ["респиратор", "респираторы"]},
+    {"canonical": "очки", "aliases": ["защитные очки", "очки"]},
+    {"canonical": "мешки", "aliases": ["мешок", "мешки", "мешки для мусора"]},
+    {"canonical": "кисть", "aliases": ["кисть", "кисти", "малярная кисть"]},
+    {"canonical": "валик", "aliases": ["валик", "валики", "малярный валик"]},
+    {"canonical": "ванночка", "aliases": ["ванночка", "малярная ванночка"]},
+    {"canonical": "удлинитель", "aliases": ["удлинитель", "удлинители"]},
+    {
+        "canonical": "шкурка",
+        "aliases": [
+            "шкурка",
+            "шлифовальная шкурка",
+            "наждачка",
+            "наждачная бумага",
+        ],
+    },
+    {"canonical": "лента фум", "aliases": ["лента фум", "фум лента", "фумка"]},
+    {
+        "canonical": "хомут",
+        "aliases": ["хомут", "хомуты", "червячный хомут", "металлический хомут"],
+    },
+    {
+        "canonical": "гипсокартон",
+        "aliases": [
+            "гкл",
+            "гипсокартон",
+            "гипсокартонный лист",
+            "лист гипсокартонный",
+        ],
+    },
+    {"canonical": "профиль", "aliases": ["профиль", "профили", "профиль для гкл"]},
+    {"canonical": "подвес", "aliases": ["подвес", "прямой подвес"]},
+    {"canonical": "соединитель", "aliases": ["соединитель", "краб", "соединитель краб"]},
+    {"canonical": "уголок", "aliases": ["уголок", "перфорированный уголок"]},
+    {"canonical": "лента серпянка", "aliases": ["серпянка", "лента серпянка"]},
+    {"canonical": "лента демпферная", "aliases": ["демпферная лента", "лента демпферная"]},
+    {"canonical": "болт", "aliases": ["болт", "болты", "болтов"]},
+    {"canonical": "гайка", "aliases": ["гайка", "гайки", "гаек"]},
+    {"canonical": "шайба", "aliases": ["шайба", "шайбы", "шайб"]},
+    {"canonical": "шпилька", "aliases": ["шпилька", "шпильки", "резьбовая шпилька"]},
+    {"canonical": "анкер", "aliases": ["анкер", "анкеры", "анкеров"]},
+    {"canonical": "гвозди", "aliases": ["гвоздь", "гвозди", "гвоздей"]},
+    {
+        "canonical": "дюбель",
+        "aliases": [
+            "дюбель",
+            "дюбели",
+            "дюбелей",
+            "дюбель гвоздь",
+            "дюбель-гвоздь",
+        ],
+    },
+    {"canonical": "ключ", "aliases": ["ключ", "ключи", "рожковый ключ", "разводной ключ"]},
+    {"canonical": "отвертка", "aliases": ["отвертка", "отвёртка", "отвертки"]},
+    {"canonical": "молоток", "aliases": ["молоток", "молотки"]},
+    {"canonical": "ножовка", "aliases": ["ножовка", "ножовки"]},
+    {"canonical": "плоскогубцы", "aliases": ["плоскогубцы"]},
+    {"canonical": "бокорезы", "aliases": ["бокорезы", "бокорез"]},
+    {"canonical": "пассатижи", "aliases": ["пассатижи", "клещи переставные"]},
+    {"canonical": "уровень", "aliases": ["уровень", "уровни"]},
+    {"canonical": "угольник", "aliases": ["угольник", "угольники"]},
+    {"canonical": "степлер", "aliases": ["степлер", "мебельный степлер"]},
+    {"canonical": "скобы", "aliases": ["скоба", "скобы", "скобы для степлера"]},
+]
+
+
+@dataclass(frozen=True, slots=True)
+class ProductTypeMatch:
+    canonical: str
+    score: float
+    exact: bool
+    alias: str
+    token_start: int
+
+
+@dataclass(frozen=True, slots=True)
+class _AliasRecord:
+    canonical: str
+    normalized: str
+    tokens: tuple[str, ...]
+
+
+class ProductAliasIndex:
+    def __init__(self) -> None:
+        alias_to_canonical: dict[str, str] = {}
+        canonical_to_aliases: dict[str, list[str]] = defaultdict(list)
+        records: list[_AliasRecord] = []
+        single_token_records: list[_AliasRecord] = []
+        trigram_postings: dict[str, set[int]] = defaultdict(set)
+
+        for item in PRODUCT_ALIASES:
+            canonical = normalize_text(item["canonical"])
+            for raw_alias in item["aliases"]:
+                alias = normalize_text(raw_alias)
+                previous = alias_to_canonical.get(alias)
+                if previous is not None and previous != canonical:
+                    raise ValueError(
+                        f"Alias {raw_alias!r} points to both {previous!r} and {canonical!r}"
+                    )
+                alias_to_canonical[alias] = canonical
+                canonical_to_aliases[canonical].append(alias)
+
+                record = _AliasRecord(canonical, alias, tokenize(alias, already_normalized=True))
+                records.append(record)
+                if len(record.tokens) == 1 and record.tokens[0].isalpha():
+                    record_index = len(single_token_records)
+                    single_token_records.append(record)
+                    for trigram in character_trigrams(record.tokens[0]):
+                        trigram_postings[trigram].add(record_index)
+
+        self.alias_to_canonical = alias_to_canonical
+        self.canonical_to_aliases = {
+            canonical: tuple(dict.fromkeys(aliases))
+            for canonical, aliases in canonical_to_aliases.items()
+        }
+        self._records = tuple(records)
+        self._single_token_records = tuple(single_token_records)
+        self._trigram_postings = {
+            trigram: frozenset(indexes) for trigram, indexes in trigram_postings.items()
+        }
+
+    def detect(self, text: str) -> ProductTypeMatch | None:
+        normalized = normalize_text(text)
+        tokens = tokenize(normalized, already_normalized=True)
+        exact = self._find_exact(tokens, prefix_only=False)
+        if exact is not None:
+            return exact
+        return self._find_fuzzy(tokens)
+
+    def detect_catalog_type(self, text: str) -> str:
+        normalized = normalize_text(text)
+        tokens = tokenize(normalized, already_normalized=True)
+        exact = self._find_exact(tokens, prefix_only=True)
+        if exact is None:
+            raise ValueError(f"No product type alias matches catalog name: {text!r}")
+        return exact.canonical
+
+    def aliases_for(self, canonical: str) -> tuple[str, ...]:
+        return self.canonical_to_aliases.get(canonical, ())
+
+    def _find_exact(
+        self, tokens: tuple[str, ...], *, prefix_only: bool
+    ) -> ProductTypeMatch | None:
+        matches: list[ProductTypeMatch] = []
+        for record in self._records:
+            alias_length = len(record.tokens)
+            if alias_length == 0 or alias_length > len(tokens):
+                continue
+            starts = (0,) if prefix_only else range(len(tokens) - alias_length + 1)
+            for start in starts:
+                if tokens[start : start + alias_length] == record.tokens:
+                    matches.append(
+                        ProductTypeMatch(
+                            canonical=record.canonical,
+                            score=1.0,
+                            exact=True,
+                            alias=record.normalized,
+                            token_start=start,
+                        )
+                    )
+                    break
+
+        if not matches:
+            return None
+        matches.sort(
+            key=lambda match: (
+                -len(tokenize(match.alias, already_normalized=True)),
+                match.token_start,
+                -len(match.alias),
+            )
+        )
+        return matches[0]
+
+    def _find_fuzzy(self, tokens: tuple[str, ...]) -> ProductTypeMatch | None:
+        query_words = word_tokens(meaningful_tokens(tokens))
+        best: ProductTypeMatch | None = None
+
+        for token_start, query_token in enumerate(query_words):
+            if len(query_token) < 4:
+                continue
+
+            candidate_indexes: set[int] = set()
+            for trigram in character_trigrams(query_token):
+                candidate_indexes.update(self._trigram_postings.get(trigram, ()))
+            if not candidate_indexes:
+                continue
+
+            for candidate_index in candidate_indexes:
+                record = self._single_token_records[candidate_index]
+                alias_token = record.tokens[0]
+                if abs(len(query_token) - len(alias_token)) > max(4, len(alias_token) // 2):
+                    continue
+                score = word_similarity(query_token, alias_token)
+                required = fuzzy_threshold(max(len(query_token), len(alias_token)))
+                if score < required:
+                    continue
+
+                match = ProductTypeMatch(
+                    canonical=record.canonical,
+                    score=score,
+                    exact=False,
+                    alias=record.normalized,
+                    token_start=token_start,
+                )
+                if best is None or (match.score, -match.token_start) > (
+                    best.score,
+                    -best.token_start,
+                ):
+                    best = match
+
+        return best

app/normalization.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections.abc import Iterable
+
+_DECIMAL_COMMA_RE = re.compile(r"(?<=\d),(?=\d)")
+_DIMENSION_SEPARATOR_RE = re.compile(r"(?<=\d)\s*[xх×*]\s*(?=\d)", re.IGNORECASE)
+_CYRILLIC_THREAD_RE = re.compile(r"(?<![a-zа-я0-9])м(?=\d)", re.IGNORECASE)
+_CYRILLIC_GRIT_RE = re.compile(r"(?<![a-zа-я0-9])р(?=\d)", re.IGNORECASE)
+_WHITESPACE_RE = re.compile(r"\s+")
+_PUNCTUATION_RE = re.compile(r"[^0-9a-zа-я.\-+x\s]", re.IGNORECASE)
+_TOKEN_RE = re.compile(
+    r"\d+(?:\.\d+)?(?:x\d+(?:\.\d+)?)+"
+    r"|[a-zа-я]+(?:\d+(?:\.\d+)?[a-zа-я]*)?"
+    r"|\d+(?:\.\d+)?",
+    re.IGNORECASE,
+)
+
+# These tokens stay in the normalized message but do not affect product scoring.
+ZERO_WEIGHT_TOKENS = frozenset(
+    {
+        "а",
+        "без",
+        "бы",
+        "в",
+        "вам",
+        "ваш",
+        "вот",
+        "где",
+        "дайте",
+        "для",
+        "до",
+        "есть",
+        "еще",
+        "за",
+        "здравствуйте",
+        "и",
+        "из",
+        "как",
+        "какие",
+        "какой",
+        "какая",
+        "который",
+        "ли",
+        "мне",
+        "можно",
+        "на",
+        "надо",
+        "нужен",
+        "нужна",
+        "нужны",
+        "нужно",
+        "по",
+        "подскажите",
+        "посоветуйте",
+        "при",
+        "сколько",
+        "только",
+        "у",
+        "хочу",
+        "что",
+        "шт",
+        "штук",
+        "вопрос",
+    }
+)
+
+UNIT_TOKENS = frozenset(
+    {
+        "в",
+        "вольт",
+        "вольта",
+        "вольтов",
+        "вт",
+        "ватт",
+        "ватта",
+        "ваттов",
+        "мм",
+        "миллиметр",
+        "миллиметра",
+        "миллиметров",
+        "м",
+        "метр",
+        "метра",
+        "метров",
+        "см",
+        "г",
+        "кг",
+        "л",
+        "мл",
+        "дж",
+    }
+)
+
+
+def normalize_text(text: str) -> str:
+    """Create one deterministic representation for aliases and catalog text.
+
+    The Cyrillic letter ``х`` is converted only when it is a multiplication
+    separator between numbers. A global replacement would corrupt normal words
+    such as ``хомут`` and ``находится``.
+    """
+
+    value = unicodedata.normalize("NFKC", text or "").lower().replace("ё", "е")
+    value = value.replace("–", "-").replace("—", "-").replace("−", "-")
+    value = _DECIMAL_COMMA_RE.sub(".", value)
+    value = _DIMENSION_SEPARATOR_RE.sub("x", value)
+    value = _CYRILLIC_THREAD_RE.sub("m", value)
+    value = _CYRILLIC_GRIT_RE.sub("p", value)
+    value = _PUNCTUATION_RE.sub(" ", value)
+    return _WHITESPACE_RE.sub(" ", value).strip()
+
+
+def tokenize(text: str, *, already_normalized: bool = False) -> tuple[str, ...]:
+    value = text if already_normalized else normalize_text(text)
+    return tuple(match.group(0) for match in _TOKEN_RE.finditer(value))
+
+
+def meaningful_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
+    return tuple(
+        token
+        for token in tokens
+        if token not in ZERO_WEIGHT_TOKENS and token not in UNIT_TOKENS
+    )
+
+
+def word_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
+    return tuple(token for token in tokens if any(character.isalpha() for character in token))
+
+
+def normalize_code(value: str) -> str:
+    normalized = normalize_text(value)
+    return "".join(character for character in normalized if character.isalnum())

app/typo.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+
+@lru_cache(maxsize=16_384)
+def damerau_levenshtein_distance(left: str, right: str) -> int:
+    """Return unrestricted Damerau-Levenshtein distance.
+
+    In addition to insertions, deletions and substitutions, adjacent character
+    transpositions cost one operation. The implementation is the unrestricted
+    variant, not the more limited optimal-string-alignment shortcut.
+    """
+
+    if left == right:
+        return 0
+    if not left:
+        return len(right)
+    if not right:
+        return len(left)
+
+    left_length = len(left)
+    right_length = len(right)
+    maximum_distance = left_length + right_length
+    matrix = [
+        [0 for _ in range(right_length + 2)] for _ in range(left_length + 2)
+    ]
+    matrix[0][0] = maximum_distance
+
+    for left_index in range(left_length + 1):
+        matrix[left_index + 1][0] = maximum_distance
+        matrix[left_index + 1][1] = left_index
+    for right_index in range(right_length + 1):
+        matrix[0][right_index + 1] = maximum_distance
+        matrix[1][right_index + 1] = right_index
+
+    last_row_by_character: dict[str, int] = {}
+    for left_index in range(1, left_length + 1):
+        last_matching_column = 0
+        for right_index in range(1, right_length + 1):
+            matching_row = last_row_by_character.get(right[right_index - 1], 0)
+            matching_column = last_matching_column
+            substitution_cost = 1
+
+            if left[left_index - 1] == right[right_index - 1]:
+                substitution_cost = 0
+                last_matching_column = right_index
+
+            matrix[left_index + 1][right_index + 1] = min(
+                matrix[left_index][right_index] + substitution_cost,
+                matrix[left_index + 1][right_index] + 1,
+                matrix[left_index][right_index + 1] + 1,
+                matrix[matching_row][matching_column]
+                + (left_index - matching_row - 1)
+                + 1
+                + (right_index - matching_column - 1),
+            )
+        last_row_by_character[left[left_index - 1]] = left_index
+
+    return matrix[left_length + 1][right_length + 1]
+
+
+def damerau_similarity(left: str, right: str) -> float:
+    if left == right:
+        return 1.0
+    maximum_length = max(len(left), len(right))
+    if maximum_length == 0:
+        return 1.0
+    return 1.0 - damerau_levenshtein_distance(left, right) / maximum_length
+
+
+@lru_cache(maxsize=32_768)
+def character_trigrams(value: str) -> frozenset[str]:
+    padded = f"^{value}$"
+    if len(padded) <= 3:
+        return frozenset({padded})
+    return frozenset(padded[index : index + 3] for index in range(len(padded) - 2))
+
+
+def trigram_dice(left: str, right: str) -> float:
+    left_trigrams = character_trigrams(left)
+    right_trigrams = character_trigrams(right)
+    denominator = len(left_trigrams) + len(right_trigrams)
+    if denominator == 0:
+        return 1.0
+    return 2.0 * len(left_trigrams & right_trigrams) / denominator
+
+
+@lru_cache(maxsize=16_384)
+def lcs_similarity(left: str, right: str) -> float:
+    maximum_length = max(len(left), len(right))
+    if maximum_length == 0:
+        return 1.0
+
+    previous = [0] * (len(right) + 1)
+    for left_character in left:
+        current = [0]
+        for right_index, right_character in enumerate(right, start=1):
+            if left_character == right_character:
+                current.append(previous[right_index - 1] + 1)
+            else:
+                current.append(max(previous[right_index], current[-1]))
+        previous = current
+    return previous[-1] / maximum_length
+
+
+def word_similarity(left: str, right: str) -> float:
+    if left == right:
+        return 1.0
+    if not left or not right:
+        return 0.0
+
+    shorter_length = min(len(left), len(right))
+    edit = damerau_similarity(left, right)
+
+    if shorter_length <= 4:
+        # Trigrams are unstable for very short words; edit distance is the main
+        # signal and exact/prefix matching is handled before this function.
+        return edit
+
+    trigram = trigram_dice(left, right)
+    lcs = lcs_similarity(left, right)
+    return edit * 0.60 + trigram * 0.30 + lcs * 0.10
+
+
+def fuzzy_threshold(token_length: int) -> float:
+    if token_length <= 4:
+        return 0.84
+    if token_length <= 7:
+        return 0.72
+    if token_length <= 10:
+        return 0.61
+    return 0.53