Add normalization and product alias indexes

2026-08-07 13:05:58 +00:00
parent daf107feed
commit 8e1e0f82a2
3 changed files with 592 additions and 0 deletions
@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections.abc import Iterable
+
+_DECIMAL_COMMA_RE = re.compile(r"(?<=\d),(?=\d)")
+_DIMENSION_SEPARATOR_RE = re.compile(r"(?<=\d)\s*[xх×*]\s*(?=\d)", re.IGNORECASE)
+_CYRILLIC_THREAD_RE = re.compile(r"(?<![a-zа-я0-9])м(?=\d)", re.IGNORECASE)
+_CYRILLIC_GRIT_RE = re.compile(r"(?<![a-zа-я0-9])р(?=\d)", re.IGNORECASE)
+_WHITESPACE_RE = re.compile(r"\s+")
+_PUNCTUATION_RE = re.compile(r"[^0-9a-zа-я.\-+x\s]", re.IGNORECASE)
+_TOKEN_RE = re.compile(
+    r"\d+(?:\.\d+)?(?:x\d+(?:\.\d+)?)+"
+    r"|[a-zа-я]+(?:\d+(?:\.\d+)?[a-zа-я]*)?"
+    r"|\d+(?:\.\d+)?",
+    re.IGNORECASE,
+)
+
+# These tokens stay in the normalized message but do not affect product scoring.
+ZERO_WEIGHT_TOKENS = frozenset(
+    {
+        "а",
+        "без",
+        "бы",
+        "в",
+        "вам",
+        "ваш",
+        "вот",
+        "где",
+        "дайте",
+        "для",
+        "до",
+        "есть",
+        "еще",
+        "за",
+        "здравствуйте",
+        "и",
+        "из",
+        "как",
+        "какие",
+        "какой",
+        "какая",
+        "который",
+        "ли",
+        "мне",
+        "можно",
+        "на",
+        "надо",
+        "нужен",
+        "нужна",
+        "нужны",
+        "нужно",
+        "по",
+        "подскажите",
+        "посоветуйте",
+        "при",
+        "сколько",
+        "только",
+        "у",
+        "хочу",
+        "что",
+        "шт",
+        "штук",
+        "вопрос",
+    }
+)
+
+UNIT_TOKENS = frozenset(
+    {
+        "в",
+        "вольт",
+        "вольта",
+        "вольтов",
+        "вт",
+        "ватт",
+        "ватта",
+        "ваттов",
+        "мм",
+        "миллиметр",
+        "миллиметра",
+        "миллиметров",
+        "м",
+        "метр",
+        "метра",
+        "метров",
+        "см",
+        "г",
+        "кг",
+        "л",
+        "мл",
+        "дж",
+    }
+)
+
+
+def normalize_text(text: str) -> str:
+    """Create one deterministic representation for aliases and catalog text.
+
+    The Cyrillic letter ``х`` is converted only when it is a multiplication
+    separator between numbers. A global replacement would corrupt normal words
+    such as ``хомут`` and ``находится``.
+    """
+
+    value = unicodedata.normalize("NFKC", text or "").lower().replace("ё", "е")
+    value = value.replace("–", "-").replace("—", "-").replace("−", "-")
+    value = _DECIMAL_COMMA_RE.sub(".", value)
+    value = _DIMENSION_SEPARATOR_RE.sub("x", value)
+    value = _CYRILLIC_THREAD_RE.sub("m", value)
+    value = _CYRILLIC_GRIT_RE.sub("p", value)
+    value = _PUNCTUATION_RE.sub(" ", value)
+    return _WHITESPACE_RE.sub(" ", value).strip()
+
+
+def tokenize(text: str, *, already_normalized: bool = False) -> tuple[str, ...]:
+    value = text if already_normalized else normalize_text(text)
+    return tuple(match.group(0) for match in _TOKEN_RE.finditer(value))
+
+
+def meaningful_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
+    return tuple(
+        token
+        for token in tokens
+        if token not in ZERO_WEIGHT_TOKENS and token not in UNIT_TOKENS
+    )
+
+
+def word_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
+    return tuple(token for token in tokens if any(character.isalpha() for character in token))
+
+
+def normalize_code(value: str) -> str:
+    normalized = normalize_text(value)
+    return "".join(character for character in normalized if character.isalnum())