135 lines
3.6 KiB
Python
135 lines
3.6 KiB
Python
from __future__ import annotations
|
||
|
||
import re
|
||
import unicodedata
|
||
from collections.abc import Iterable
|
||
|
||
_DECIMAL_COMMA_RE = re.compile(r"(?<=\d),(?=\d)")
|
||
_DIMENSION_SEPARATOR_RE = re.compile(r"(?<=\d)\s*[xх×*]\s*(?=\d)", re.IGNORECASE)
|
||
_CYRILLIC_THREAD_RE = re.compile(r"(?<![a-zа-я0-9])м(?=\d)", re.IGNORECASE)
|
||
_CYRILLIC_GRIT_RE = re.compile(r"(?<![a-zа-я0-9])р(?=\d)", re.IGNORECASE)
|
||
_WHITESPACE_RE = re.compile(r"\s+")
|
||
_PUNCTUATION_RE = re.compile(r"[^0-9a-zа-я.\-+x\s]", re.IGNORECASE)
|
||
_TOKEN_RE = re.compile(
|
||
r"\d+(?:\.\d+)?(?:x\d+(?:\.\d+)?)+"
|
||
r"|[a-zа-я]+(?:\d+(?:\.\d+)?[a-zа-я]*)?"
|
||
r"|\d+(?:\.\d+)?",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
# These tokens stay in the normalized message but do not affect product scoring.
|
||
ZERO_WEIGHT_TOKENS = frozenset(
|
||
{
|
||
"а",
|
||
"без",
|
||
"бы",
|
||
"в",
|
||
"вам",
|
||
"ваш",
|
||
"вот",
|
||
"где",
|
||
"дайте",
|
||
"для",
|
||
"до",
|
||
"есть",
|
||
"еще",
|
||
"за",
|
||
"здравствуйте",
|
||
"и",
|
||
"из",
|
||
"как",
|
||
"какие",
|
||
"какой",
|
||
"какая",
|
||
"который",
|
||
"ли",
|
||
"мне",
|
||
"можно",
|
||
"на",
|
||
"надо",
|
||
"нужен",
|
||
"нужна",
|
||
"нужны",
|
||
"нужно",
|
||
"по",
|
||
"подскажите",
|
||
"посоветуйте",
|
||
"при",
|
||
"сколько",
|
||
"только",
|
||
"у",
|
||
"хочу",
|
||
"что",
|
||
"шт",
|
||
"штук",
|
||
"вопрос",
|
||
}
|
||
)
|
||
|
||
UNIT_TOKENS = frozenset(
|
||
{
|
||
"в",
|
||
"вольт",
|
||
"вольта",
|
||
"вольтов",
|
||
"вт",
|
||
"ватт",
|
||
"ватта",
|
||
"ваттов",
|
||
"мм",
|
||
"миллиметр",
|
||
"миллиметра",
|
||
"миллиметров",
|
||
"м",
|
||
"метр",
|
||
"метра",
|
||
"метров",
|
||
"см",
|
||
"г",
|
||
"кг",
|
||
"л",
|
||
"мл",
|
||
"дж",
|
||
}
|
||
)
|
||
|
||
|
||
def normalize_text(text: str) -> str:
|
||
"""Create one deterministic representation for aliases and catalog text.
|
||
|
||
The Cyrillic letter ``х`` is converted only when it is a multiplication
|
||
separator between numbers. A global replacement would corrupt normal words
|
||
such as ``хомут`` and ``находится``.
|
||
"""
|
||
|
||
value = unicodedata.normalize("NFKC", text or "").lower().replace("ё", "е")
|
||
value = value.replace("–", "-").replace("—", "-").replace("−", "-")
|
||
value = _DECIMAL_COMMA_RE.sub(".", value)
|
||
value = _DIMENSION_SEPARATOR_RE.sub("x", value)
|
||
value = _CYRILLIC_THREAD_RE.sub("m", value)
|
||
value = _CYRILLIC_GRIT_RE.sub("p", value)
|
||
value = _PUNCTUATION_RE.sub(" ", value)
|
||
return _WHITESPACE_RE.sub(" ", value).strip()
|
||
|
||
|
||
def tokenize(text: str, *, already_normalized: bool = False) -> tuple[str, ...]:
|
||
value = text if already_normalized else normalize_text(text)
|
||
return tuple(match.group(0) for match in _TOKEN_RE.finditer(value))
|
||
|
||
|
||
def meaningful_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
|
||
return tuple(
|
||
token
|
||
for token in tokens
|
||
if token not in ZERO_WEIGHT_TOKENS and token not in UNIT_TOKENS
|
||
)
|
||
|
||
|
||
def word_tokens(tokens: Iterable[str]) -> tuple[str, ...]:
|
||
return tuple(token for token in tokens if any(character.isalpha() for character in token))
|
||
|
||
|
||
def normalize_code(value: str) -> str:
|
||
normalized = normalize_text(value)
|
||
return "".join(character for character in normalized if character.isalnum())
|