from __future__ import annotations import re import unicodedata from collections.abc import Iterable _DECIMAL_COMMA_RE = re.compile(r"(?<=\d),(?=\d)") _DIMENSION_SEPARATOR_RE = re.compile(r"(?<=\d)\s*[xх×*]\s*(?=\d)", re.IGNORECASE) _CYRILLIC_THREAD_RE = re.compile(r"(? str: """Create one deterministic representation for aliases and catalog text. The Cyrillic letter ``х`` is converted only when it is a multiplication separator between numbers. A global replacement would corrupt normal words such as ``хомут`` and ``находится``. """ value = unicodedata.normalize("NFKC", text or "").lower().replace("ё", "е") value = value.replace("–", "-").replace("—", "-").replace("−", "-") value = _DECIMAL_COMMA_RE.sub(".", value) value = _DIMENSION_SEPARATOR_RE.sub("x", value) value = _CYRILLIC_THREAD_RE.sub("m", value) value = _CYRILLIC_GRIT_RE.sub("p", value) value = _PUNCTUATION_RE.sub(" ", value) return _WHITESPACE_RE.sub(" ", value).strip() def tokenize(text: str, *, already_normalized: bool = False) -> tuple[str, ...]: value = text if already_normalized else normalize_text(text) return tuple(match.group(0) for match in _TOKEN_RE.finditer(value)) def meaningful_tokens(tokens: Iterable[str]) -> tuple[str, ...]: return tuple( token for token in tokens if token not in ZERO_WEIGHT_TOKENS and token not in UNIT_TOKENS ) def word_tokens(tokens: Iterable[str]) -> tuple[str, ...]: return tuple(token for token in tokens if any(character.isalpha() for character in token)) def normalize_code(value: str) -> str: normalized = normalize_text(value) return "".join(character for character in normalized if character.isalnum())