Files
test-search-engine/app/attributes.py
T

698 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import re
from dataclasses import dataclass
from decimal import Decimal
from app.normalization import normalize_code, normalize_text, tokenize
from app.typo import fuzzy_threshold, word_similarity
_NUMBER = r"\d+(?:\.\d+)?"
_DIMENSIONS_RE = re.compile(
rf"(?<![a-zа-я0-9.])({_NUMBER}(?:x{_NUMBER}){{1,3}})(?![a-zа-я0-9.])",
re.IGNORECASE,
)
_PAIR_WITH_NA_RE = re.compile(rf"\b({_NUMBER})\s+на\s+({_NUMBER})\b", re.IGNORECASE)
_MM_RE = re.compile(rf"\b({_NUMBER})\s*(?:мм|миллиметр\w*)\b", re.IGNORECASE)
_METER_RE = re.compile(rf"\b({_NUMBER})\s*(?:м|метр\w*)\b", re.IGNORECASE)
_THREAD_RE = re.compile(rf"(?<![a-zа-я0-9])m\s*({_NUMBER})(?=x|\b)", re.IGNORECASE)
_THREAD_LENGTH_RE = re.compile(rf"(?<![a-zа-я0-9])m\s*({_NUMBER})x({_NUMBER})", re.IGNORECASE)
_PROFILE_RE = re.compile(
rf"(?<![a-z0-9])(ph\d+|pz\d+|t\d+|sl{_NUMBER})(?=x|[^a-z0-9]|$)",
re.IGNORECASE,
)
_PROFILE_LENGTH_RE = re.compile(
rf"(?<![a-z0-9])(ph\d+|pz\d+|t\d+|sl{_NUMBER})x({_NUMBER})",
re.IGNORECASE,
)
_GRIT_RE = re.compile(r"(?<![a-z0-9])p\s*(\d{2,4})(?![a-z0-9])", re.IGNORECASE)
_VOLTAGE_RE = re.compile(
rf"(?<![a-zа-я0-9-])({_NUMBER})\s*(?:v|в|вольт\w*)\b", re.IGNORECASE
)
_POWER_RE = re.compile(rf"\b({_NUMBER})\s*(?:w|вт|ватт\w*)\b", re.IGNORECASE)
_TEETH_RE = re.compile(r"\b(\d+)\s*(?:зуб(?:ьев|а|ов)?|[zt])\b", re.IGNORECASE)
_SAW_COMPACT_RE = re.compile(
rf"(?<![a-zа-я0-9.])({_NUMBER})(?:x|\s+)(\d+)\s*[zt]\b",
re.IGNORECASE,
)
_PACK_SIZE_RE = re.compile(
r"(?:пачк\w*|упаковк\w*|\bуп\.?)(?:\s+(?:по|на))?\s*(\d+)\s*(?:шт\w*)?",
re.IGNORECASE,
)
_CATALOG_PACK_RE = re.compile(r"\bуп\.?\s*(\d+)\s*шт", re.IGNORECASE)
_MODEL_RE = re.compile(
r"(?<![a-zа-я0-9])([a-zа-я]{2,6}-\d+[a-zа-я]?|[a-zа-я]{2,6}\d+[a-zа-я]?)(?![a-zа-я0-9])",
re.IGNORECASE,
)
_NORMALIZED_MODEL_RE = re.compile(r"^([a-zа-я]{2,6})(\d+)([a-zа-я]?)$", re.IGNORECASE)
_PAIR_DIMENSION_TYPES = frozenset(
{
"саморез",
"бур",
"дюбель",
"анкер",
"гвозди",
"труба",
"болт",
"шпилька",
"стяжка",
}
)
_SINGLE_SIZE_AFTER_NA_TYPES = frozenset(
{"ушм", "диск", "круг", "сверло", "бита", "уровень", "лента фум"}
)
BRAND_ALIASES = [
{
"canonical": "prowerk",
"aliases": ["prowerk", "проверк", "проуверк", "проуорк"],
},
{
"canonical": "техноресурс",
"aliases": ["техноресурс", "техно ресурс"],
},
{
"canonical": "арсенал-т",
"aliases": ["арсенал-т", "арсенал т", "арсенал"],
},
{
"canonical": "toolkraft",
"aliases": ["toolkraft", "тулкрафт", "тул крафт"],
},
{
"canonical": "makita",
"aliases": ["makita", "макита", "макиты", "макиту"],
},
{"canonical": "bosch", "aliases": ["bosch", "бош", "боша"]},
]
@dataclass(frozen=True, slots=True)
class BrandMatch:
canonical: str
score: float
alias: str
token_start: int
class BrandIndex:
def __init__(self) -> None:
records: list[tuple[str, str, tuple[str, ...]]] = []
for item in BRAND_ALIASES:
canonical = normalize_text(item["canonical"])
for raw_alias in item["aliases"]:
alias = normalize_text(raw_alias)
records.append((canonical, alias, tokenize(alias, already_normalized=True)))
self._records = tuple(records)
def detect(self, text: str) -> BrandMatch | None:
normalized = normalize_text(text)
tokens = tokenize(normalized, already_normalized=True)
exact_matches: list[BrandMatch] = []
for canonical, alias, alias_tokens in self._records:
for start in range(len(tokens) - len(alias_tokens) + 1):
if tokens[start : start + len(alias_tokens)] == alias_tokens:
exact_matches.append(BrandMatch(canonical, 1.0, alias, start))
break
if exact_matches:
exact_matches.sort(key=lambda item: (-len(tokenize(item.alias)), item.token_start))
return exact_matches[0]
best: BrandMatch | None = None
for start, query_token in enumerate(tokens):
if len(query_token) < 5 or not query_token.isalpha():
continue
for canonical, alias, alias_tokens in self._records:
if len(alias_tokens) != 1:
continue
alias_token = alias_tokens[0]
if abs(len(query_token) - len(alias_token)) > 3:
continue
score = word_similarity(query_token, alias_token)
if score < max(0.70, fuzzy_threshold(max(len(query_token), len(alias_token)))):
continue
candidate = BrandMatch(canonical, score, alias, start)
if best is None or candidate.score > best.score:
best = candidate
return best
@dataclass(frozen=True, slots=True)
class ParsedAttributes:
dimensions: tuple[Decimal, ...] = ()
mm_values: tuple[Decimal, ...] = ()
meter_values: tuple[Decimal, ...] = ()
thread_size: Decimal | None = None
thread_length_mm: Decimal | None = None
profile: str | None = None
grit: int | None = None
voltage_v: Decimal | None = None
power_w: Decimal | None = None
teeth: int | None = None
pack_size: int | None = None
unit_hint: str | None = None
brand: str | None = None
brand_required: bool = False
model_codes: frozenset[str] = frozenset()
sds_type: str | None = None
variant: str | None = None
cheap_preference: bool = False
@property
def specificity(self) -> int:
return sum(
(
bool(self.dimensions),
bool(self.mm_values),
bool(self.meter_values),
self.thread_size is not None,
self.thread_length_mm is not None,
self.profile is not None,
self.grit is not None,
self.voltage_v is not None,
self.power_w is not None,
self.teeth is not None,
self.pack_size is not None,
self.unit_hint is not None,
self.brand is not None and self.brand_required,
bool(self.model_codes),
self.sds_type is not None,
self.variant is not None,
)
)
@dataclass(frozen=True, slots=True)
class Compatibility:
compatible: bool
matched_fields: int
total_fields: int
conflict: str | None = None
@property
def score(self) -> float:
if self.total_fields == 0:
return 1.0
return self.matched_fields / self.total_fields
def _decimal(value: str) -> Decimal:
return Decimal(value).normalize()
def _unique_decimals(values: list[Decimal]) -> tuple[Decimal, ...]:
return tuple(dict.fromkeys(values))
def _extract_dimensions(normalized: str) -> tuple[Decimal, ...]:
match = _DIMENSIONS_RE.search(normalized)
if match is None:
return ()
return tuple(_decimal(part) for part in match.group(1).split("x"))
def _extract_model_codes(normalized: str) -> frozenset[str]:
excluded_prefixes = ("ph", "pz", "sl", "ffp", "din")
codes: set[str] = set()
for match in _MODEL_RE.finditer(normalized):
raw = match.group(1)
code = normalize_code(raw)
if code.startswith(excluded_prefixes):
# DIN/FFP are useful exact terms for lexical ranking, but they are
# standards/classes rather than unique product models.
continue
if re.fullmatch(r"(?:t|m|p)\d+(?:\d+)?", code):
continue
codes.add(code)
return frozenset(codes)
def model_code_matches(query_code: str, candidate_code: str) -> bool:
"""Match a complete model or a model with only its final letter omitted.
Numeric parts remain exact. Thus ``арс12`` may complete to
``арс12л``, but it must not match the different model ``арс125``.
"""
if query_code == candidate_code:
return True
query_match = _NORMALIZED_MODEL_RE.fullmatch(query_code)
candidate_match = _NORMALIZED_MODEL_RE.fullmatch(candidate_code)
if query_match is None or candidate_match is None:
return False
query_prefix, query_number, query_suffix = query_match.groups()
candidate_prefix, candidate_number, candidate_suffix = candidate_match.groups()
return (
query_prefix == candidate_prefix
and query_number == candidate_number
and not query_suffix
and bool(candidate_suffix)
)
def _detect_sds(normalized: str) -> str | None:
flattened = normalized.replace("-", " ")
if "sds max" in flattened or "сдс макс" in flattened:
return "sds-max"
if "sds plus" in flattened or "сдс плюс" in flattened:
return "sds-plus"
tokens = set(tokenize(flattened, already_normalized=True))
if "sds" in tokens or "сдс" in tokens:
return "sds"
return None
def _detect_variant(
normalized: str, product_type: str, *, catalog_item: bool
) -> str | None:
text = f" {normalized.replace('-', ' ')} "
if product_type == "саморез":
is_gkl = " гкл " in text or " гипсокартон" in text
has_wood = "по дерев" in text or "гипсокартон дерев" in text
has_metal = "по металл" in text or "гипсокартон металл" in text
if is_gkl and has_wood:
return "gkl_wood"
if is_gkl and has_metal:
return "gkl_metal"
if is_gkl:
return "gkl"
if has_wood:
return "wood"
if has_metal:
return "metal"
if product_type == "кабель":
if " шввп " in text:
return "shvvp"
if " пвс " in text:
return "pvs"
if " ввг" in text:
if " ls " in text or " лс " in text:
return "vvg_ls"
return "vvg"
if product_type == "провод" and " пугв " in text:
return "pugv"
if product_type == "диск":
if "пильн" in text:
return "saw_wood"
if "зачист" in text:
return "grinding"
if "лепест" in text:
return "flap"
if "отрез" in text:
if "нержав" in text:
return "cut_stainless"
if "камн" in text:
return "cut_stone"
if "металл" in text:
return "cut_metal"
return "cut"
if product_type == "сверло":
if "по металл" in text:
return "metal"
if "по дерев" in text:
return "wood"
if "по бетон" in text:
return "concrete"
if product_type == "перчатки":
for marker, variant in (
("нитрил", "nitrile"),
("спилк", "split_leather"),
("прорезин", "rubberized"),
("латекс", "latex"),
("полиуретан", "polyurethane"),
("пвх", "pvc"),
):
if marker in text:
return variant
if product_type == "дюбель":
if "гвозд" in text:
return "nail"
if "распор" in text:
return "spacer"
if product_type == "шайба":
if "гровер" in text:
return "spring"
if "плоск" in text:
return "flat"
if product_type == "анкер":
if "клинов" in text:
return "wedge"
if "с гайк" in text:
return "with_nut"
if product_type == "ключ":
if "разводн" in text:
return "adjustable"
if "рожков" in text:
return "open_end"
if product_type == "гипсокартон":
if "влагостой" in text:
return "moisture"
if "огнестой" in text:
return "fire"
if catalog_item:
return "standard"
if "обычн" in text or "стандартн" in text:
return "standard"
if product_type == "профиль":
if "направля" in text or " пн " in text:
return "guide"
if "стоеч" in text or " пс " in text:
return "stud"
if "потолоч" in text or " пп " in text:
return "ceiling"
if product_type == "герметик":
if "силикон" in text:
return "silicone"
if "акрил" in text:
return "acrylic"
if product_type == "пена":
if "профессион" in text:
return "professional"
if "бытов" in text:
return "household"
if product_type == "хомут" and "червяч" in text:
return "worm"
return None
def parse_attributes(
text: str,
product_type: str,
*,
unit: str | None = None,
catalog_item: bool = False,
brand_index: BrandIndex | None = None,
) -> ParsedAttributes:
normalized = normalize_text(text)
dimensions = _extract_dimensions(normalized)
compact_saw_match = _SAW_COMPACT_RE.search(normalized)
teeth_match = _TEETH_RE.search(normalized)
teeth = (
int(compact_saw_match.group(2))
if compact_saw_match is not None
else int(teeth_match.group(1))
if teeth_match is not None
else None
)
if not dimensions and product_type in _PAIR_DIMENSION_TYPES:
pair_match = _PAIR_WITH_NA_RE.search(normalized)
if pair_match is not None:
dimensions = (_decimal(pair_match.group(1)), _decimal(pair_match.group(2)))
if product_type == "труба" and len(dimensions) == 2:
wall_match = re.search(rf"\bстенк\w*\s+({_NUMBER}|полтора)\b", normalized)
if wall_match is not None:
wall_value = Decimal("1.5") if wall_match.group(1) == "полтора" else _decimal(
wall_match.group(1)
)
dimensions = (*dimensions, wall_value)
mm_values = [_decimal(match.group(1)) for match in _MM_RE.finditer(normalized)]
meter_values = [_decimal(match.group(1)) for match in _METER_RE.finditer(normalized)]
if compact_saw_match is not None:
mm_values.append(_decimal(compact_saw_match.group(1)))
# ``190 на 48 зубьев`` describes diameter and tooth count, not 190x48.
saw_pair = re.search(
rf"\b({_NUMBER})\s+на\s+\d+\s*(?:зуб(?:ьев|а|ов)?)\b",
normalized,
)
if saw_pair is not None:
mm_values.append(_decimal(saw_pair.group(1)))
if (
not catalog_item
and product_type in _SINGLE_SIZE_AFTER_NA_TYPES
and saw_pair is None
):
after_na = re.search(rf"\bна\s+({_NUMBER})(?!\s*(?:в|вольт|вт|ватт))", normalized)
if after_na is not None:
value = _decimal(after_na.group(1))
if value not in mm_values:
mm_values.append(value)
if not dimensions and not catalog_item and product_type == "гипсокартон":
# Colloquial requests often contain only sheet thickness: ``гкл 9.5``.
bare_number = re.search(rf"(?<![a-zа-я0-9])({_NUMBER})(?![a-zа-я0-9])", normalized)
if bare_number is not None:
dimensions = (_decimal(bare_number.group(1)),)
if not catalog_item and "метров" in normalized and not meter_values:
meter_values.append(Decimal("1"))
if not catalog_item and "метровая" in normalized and not meter_values:
meter_values.append(Decimal("1"))
pack_match = (_CATALOG_PACK_RE if catalog_item else _PACK_SIZE_RE).search(normalized)
pack_size = int(pack_match.group(1)) if pack_match else None
bare_size_types = {
"ушм",
"диск",
"круг",
"сверло",
"бита",
"уровень",
"лента фум",
"нож",
"лезвия",
"кисть",
"валик",
}
if (
not catalog_item
and product_type in bare_size_types
and not dimensions
and not mm_values
):
bare_numbers = [
_decimal(match.group(1))
for match in re.finditer(
rf"(?<![a-zа-я0-9.\-])({_NUMBER})(?![a-zа-я0-9.])", normalized
)
]
if teeth is not None:
bare_numbers = [value for value in bare_numbers if value != Decimal(teeth)]
if pack_size is not None:
bare_numbers = [value for value in bare_numbers if value != Decimal(pack_size)]
if bare_numbers:
mm_values.append(bare_numbers[0])
thread_match = _THREAD_RE.search(normalized)
thread_size = _decimal(thread_match.group(1)) if thread_match else None
thread_length_match = _THREAD_LENGTH_RE.search(normalized)
thread_length_mm = (
_decimal(thread_length_match.group(2)) if thread_length_match else None
)
if thread_length_mm is None and product_type == "шпилька" and Decimal("1") in meter_values:
thread_length_mm = Decimal("1000")
profile_match = _PROFILE_RE.search(normalized)
profile = profile_match.group(1).upper() if profile_match else None
profile_length_match = _PROFILE_LENGTH_RE.search(normalized)
if profile_length_match is not None:
profile = profile_length_match.group(1).upper()
profile_length = _decimal(profile_length_match.group(2))
if profile_length not in mm_values:
mm_values.append(profile_length)
grit_match = _GRIT_RE.search(normalized)
grit = int(grit_match.group(1)) if grit_match else None
voltage_match = _VOLTAGE_RE.search(normalized)
voltage_v = _decimal(voltage_match.group(1)) if voltage_match else None
power_match = _POWER_RE.search(normalized)
power_w = _decimal(power_match.group(1)) if power_match else None
unit_hint: str | None = None
if not catalog_item:
if re.search(r"\b(?:пачк\w*|упаковк\w*|уп\.)\b", normalized):
unit_hint = "уп"
elif re.search(r"\b(?:листами|листах|лист)\b", normalized):
unit_hint = "лист"
elif re.search(r"\b(?:за\s+метр|погонн\w*\s+метр)\b", normalized):
unit_hint = "м"
elif re.search(r"\b(?:килограмм\w*|кг)\b", normalized):
unit_hint = "кг"
elif unit:
unit_hint = normalize_text(unit)
active_brand_index = brand_index or BrandIndex()
brand_match = active_brand_index.detect(normalized)
brand = brand_match.canonical if brand_match else None
brand_required = False
if brand is not None and not catalog_item:
reference_markers = ("как у", "как ", "аналог", "вместо", "только дешевле")
brand_required = not any(marker in normalized for marker in reference_markers)
cheap_preference = bool(re.search(r"\b(?:дешев\w*|недорог\w*|бюджет\w*)\b", normalized))
return ParsedAttributes(
dimensions=dimensions,
mm_values=_unique_decimals(mm_values),
meter_values=_unique_decimals(meter_values),
thread_size=thread_size,
thread_length_mm=thread_length_mm,
profile=profile,
grit=grit,
voltage_v=voltage_v,
power_w=power_w,
teeth=teeth,
pack_size=pack_size,
unit_hint=unit_hint,
brand=brand,
brand_required=brand_required,
model_codes=_extract_model_codes(normalized),
sds_type=_detect_sds(normalized),
variant=_detect_variant(normalized, product_type, catalog_item=catalog_item),
cheap_preference=cheap_preference,
)
def _variant_compatible(product_type: str, query: str, candidate: str | None) -> bool:
if candidate is None:
return False
if product_type == "саморез":
if query == "gkl":
return candidate.startswith("gkl_")
return query == candidate
if product_type == "кабель" and query == "vvg":
# Omitting LS is not enough evidence to reject the safer cable variant.
# It therefore remains a possible candidate and may produce ambiguous.
return candidate in {"vvg", "vvg_ls"}
if product_type == "диск" and query == "cut":
return candidate.startswith("cut_")
return query == candidate
def _dimensions_compatible(
query: tuple[Decimal, ...], candidate: ParsedAttributes
) -> bool:
if candidate.dimensions and len(query) <= len(candidate.dimensions):
return all(value == candidate.dimensions[index] for index, value in enumerate(query))
if len(query) == 1:
value = query[0]
if candidate.mm_values and value in candidate.mm_values:
return True
if candidate.dimensions and candidate.dimensions[0] == value:
return True
return False
def compare_attributes(
product_type: str,
query: ParsedAttributes,
candidate: ParsedAttributes,
*,
candidate_unit: str,
) -> Compatibility:
matched = 0
total = 0
def require(condition: bool, name: str) -> Compatibility | None:
nonlocal matched, total
total += 1
if not condition:
return Compatibility(False, matched, total, name)
matched += 1
return None
if query.variant is not None:
conflict = require(
_variant_compatible(product_type, query.variant, candidate.variant), "variant"
)
if conflict:
return conflict
if query.dimensions:
conflict = require(_dimensions_compatible(query.dimensions, candidate), "dimensions")
if conflict:
return conflict
for value in query.mm_values:
condition = (
value in candidate.mm_values
or bool(candidate.dimensions and candidate.dimensions[0] == value)
or candidate.thread_length_mm == value
)
conflict = require(condition, "millimetres")
if conflict:
return conflict
for value in query.meter_values:
condition = value in candidate.meter_values or candidate.thread_length_mm == value * 1000
conflict = require(condition, "metres")
if conflict:
return conflict
for query_value, candidate_value, name in (
(query.thread_size, candidate.thread_size, "thread"),
(query.profile, candidate.profile, "profile"),
(query.grit, candidate.grit, "grit"),
(query.voltage_v, candidate.voltage_v, "voltage"),
(query.power_w, candidate.power_w, "power"),
(query.teeth, candidate.teeth, "teeth"),
(query.pack_size, candidate.pack_size, "pack_size"),
):
if query_value is not None:
conflict = require(query_value == candidate_value, name)
if conflict:
return conflict
if query.unit_hint is not None:
conflict = require(normalize_text(candidate_unit) == query.unit_hint, "unit")
if conflict:
return conflict
if query.brand is not None and query.brand_required:
conflict = require(query.brand == candidate.brand, "brand")
if conflict:
return conflict
if query.model_codes:
conflict = require(
all(
any(
model_code_matches(query_code, candidate_code)
for candidate_code in candidate.model_codes
)
for query_code in query.model_codes
),
"model",
)
if conflict:
return conflict
if query.sds_type is not None:
condition = (
candidate.sds_type is not None
if query.sds_type == "sds"
else query.sds_type == candidate.sds_type
)
conflict = require(condition, "sds")
if conflict:
return conflict
return Compatibility(True, matched, total)