Validate product attributes and decide match status
This commit is contained in:
1 parent
da8ad74498
commit
a67826daa4
4 files changed
+927
-15
No files matched your search
@@ -0,0 +1,647 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
|
||||
from app.normalization import normalize_code, normalize_text, tokenize
|
||||
from app.typo import fuzzy_threshold, word_similarity
|
||||
|
||||
_NUMBER = r"\d+(?:\.\d+)?"
|
||||
_DIMENSIONS_RE = re.compile(
|
||||
rf"(?<![a-zа-я0-9.])({_NUMBER}(?:x{_NUMBER}){{1,3}})(?![a-zа-я0-9.])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PAIR_WITH_NA_RE = re.compile(rf"\b({_NUMBER})\s+на\s+({_NUMBER})\b", re.IGNORECASE)
|
||||
_MM_RE = re.compile(rf"\b({_NUMBER})\s*(?:мм|миллиметр\w*)\b", re.IGNORECASE)
|
||||
_METER_RE = re.compile(rf"\b({_NUMBER})\s*(?:м|метр\w*)\b", re.IGNORECASE)
|
||||
_THREAD_RE = re.compile(rf"(?<![a-zа-я0-9])m\s*({_NUMBER})(?=x|\b)", re.IGNORECASE)
|
||||
_THREAD_LENGTH_RE = re.compile(rf"(?<![a-zа-я0-9])m\s*({_NUMBER})x({_NUMBER})", re.IGNORECASE)
|
||||
_PROFILE_RE = re.compile(
|
||||
rf"(?<![a-z0-9])(ph\d+|pz\d+|t\d+|sl{_NUMBER})(?=x|[^a-z0-9]|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PROFILE_LENGTH_RE = re.compile(
|
||||
rf"(?<![a-z0-9])(ph\d+|pz\d+|t\d+|sl{_NUMBER})x({_NUMBER})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_GRIT_RE = re.compile(r"(?<![a-z0-9])p\s*(\d{2,4})(?![a-z0-9])", re.IGNORECASE)
|
||||
_VOLTAGE_RE = re.compile(
|
||||
rf"(?<![a-zа-я0-9-])({_NUMBER})\s*(?:v|в|вольт\w*)\b", re.IGNORECASE
|
||||
)
|
||||
_POWER_RE = re.compile(rf"\b({_NUMBER})\s*(?:w|вт|ватт\w*)\b", re.IGNORECASE)
|
||||
_TEETH_RE = re.compile(r"\b(\d+)\s*(?:зуб(?:ьев|а|ов)?|z)\b", re.IGNORECASE)
|
||||
_PACK_SIZE_RE = re.compile(
|
||||
r"(?:пачк\w*|упаковк\w*|\bуп\.?)(?:\s+(?:по|на))?\s*(\d+)\s*(?:шт\w*)?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CATALOG_PACK_RE = re.compile(r"\bуп\.?\s*(\d+)\s*шт", re.IGNORECASE)
|
||||
_MODEL_RE = re.compile(
|
||||
r"(?<![a-zа-я0-9])([a-zа-я]{2,6}-\d+[a-zа-я]?|[a-zа-я]{2,6}\d+[a-zа-я]?)(?![a-zа-я0-9])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_PAIR_DIMENSION_TYPES = frozenset(
|
||||
{
|
||||
"саморез",
|
||||
"бур",
|
||||
"дюбель",
|
||||
"анкер",
|
||||
"гвозди",
|
||||
"труба",
|
||||
"болт",
|
||||
"шпилька",
|
||||
"стяжка",
|
||||
}
|
||||
)
|
||||
_SINGLE_SIZE_AFTER_NA_TYPES = frozenset(
|
||||
{"ушм", "диск", "круг", "сверло", "бита", "уровень", "лента фум"}
|
||||
)
|
||||
|
||||
BRAND_ALIASES = [
|
||||
{
|
||||
"canonical": "prowerk",
|
||||
"aliases": ["prowerk", "проверк", "проуверк", "проуорк"],
|
||||
},
|
||||
{
|
||||
"canonical": "техноресурс",
|
||||
"aliases": ["техноресурс", "техно ресурс"],
|
||||
},
|
||||
{
|
||||
"canonical": "арсенал-т",
|
||||
"aliases": ["арсенал-т", "арсенал т", "арсенал"],
|
||||
},
|
||||
{
|
||||
"canonical": "toolkraft",
|
||||
"aliases": ["toolkraft", "тулкрафт", "тул крафт"],
|
||||
},
|
||||
{
|
||||
"canonical": "makita",
|
||||
"aliases": ["makita", "макита", "макиты", "макиту"],
|
||||
},
|
||||
{"canonical": "bosch", "aliases": ["bosch", "бош", "боша"]},
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BrandMatch:
|
||||
canonical: str
|
||||
score: float
|
||||
alias: str
|
||||
token_start: int
|
||||
|
||||
|
||||
class BrandIndex:
|
||||
def __init__(self) -> None:
|
||||
records: list[tuple[str, str, tuple[str, ...]]] = []
|
||||
for item in BRAND_ALIASES:
|
||||
canonical = normalize_text(item["canonical"])
|
||||
for raw_alias in item["aliases"]:
|
||||
alias = normalize_text(raw_alias)
|
||||
records.append((canonical, alias, tokenize(alias, already_normalized=True)))
|
||||
self._records = tuple(records)
|
||||
|
||||
def detect(self, text: str) -> BrandMatch | None:
|
||||
normalized = normalize_text(text)
|
||||
tokens = tokenize(normalized, already_normalized=True)
|
||||
exact_matches: list[BrandMatch] = []
|
||||
|
||||
for canonical, alias, alias_tokens in self._records:
|
||||
for start in range(len(tokens) - len(alias_tokens) + 1):
|
||||
if tokens[start : start + len(alias_tokens)] == alias_tokens:
|
||||
exact_matches.append(BrandMatch(canonical, 1.0, alias, start))
|
||||
break
|
||||
|
||||
if exact_matches:
|
||||
exact_matches.sort(key=lambda item: (-len(tokenize(item.alias)), item.token_start))
|
||||
return exact_matches[0]
|
||||
|
||||
best: BrandMatch | None = None
|
||||
for start, query_token in enumerate(tokens):
|
||||
if len(query_token) < 5 or not query_token.isalpha():
|
||||
continue
|
||||
for canonical, alias, alias_tokens in self._records:
|
||||
if len(alias_tokens) != 1:
|
||||
continue
|
||||
alias_token = alias_tokens[0]
|
||||
if abs(len(query_token) - len(alias_token)) > 3:
|
||||
continue
|
||||
score = word_similarity(query_token, alias_token)
|
||||
if score < max(0.70, fuzzy_threshold(max(len(query_token), len(alias_token)))):
|
||||
continue
|
||||
candidate = BrandMatch(canonical, score, alias, start)
|
||||
if best is None or candidate.score > best.score:
|
||||
best = candidate
|
||||
return best
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParsedAttributes:
|
||||
dimensions: tuple[Decimal, ...] = ()
|
||||
mm_values: tuple[Decimal, ...] = ()
|
||||
meter_values: tuple[Decimal, ...] = ()
|
||||
thread_size: Decimal | None = None
|
||||
thread_length_mm: Decimal | None = None
|
||||
profile: str | None = None
|
||||
grit: int | None = None
|
||||
voltage_v: Decimal | None = None
|
||||
power_w: Decimal | None = None
|
||||
teeth: int | None = None
|
||||
pack_size: int | None = None
|
||||
unit_hint: str | None = None
|
||||
brand: str | None = None
|
||||
brand_required: bool = False
|
||||
model_codes: frozenset[str] = frozenset()
|
||||
sds_type: str | None = None
|
||||
variant: str | None = None
|
||||
cheap_preference: bool = False
|
||||
|
||||
@property
|
||||
def specificity(self) -> int:
|
||||
return sum(
|
||||
(
|
||||
bool(self.dimensions),
|
||||
bool(self.mm_values),
|
||||
bool(self.meter_values),
|
||||
self.thread_size is not None,
|
||||
self.thread_length_mm is not None,
|
||||
self.profile is not None,
|
||||
self.grit is not None,
|
||||
self.voltage_v is not None,
|
||||
self.power_w is not None,
|
||||
self.teeth is not None,
|
||||
self.pack_size is not None,
|
||||
self.unit_hint is not None,
|
||||
self.brand is not None and self.brand_required,
|
||||
bool(self.model_codes),
|
||||
self.sds_type is not None,
|
||||
self.variant is not None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Compatibility:
|
||||
compatible: bool
|
||||
matched_fields: int
|
||||
total_fields: int
|
||||
conflict: str | None = None
|
||||
|
||||
@property
|
||||
def score(self) -> float:
|
||||
if self.total_fields == 0:
|
||||
return 1.0
|
||||
return self.matched_fields / self.total_fields
|
||||
|
||||
|
||||
def _decimal(value: str) -> Decimal:
|
||||
return Decimal(value).normalize()
|
||||
|
||||
|
||||
def _unique_decimals(values: list[Decimal]) -> tuple[Decimal, ...]:
|
||||
return tuple(dict.fromkeys(values))
|
||||
|
||||
|
||||
def _extract_dimensions(normalized: str) -> tuple[Decimal, ...]:
|
||||
match = _DIMENSIONS_RE.search(normalized)
|
||||
if match is None:
|
||||
return ()
|
||||
return tuple(_decimal(part) for part in match.group(1).split("x"))
|
||||
|
||||
|
||||
def _extract_model_codes(normalized: str) -> frozenset[str]:
|
||||
excluded_prefixes = ("ph", "pz", "sl", "ffp", "din")
|
||||
codes: set[str] = set()
|
||||
for match in _MODEL_RE.finditer(normalized):
|
||||
raw = match.group(1)
|
||||
code = normalize_code(raw)
|
||||
if code.startswith(excluded_prefixes):
|
||||
# DIN/FFP are useful exact terms for lexical ranking, but they are
|
||||
# standards/classes rather than unique product models.
|
||||
continue
|
||||
if re.fullmatch(r"(?:t|m|p)\d+(?:\d+)?", code):
|
||||
continue
|
||||
codes.add(code)
|
||||
return frozenset(codes)
|
||||
|
||||
|
||||
def _detect_sds(normalized: str) -> str | None:
|
||||
flattened = normalized.replace("-", " ")
|
||||
if "sds max" in flattened or "сдс макс" in flattened:
|
||||
return "sds-max"
|
||||
if "sds plus" in flattened or "сдс плюс" in flattened:
|
||||
return "sds-plus"
|
||||
tokens = set(tokenize(flattened, already_normalized=True))
|
||||
if "sds" in tokens or "сдс" in tokens:
|
||||
return "sds"
|
||||
return None
|
||||
|
||||
|
||||
def _detect_variant(
|
||||
normalized: str, product_type: str, *, catalog_item: bool
|
||||
) -> str | None:
|
||||
text = f" {normalized.replace('-', ' ')} "
|
||||
|
||||
if product_type == "саморез":
|
||||
is_gkl = " гкл " in text or " гипсокартон" in text
|
||||
has_wood = "по дерев" in text or "гипсокартон дерев" in text
|
||||
has_metal = "по металл" in text or "гипсокартон металл" in text
|
||||
if is_gkl and has_wood:
|
||||
return "gkl_wood"
|
||||
if is_gkl and has_metal:
|
||||
return "gkl_metal"
|
||||
if is_gkl:
|
||||
return "gkl"
|
||||
if has_wood:
|
||||
return "wood"
|
||||
if has_metal:
|
||||
return "metal"
|
||||
|
||||
if product_type == "кабель":
|
||||
if " шввп " in text:
|
||||
return "shvvp"
|
||||
if " пвс " in text:
|
||||
return "pvs"
|
||||
if " ввг" in text:
|
||||
if " ls " in text or " лс " in text:
|
||||
return "vvg_ls"
|
||||
return "vvg"
|
||||
|
||||
if product_type == "провод" and " пугв " in text:
|
||||
return "pugv"
|
||||
|
||||
if product_type == "диск":
|
||||
if "пильн" in text:
|
||||
return "saw_wood"
|
||||
if "зачист" in text:
|
||||
return "grinding"
|
||||
if "лепест" in text:
|
||||
return "flap"
|
||||
if "отрез" in text:
|
||||
if "нержав" in text:
|
||||
return "cut_stainless"
|
||||
if "камн" in text:
|
||||
return "cut_stone"
|
||||
if "металл" in text:
|
||||
return "cut_metal"
|
||||
return "cut"
|
||||
|
||||
if product_type == "сверло":
|
||||
if "по металл" in text:
|
||||
return "metal"
|
||||
if "по дерев" in text:
|
||||
return "wood"
|
||||
if "по бетон" in text:
|
||||
return "concrete"
|
||||
|
||||
if product_type == "перчатки":
|
||||
for marker, variant in (
|
||||
("нитрил", "nitrile"),
|
||||
("спилк", "split_leather"),
|
||||
("прорезин", "rubberized"),
|
||||
("латекс", "latex"),
|
||||
("полиуретан", "polyurethane"),
|
||||
("пвх", "pvc"),
|
||||
):
|
||||
if marker in text:
|
||||
return variant
|
||||
|
||||
if product_type == "дюбель":
|
||||
if "гвозд" in text:
|
||||
return "nail"
|
||||
if "распор" in text:
|
||||
return "spacer"
|
||||
|
||||
if product_type == "шайба":
|
||||
if "гровер" in text:
|
||||
return "spring"
|
||||
if "плоск" in text:
|
||||
return "flat"
|
||||
|
||||
if product_type == "анкер":
|
||||
if "клинов" in text:
|
||||
return "wedge"
|
||||
if "с гайк" in text:
|
||||
return "with_nut"
|
||||
|
||||
if product_type == "ключ":
|
||||
if "разводн" in text:
|
||||
return "adjustable"
|
||||
if "рожков" in text:
|
||||
return "open_end"
|
||||
|
||||
if product_type == "гипсокартон":
|
||||
if "влагостой" in text:
|
||||
return "moisture"
|
||||
if "огнестой" in text:
|
||||
return "fire"
|
||||
if catalog_item:
|
||||
return "standard"
|
||||
if "обычн" in text or "стандартн" in text:
|
||||
return "standard"
|
||||
|
||||
if product_type == "профиль":
|
||||
if "направля" in text or " пн " in text:
|
||||
return "guide"
|
||||
if "стоеч" in text or " пс " in text:
|
||||
return "stud"
|
||||
if "потолоч" in text or " пп " in text:
|
||||
return "ceiling"
|
||||
|
||||
if product_type == "герметик":
|
||||
if "силикон" in text:
|
||||
return "silicone"
|
||||
if "акрил" in text:
|
||||
return "acrylic"
|
||||
|
||||
if product_type == "пена":
|
||||
if "профессион" in text:
|
||||
return "professional"
|
||||
if "бытов" in text:
|
||||
return "household"
|
||||
|
||||
if product_type == "хомут" and "червяч" in text:
|
||||
return "worm"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_attributes(
|
||||
text: str,
|
||||
product_type: str,
|
||||
*,
|
||||
unit: str | None = None,
|
||||
catalog_item: bool = False,
|
||||
brand_index: BrandIndex | None = None,
|
||||
) -> ParsedAttributes:
|
||||
normalized = normalize_text(text)
|
||||
dimensions = _extract_dimensions(normalized)
|
||||
|
||||
teeth_match = _TEETH_RE.search(normalized)
|
||||
teeth = int(teeth_match.group(1)) if teeth_match else None
|
||||
|
||||
if not dimensions and product_type in _PAIR_DIMENSION_TYPES:
|
||||
pair_match = _PAIR_WITH_NA_RE.search(normalized)
|
||||
if pair_match is not None:
|
||||
dimensions = (_decimal(pair_match.group(1)), _decimal(pair_match.group(2)))
|
||||
|
||||
if product_type == "труба" and len(dimensions) == 2:
|
||||
wall_match = re.search(rf"\bстенк\w*\s+({_NUMBER}|полтора)\b", normalized)
|
||||
if wall_match is not None:
|
||||
wall_value = Decimal("1.5") if wall_match.group(1) == "полтора" else _decimal(
|
||||
wall_match.group(1)
|
||||
)
|
||||
dimensions = (*dimensions, wall_value)
|
||||
|
||||
mm_values = [_decimal(match.group(1)) for match in _MM_RE.finditer(normalized)]
|
||||
meter_values = [_decimal(match.group(1)) for match in _METER_RE.finditer(normalized)]
|
||||
|
||||
# ``190 на 48 зубьев`` describes diameter and tooth count, not 190x48.
|
||||
saw_pair = re.search(
|
||||
rf"\b({_NUMBER})\s+на\s+\d+\s*(?:зуб(?:ьев|а|ов)?)\b",
|
||||
normalized,
|
||||
)
|
||||
if saw_pair is not None:
|
||||
mm_values.append(_decimal(saw_pair.group(1)))
|
||||
|
||||
if (
|
||||
not catalog_item
|
||||
and product_type in _SINGLE_SIZE_AFTER_NA_TYPES
|
||||
and saw_pair is None
|
||||
):
|
||||
after_na = re.search(rf"\bна\s+({_NUMBER})(?!\s*(?:в|вольт|вт|ватт))", normalized)
|
||||
if after_na is not None:
|
||||
value = _decimal(after_na.group(1))
|
||||
if value not in mm_values:
|
||||
mm_values.append(value)
|
||||
|
||||
if not dimensions and not catalog_item and product_type == "гипсокартон":
|
||||
# Colloquial requests often contain only sheet thickness: ``гкл 9.5``.
|
||||
bare_number = re.search(rf"(?<![a-zа-я0-9])({_NUMBER})(?![a-zа-я0-9])", normalized)
|
||||
if bare_number is not None:
|
||||
dimensions = (_decimal(bare_number.group(1)),)
|
||||
|
||||
if not catalog_item and "метров" in normalized and not meter_values:
|
||||
meter_values.append(Decimal("1"))
|
||||
if not catalog_item and "метровая" in normalized and not meter_values:
|
||||
meter_values.append(Decimal("1"))
|
||||
|
||||
bare_size_types = {
|
||||
"ушм",
|
||||
"диск",
|
||||
"круг",
|
||||
"сверло",
|
||||
"бита",
|
||||
"уровень",
|
||||
"лента фум",
|
||||
"нож",
|
||||
"лезвия",
|
||||
"кисть",
|
||||
"валик",
|
||||
}
|
||||
if (
|
||||
not catalog_item
|
||||
and product_type in bare_size_types
|
||||
and not dimensions
|
||||
and not mm_values
|
||||
):
|
||||
bare_numbers = [
|
||||
_decimal(match.group(1))
|
||||
for match in re.finditer(
|
||||
rf"(?<![a-zа-я0-9.\-])({_NUMBER})(?![a-zа-я0-9.])", normalized
|
||||
)
|
||||
]
|
||||
if teeth is not None:
|
||||
bare_numbers = [value for value in bare_numbers if value != Decimal(teeth)]
|
||||
if bare_numbers:
|
||||
mm_values.append(bare_numbers[0])
|
||||
|
||||
thread_match = _THREAD_RE.search(normalized)
|
||||
thread_size = _decimal(thread_match.group(1)) if thread_match else None
|
||||
thread_length_match = _THREAD_LENGTH_RE.search(normalized)
|
||||
thread_length_mm = (
|
||||
_decimal(thread_length_match.group(2)) if thread_length_match else None
|
||||
)
|
||||
if thread_length_mm is None and product_type == "шпилька" and Decimal("1") in meter_values:
|
||||
thread_length_mm = Decimal("1000")
|
||||
|
||||
profile_match = _PROFILE_RE.search(normalized)
|
||||
profile = profile_match.group(1).upper() if profile_match else None
|
||||
profile_length_match = _PROFILE_LENGTH_RE.search(normalized)
|
||||
if profile_length_match is not None:
|
||||
profile = profile_length_match.group(1).upper()
|
||||
profile_length = _decimal(profile_length_match.group(2))
|
||||
if profile_length not in mm_values:
|
||||
mm_values.append(profile_length)
|
||||
|
||||
grit_match = _GRIT_RE.search(normalized)
|
||||
grit = int(grit_match.group(1)) if grit_match else None
|
||||
voltage_match = _VOLTAGE_RE.search(normalized)
|
||||
voltage_v = _decimal(voltage_match.group(1)) if voltage_match else None
|
||||
power_match = _POWER_RE.search(normalized)
|
||||
power_w = _decimal(power_match.group(1)) if power_match else None
|
||||
|
||||
pack_match = (_CATALOG_PACK_RE if catalog_item else _PACK_SIZE_RE).search(normalized)
|
||||
pack_size = int(pack_match.group(1)) if pack_match else None
|
||||
|
||||
unit_hint: str | None = None
|
||||
if not catalog_item:
|
||||
if re.search(r"\b(?:пачк\w*|упаковк\w*|уп\.)\b", normalized):
|
||||
unit_hint = "уп"
|
||||
elif re.search(r"\b(?:листами|листах|лист)\b", normalized):
|
||||
unit_hint = "лист"
|
||||
elif re.search(r"\b(?:за\s+метр|погонн\w*\s+метр)\b", normalized):
|
||||
unit_hint = "м"
|
||||
elif re.search(r"\b(?:килограмм\w*|кг)\b", normalized):
|
||||
unit_hint = "кг"
|
||||
elif unit:
|
||||
unit_hint = normalize_text(unit)
|
||||
|
||||
active_brand_index = brand_index or BrandIndex()
|
||||
brand_match = active_brand_index.detect(normalized)
|
||||
brand = brand_match.canonical if brand_match else None
|
||||
brand_required = False
|
||||
if brand is not None and not catalog_item:
|
||||
reference_markers = ("как у", "как ", "аналог", "вместо", "только дешевле")
|
||||
brand_required = not any(marker in normalized for marker in reference_markers)
|
||||
|
||||
cheap_preference = bool(re.search(r"\b(?:дешев\w*|недорог\w*|бюджет\w*)\b", normalized))
|
||||
|
||||
return ParsedAttributes(
|
||||
dimensions=dimensions,
|
||||
mm_values=_unique_decimals(mm_values),
|
||||
meter_values=_unique_decimals(meter_values),
|
||||
thread_size=thread_size,
|
||||
thread_length_mm=thread_length_mm,
|
||||
profile=profile,
|
||||
grit=grit,
|
||||
voltage_v=voltage_v,
|
||||
power_w=power_w,
|
||||
teeth=teeth,
|
||||
pack_size=pack_size,
|
||||
unit_hint=unit_hint,
|
||||
brand=brand,
|
||||
brand_required=brand_required,
|
||||
model_codes=_extract_model_codes(normalized),
|
||||
sds_type=_detect_sds(normalized),
|
||||
variant=_detect_variant(normalized, product_type, catalog_item=catalog_item),
|
||||
cheap_preference=cheap_preference,
|
||||
)
|
||||
|
||||
|
||||
def _variant_compatible(product_type: str, query: str, candidate: str | None) -> bool:
|
||||
if candidate is None:
|
||||
return False
|
||||
if product_type == "саморез":
|
||||
if query == "gkl":
|
||||
return candidate.startswith("gkl_")
|
||||
return query == candidate
|
||||
if product_type == "кабель" and query == "vvg":
|
||||
# Omitting LS is not enough evidence to reject the safer cable variant.
|
||||
# It therefore remains a possible candidate and may produce ambiguous.
|
||||
return candidate in {"vvg", "vvg_ls"}
|
||||
if product_type == "диск" and query == "cut":
|
||||
return candidate.startswith("cut_")
|
||||
return query == candidate
|
||||
|
||||
|
||||
def _dimensions_compatible(
|
||||
query: tuple[Decimal, ...], candidate: ParsedAttributes
|
||||
) -> bool:
|
||||
if candidate.dimensions and len(query) <= len(candidate.dimensions):
|
||||
return all(value == candidate.dimensions[index] for index, value in enumerate(query))
|
||||
if len(query) == 1:
|
||||
value = query[0]
|
||||
if candidate.mm_values and value in candidate.mm_values:
|
||||
return True
|
||||
if candidate.dimensions and candidate.dimensions[0] == value:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def compare_attributes(
|
||||
product_type: str,
|
||||
query: ParsedAttributes,
|
||||
candidate: ParsedAttributes,
|
||||
*,
|
||||
candidate_unit: str,
|
||||
) -> Compatibility:
|
||||
matched = 0
|
||||
total = 0
|
||||
|
||||
def require(condition: bool, name: str) -> Compatibility | None:
|
||||
nonlocal matched, total
|
||||
total += 1
|
||||
if not condition:
|
||||
return Compatibility(False, matched, total, name)
|
||||
matched += 1
|
||||
return None
|
||||
|
||||
if query.variant is not None:
|
||||
conflict = require(
|
||||
_variant_compatible(product_type, query.variant, candidate.variant), "variant"
|
||||
)
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
if query.dimensions:
|
||||
conflict = require(_dimensions_compatible(query.dimensions, candidate), "dimensions")
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
for value in query.mm_values:
|
||||
condition = (
|
||||
value in candidate.mm_values
|
||||
or bool(candidate.dimensions and candidate.dimensions[0] == value)
|
||||
or candidate.thread_length_mm == value
|
||||
)
|
||||
conflict = require(condition, "millimetres")
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
for value in query.meter_values:
|
||||
condition = value in candidate.meter_values or candidate.thread_length_mm == value * 1000
|
||||
conflict = require(condition, "metres")
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
for query_value, candidate_value, name in (
|
||||
(query.thread_size, candidate.thread_size, "thread"),
|
||||
(query.profile, candidate.profile, "profile"),
|
||||
(query.grit, candidate.grit, "grit"),
|
||||
(query.voltage_v, candidate.voltage_v, "voltage"),
|
||||
(query.power_w, candidate.power_w, "power"),
|
||||
(query.teeth, candidate.teeth, "teeth"),
|
||||
(query.pack_size, candidate.pack_size, "pack_size"),
|
||||
):
|
||||
if query_value is not None:
|
||||
conflict = require(query_value == candidate_value, name)
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
if query.unit_hint is not None:
|
||||
conflict = require(normalize_text(candidate_unit) == query.unit_hint, "unit")
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
if query.brand is not None and query.brand_required:
|
||||
conflict = require(query.brand == candidate.brand, "brand")
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
if query.model_codes:
|
||||
conflict = require(query.model_codes.issubset(candidate.model_codes), "model")
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
if query.sds_type is not None:
|
||||
condition = (
|
||||
candidate.sds_type is not None
|
||||
if query.sds_type == "sds"
|
||||
else query.sds_type == candidate.sds_type
|
||||
)
|
||||
conflict = require(condition, "sds")
|
||||
if conflict:
|
||||
return conflict
|
||||
|
||||
return Compatibility(True, matched, total)
|
||||
@@ -5,6 +5,10 @@ from dataclasses import dataclass
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
|
||||
from app.aliases import ProductAliasIndex
|
||||
from app.attributes import BrandIndex, ParsedAttributes, parse_attributes
|
||||
from app.normalization import meaningful_tokens, normalize_text, tokenize
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RawCatalogItem:
|
||||
@@ -14,6 +18,18 @@ class RawCatalogItem:
|
||||
price: Decimal
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CatalogItem:
|
||||
sku: str
|
||||
name: str
|
||||
unit: str
|
||||
price: Decimal
|
||||
normalized_name: str
|
||||
search_tokens: tuple[str, ...]
|
||||
product_type: str
|
||||
attributes: ParsedAttributes
|
||||
|
||||
|
||||
def load_raw_catalog(path: Path) -> tuple[RawCatalogItem, ...]:
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"Catalog file does not exist: {path}")
|
||||
@@ -52,3 +68,36 @@ def load_raw_catalog(path: Path) -> tuple[RawCatalogItem, ...]:
|
||||
if not items:
|
||||
raise ValueError("Catalog is empty")
|
||||
return tuple(items)
|
||||
|
||||
|
||||
def build_catalog(
|
||||
raw_items: tuple[RawCatalogItem, ...],
|
||||
alias_index: ProductAliasIndex,
|
||||
brand_index: BrandIndex,
|
||||
) -> tuple[CatalogItem, ...]:
|
||||
result: list[CatalogItem] = []
|
||||
for raw in raw_items:
|
||||
normalized = normalize_text(raw.name)
|
||||
product_type = alias_index.detect_catalog_type(raw.name)
|
||||
attributes = parse_attributes(
|
||||
raw.name,
|
||||
product_type,
|
||||
unit=raw.unit,
|
||||
catalog_item=True,
|
||||
brand_index=brand_index,
|
||||
)
|
||||
result.append(
|
||||
CatalogItem(
|
||||
sku=raw.sku,
|
||||
name=raw.name,
|
||||
unit=raw.unit,
|
||||
price=raw.price,
|
||||
normalized_name=normalized,
|
||||
search_tokens=meaningful_tokens(
|
||||
tokenize(normalized, already_normalized=True)
|
||||
),
|
||||
product_type=product_type,
|
||||
attributes=attributes,
|
||||
)
|
||||
)
|
||||
return tuple(result)
|
||||
+13
-15
@@ -5,26 +5,24 @@ from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.catalog import load_raw_catalog
|
||||
from app.models import MatchRequest, MatchResponse, MatchResult
|
||||
from app.matcher import CatalogMatcher
|
||||
from app.models import MatchRequest, MatchResponse
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
CATALOG_PATH = Path(os.environ.get("CATALOG_PATH", BASE_DIR / "catalog_excel.csv"))
|
||||
RAW_CATALOG = load_raw_catalog(CATALOG_PATH)
|
||||
MATCHER = CatalogMatcher(CATALOG_PATH)
|
||||
|
||||
app = FastAPI(title="Offline catalog matcher", version="0.1.0")
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, object]:
|
||||
return {"status": "ok", "catalog_items": len(RAW_CATALOG)}
|
||||
# The task asks for one application endpoint. OpenAPI/Swagger routes are disabled
|
||||
# so the running service exposes only POST /match.
|
||||
app = FastAPI(
|
||||
title="Offline catalog matcher",
|
||||
version="0.2.0",
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
openapi_url=None,
|
||||
)
|
||||
|
||||
|
||||
@app.post("/match", response_model=MatchResponse)
|
||||
def match(request: MatchRequest) -> MatchResponse:
|
||||
return MatchResponse(
|
||||
results=[
|
||||
MatchResult(message=message, status="not_found", candidates=[])
|
||||
for message in request.messages
|
||||
]
|
||||
)
|
||||
return MatchResponse(results=[MATCHER.match(message) for message in request.messages])
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.aliases import ProductAliasIndex, ProductTypeMatch
|
||||
from app.attributes import BrandIndex, Compatibility, compare_attributes, parse_attributes
|
||||
from app.catalog import CatalogItem, build_catalog, load_raw_catalog
|
||||
from app.intents import detect_service_intent
|
||||
from app.models import Candidate, MatchResult
|
||||
from app.search_index import LexicalIndex, LexicalScore
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RankedCandidate:
|
||||
item: CatalogItem
|
||||
lexical: LexicalScore
|
||||
compatibility: Compatibility
|
||||
ranking_score: float
|
||||
|
||||
|
||||
class CatalogMatcher:
|
||||
def __init__(self, catalog_path: Path) -> None:
|
||||
self.alias_index = ProductAliasIndex()
|
||||
self.brand_index = BrandIndex()
|
||||
self.items = build_catalog(
|
||||
load_raw_catalog(catalog_path), self.alias_index, self.brand_index
|
||||
)
|
||||
self.lexical_index = LexicalIndex(self.items)
|
||||
|
||||
type_index: dict[str, list[int]] = defaultdict(list)
|
||||
model_index: dict[str, set[int]] = defaultdict(set)
|
||||
for index, item in enumerate(self.items):
|
||||
type_index[item.product_type].append(index)
|
||||
for model_code in item.attributes.model_codes:
|
||||
model_index[model_code].add(index)
|
||||
self.type_index = {
|
||||
product_type: tuple(indexes) for product_type, indexes in type_index.items()
|
||||
}
|
||||
self.model_index = {
|
||||
model: frozenset(indexes) for model, indexes in model_index.items()
|
||||
}
|
||||
|
||||
def match(self, message: str) -> MatchResult:
|
||||
if not message or not message.strip():
|
||||
return MatchResult(message=message, status="not_found", candidates=[])
|
||||
|
||||
product_match = self.alias_index.detect(message)
|
||||
provisional_attributes = parse_attributes(
|
||||
message,
|
||||
product_match.canonical if product_match else "",
|
||||
brand_index=self.brand_index,
|
||||
)
|
||||
|
||||
if product_match is None:
|
||||
model_candidates = self._model_candidates(provisional_attributes.model_codes)
|
||||
if model_candidates:
|
||||
product_types = {self.items[index].product_type for index in model_candidates}
|
||||
if len(product_types) == 1:
|
||||
product_type = next(iter(product_types))
|
||||
product_match = ProductTypeMatch(
|
||||
canonical=product_type,
|
||||
score=1.0,
|
||||
exact=True,
|
||||
alias="model_code",
|
||||
token_start=0,
|
||||
)
|
||||
if product_match is None:
|
||||
# Product evidence always has priority. Service intent detection
|
||||
# is only a diagnostic fallback; both unknown and service
|
||||
# messages share the required not_found response.
|
||||
detect_service_intent(message)
|
||||
return MatchResult(message=message, status="not_found", candidates=[])
|
||||
|
||||
product_type = product_match.canonical
|
||||
query_attributes = parse_attributes(
|
||||
message,
|
||||
product_type,
|
||||
brand_index=self.brand_index,
|
||||
)
|
||||
candidate_indices = self.type_index.get(product_type, ())
|
||||
|
||||
model_candidates = self._model_candidates(query_attributes.model_codes)
|
||||
if model_candidates:
|
||||
candidate_indices = tuple(
|
||||
index for index in candidate_indices if index in model_candidates
|
||||
)
|
||||
|
||||
compatible: dict[int, Compatibility] = {}
|
||||
for index in candidate_indices:
|
||||
item = self.items[index]
|
||||
comparison = compare_attributes(
|
||||
product_type,
|
||||
query_attributes,
|
||||
item.attributes,
|
||||
candidate_unit=item.unit,
|
||||
)
|
||||
if comparison.compatible:
|
||||
compatible[index] = comparison
|
||||
|
||||
if not compatible:
|
||||
return MatchResult(message=message, status="not_found", candidates=[])
|
||||
|
||||
lexical_scores = self.lexical_index.rank(
|
||||
message, product_type, tuple(compatible.keys())
|
||||
)
|
||||
ranked = self._apply_preferences(
|
||||
lexical_scores, compatible, cheap=query_attributes.cheap_preference
|
||||
)
|
||||
if not ranked:
|
||||
return MatchResult(message=message, status="not_found", candidates=[])
|
||||
|
||||
status = "matched" if len(ranked) == 1 else "ambiguous"
|
||||
response_candidates = [
|
||||
Candidate(
|
||||
sku=candidate.item.sku,
|
||||
confidence=self._confidence(
|
||||
candidate,
|
||||
product_match=product_match,
|
||||
candidate_count=len(ranked),
|
||||
specificity=query_attributes.specificity,
|
||||
status=status,
|
||||
rank=rank,
|
||||
),
|
||||
)
|
||||
for rank, candidate in enumerate(ranked[:3])
|
||||
]
|
||||
return MatchResult(
|
||||
message=message,
|
||||
status=status,
|
||||
candidates=response_candidates,
|
||||
)
|
||||
|
||||
def _model_candidates(self, model_codes: frozenset[str]) -> frozenset[int]:
|
||||
if not model_codes:
|
||||
return frozenset()
|
||||
sets = [self.model_index.get(code, frozenset()) for code in model_codes]
|
||||
if not sets or any(not indexes for indexes in sets):
|
||||
return frozenset()
|
||||
result = set(sets[0])
|
||||
for indexes in sets[1:]:
|
||||
result.intersection_update(indexes)
|
||||
return frozenset(result)
|
||||
|
||||
def _apply_preferences(
|
||||
self,
|
||||
lexical_scores: list[LexicalScore],
|
||||
compatible: dict[int, Compatibility],
|
||||
*,
|
||||
cheap: bool,
|
||||
) -> list[_RankedCandidate]:
|
||||
if not lexical_scores:
|
||||
return []
|
||||
|
||||
prices = [self.items[score.item_index].price for score in lexical_scores]
|
||||
minimum_price = min(prices)
|
||||
maximum_price = max(prices)
|
||||
price_range = maximum_price - minimum_price
|
||||
|
||||
ranked: list[_RankedCandidate] = []
|
||||
for lexical in lexical_scores:
|
||||
item = self.items[lexical.item_index]
|
||||
if cheap:
|
||||
cheap_score = (
|
||||
1.0
|
||||
if price_range == 0
|
||||
else float((maximum_price - item.price) / price_range)
|
||||
)
|
||||
ranking_score = lexical.score * 0.68 + cheap_score * 0.32
|
||||
else:
|
||||
ranking_score = lexical.score
|
||||
ranked.append(
|
||||
_RankedCandidate(
|
||||
item=item,
|
||||
lexical=lexical,
|
||||
compatibility=compatible[lexical.item_index],
|
||||
ranking_score=ranking_score,
|
||||
)
|
||||
)
|
||||
|
||||
ranked.sort(
|
||||
key=lambda candidate: (
|
||||
-candidate.ranking_score,
|
||||
candidate.item.price if cheap else candidate.item.sku,
|
||||
)
|
||||
)
|
||||
return ranked
|
||||
|
||||
@staticmethod
|
||||
def _confidence(
|
||||
candidate: _RankedCandidate,
|
||||
*,
|
||||
product_match: ProductTypeMatch,
|
||||
candidate_count: int,
|
||||
specificity: int,
|
||||
status: str,
|
||||
rank: int,
|
||||
) -> float:
|
||||
attribute_signal = (
|
||||
candidate.compatibility.score
|
||||
if candidate.compatibility.total_fields > 0
|
||||
else 0.55
|
||||
)
|
||||
uniqueness_signal = 1.0 if candidate_count == 1 else max(0.35, 0.72 - rank * 0.10)
|
||||
specificity_signal = min(1.0, 0.35 + specificity * 0.16)
|
||||
value = (
|
||||
product_match.score * 0.30
|
||||
+ candidate.lexical.score * 0.27
|
||||
+ attribute_signal * 0.23
|
||||
+ uniqueness_signal * 0.12
|
||||
+ specificity_signal * 0.08
|
||||
)
|
||||
if status == "matched":
|
||||
value = max(value, 0.82)
|
||||
else:
|
||||
value = min(value, 0.84)
|
||||
return round(max(0.01, min(0.99, value)), 4)
|
||||
Reference in new issue
Block a user