diff --git a/app/attributes.py b/app/attributes.py new file mode 100644 index 0000000..2e312fc --- /dev/null +++ b/app/attributes.py @@ -0,0 +1,647 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from decimal import Decimal + +from app.normalization import normalize_code, normalize_text, tokenize +from app.typo import fuzzy_threshold, word_similarity + +_NUMBER = r"\d+(?:\.\d+)?" +_DIMENSIONS_RE = re.compile( + rf"(? None: + records: list[tuple[str, str, tuple[str, ...]]] = [] + for item in BRAND_ALIASES: + canonical = normalize_text(item["canonical"]) + for raw_alias in item["aliases"]: + alias = normalize_text(raw_alias) + records.append((canonical, alias, tokenize(alias, already_normalized=True))) + self._records = tuple(records) + + def detect(self, text: str) -> BrandMatch | None: + normalized = normalize_text(text) + tokens = tokenize(normalized, already_normalized=True) + exact_matches: list[BrandMatch] = [] + + for canonical, alias, alias_tokens in self._records: + for start in range(len(tokens) - len(alias_tokens) + 1): + if tokens[start : start + len(alias_tokens)] == alias_tokens: + exact_matches.append(BrandMatch(canonical, 1.0, alias, start)) + break + + if exact_matches: + exact_matches.sort(key=lambda item: (-len(tokenize(item.alias)), item.token_start)) + return exact_matches[0] + + best: BrandMatch | None = None + for start, query_token in enumerate(tokens): + if len(query_token) < 5 or not query_token.isalpha(): + continue + for canonical, alias, alias_tokens in self._records: + if len(alias_tokens) != 1: + continue + alias_token = alias_tokens[0] + if abs(len(query_token) - len(alias_token)) > 3: + continue + score = word_similarity(query_token, alias_token) + if score < max(0.70, fuzzy_threshold(max(len(query_token), len(alias_token)))): + continue + candidate = BrandMatch(canonical, score, alias, start) + if best is None or candidate.score > best.score: + best = candidate + return best + + +@dataclass(frozen=True, slots=True) +class ParsedAttributes: + dimensions: tuple[Decimal, ...] = () + mm_values: tuple[Decimal, ...] = () + meter_values: tuple[Decimal, ...] = () + thread_size: Decimal | None = None + thread_length_mm: Decimal | None = None + profile: str | None = None + grit: int | None = None + voltage_v: Decimal | None = None + power_w: Decimal | None = None + teeth: int | None = None + pack_size: int | None = None + unit_hint: str | None = None + brand: str | None = None + brand_required: bool = False + model_codes: frozenset[str] = frozenset() + sds_type: str | None = None + variant: str | None = None + cheap_preference: bool = False + + @property + def specificity(self) -> int: + return sum( + ( + bool(self.dimensions), + bool(self.mm_values), + bool(self.meter_values), + self.thread_size is not None, + self.thread_length_mm is not None, + self.profile is not None, + self.grit is not None, + self.voltage_v is not None, + self.power_w is not None, + self.teeth is not None, + self.pack_size is not None, + self.unit_hint is not None, + self.brand is not None and self.brand_required, + bool(self.model_codes), + self.sds_type is not None, + self.variant is not None, + ) + ) + + +@dataclass(frozen=True, slots=True) +class Compatibility: + compatible: bool + matched_fields: int + total_fields: int + conflict: str | None = None + + @property + def score(self) -> float: + if self.total_fields == 0: + return 1.0 + return self.matched_fields / self.total_fields + + +def _decimal(value: str) -> Decimal: + return Decimal(value).normalize() + + +def _unique_decimals(values: list[Decimal]) -> tuple[Decimal, ...]: + return tuple(dict.fromkeys(values)) + + +def _extract_dimensions(normalized: str) -> tuple[Decimal, ...]: + match = _DIMENSIONS_RE.search(normalized) + if match is None: + return () + return tuple(_decimal(part) for part in match.group(1).split("x")) + + +def _extract_model_codes(normalized: str) -> frozenset[str]: + excluded_prefixes = ("ph", "pz", "sl", "ffp", "din") + codes: set[str] = set() + for match in _MODEL_RE.finditer(normalized): + raw = match.group(1) + code = normalize_code(raw) + if code.startswith(excluded_prefixes): + # DIN/FFP are useful exact terms for lexical ranking, but they are + # standards/classes rather than unique product models. + continue + if re.fullmatch(r"(?:t|m|p)\d+(?:\d+)?", code): + continue + codes.add(code) + return frozenset(codes) + + +def _detect_sds(normalized: str) -> str | None: + flattened = normalized.replace("-", " ") + if "sds max" in flattened or "сдс макс" in flattened: + return "sds-max" + if "sds plus" in flattened or "сдс плюс" in flattened: + return "sds-plus" + tokens = set(tokenize(flattened, already_normalized=True)) + if "sds" in tokens or "сдс" in tokens: + return "sds" + return None + + +def _detect_variant( + normalized: str, product_type: str, *, catalog_item: bool +) -> str | None: + text = f" {normalized.replace('-', ' ')} " + + if product_type == "саморез": + is_gkl = " гкл " in text or " гипсокартон" in text + has_wood = "по дерев" in text or "гипсокартон дерев" in text + has_metal = "по металл" in text or "гипсокартон металл" in text + if is_gkl and has_wood: + return "gkl_wood" + if is_gkl and has_metal: + return "gkl_metal" + if is_gkl: + return "gkl" + if has_wood: + return "wood" + if has_metal: + return "metal" + + if product_type == "кабель": + if " шввп " in text: + return "shvvp" + if " пвс " in text: + return "pvs" + if " ввг" in text: + if " ls " in text or " лс " in text: + return "vvg_ls" + return "vvg" + + if product_type == "провод" and " пугв " in text: + return "pugv" + + if product_type == "диск": + if "пильн" in text: + return "saw_wood" + if "зачист" in text: + return "grinding" + if "лепест" in text: + return "flap" + if "отрез" in text: + if "нержав" in text: + return "cut_stainless" + if "камн" in text: + return "cut_stone" + if "металл" in text: + return "cut_metal" + return "cut" + + if product_type == "сверло": + if "по металл" in text: + return "metal" + if "по дерев" in text: + return "wood" + if "по бетон" in text: + return "concrete" + + if product_type == "перчатки": + for marker, variant in ( + ("нитрил", "nitrile"), + ("спилк", "split_leather"), + ("прорезин", "rubberized"), + ("латекс", "latex"), + ("полиуретан", "polyurethane"), + ("пвх", "pvc"), + ): + if marker in text: + return variant + + if product_type == "дюбель": + if "гвозд" in text: + return "nail" + if "распор" in text: + return "spacer" + + if product_type == "шайба": + if "гровер" in text: + return "spring" + if "плоск" in text: + return "flat" + + if product_type == "анкер": + if "клинов" in text: + return "wedge" + if "с гайк" in text: + return "with_nut" + + if product_type == "ключ": + if "разводн" in text: + return "adjustable" + if "рожков" in text: + return "open_end" + + if product_type == "гипсокартон": + if "влагостой" in text: + return "moisture" + if "огнестой" in text: + return "fire" + if catalog_item: + return "standard" + if "обычн" in text or "стандартн" in text: + return "standard" + + if product_type == "профиль": + if "направля" in text or " пн " in text: + return "guide" + if "стоеч" in text or " пс " in text: + return "stud" + if "потолоч" in text or " пп " in text: + return "ceiling" + + if product_type == "герметик": + if "силикон" in text: + return "silicone" + if "акрил" in text: + return "acrylic" + + if product_type == "пена": + if "профессион" in text: + return "professional" + if "бытов" in text: + return "household" + + if product_type == "хомут" and "червяч" in text: + return "worm" + + return None + + +def parse_attributes( + text: str, + product_type: str, + *, + unit: str | None = None, + catalog_item: bool = False, + brand_index: BrandIndex | None = None, +) -> ParsedAttributes: + normalized = normalize_text(text) + dimensions = _extract_dimensions(normalized) + + teeth_match = _TEETH_RE.search(normalized) + teeth = int(teeth_match.group(1)) if teeth_match else None + + if not dimensions and product_type in _PAIR_DIMENSION_TYPES: + pair_match = _PAIR_WITH_NA_RE.search(normalized) + if pair_match is not None: + dimensions = (_decimal(pair_match.group(1)), _decimal(pair_match.group(2))) + + if product_type == "труба" and len(dimensions) == 2: + wall_match = re.search(rf"\bстенк\w*\s+({_NUMBER}|полтора)\b", normalized) + if wall_match is not None: + wall_value = Decimal("1.5") if wall_match.group(1) == "полтора" else _decimal( + wall_match.group(1) + ) + dimensions = (*dimensions, wall_value) + + mm_values = [_decimal(match.group(1)) for match in _MM_RE.finditer(normalized)] + meter_values = [_decimal(match.group(1)) for match in _METER_RE.finditer(normalized)] + + # ``190 на 48 зубьев`` describes diameter and tooth count, not 190x48. + saw_pair = re.search( + rf"\b({_NUMBER})\s+на\s+\d+\s*(?:зуб(?:ьев|а|ов)?)\b", + normalized, + ) + if saw_pair is not None: + mm_values.append(_decimal(saw_pair.group(1))) + + if ( + not catalog_item + and product_type in _SINGLE_SIZE_AFTER_NA_TYPES + and saw_pair is None + ): + after_na = re.search(rf"\bна\s+({_NUMBER})(?!\s*(?:в|вольт|вт|ватт))", normalized) + if after_na is not None: + value = _decimal(after_na.group(1)) + if value not in mm_values: + mm_values.append(value) + + if not dimensions and not catalog_item and product_type == "гипсокартон": + # Colloquial requests often contain only sheet thickness: ``гкл 9.5``. + bare_number = re.search(rf"(? bool: + if candidate is None: + return False + if product_type == "саморез": + if query == "gkl": + return candidate.startswith("gkl_") + return query == candidate + if product_type == "кабель" and query == "vvg": + # Omitting LS is not enough evidence to reject the safer cable variant. + # It therefore remains a possible candidate and may produce ambiguous. + return candidate in {"vvg", "vvg_ls"} + if product_type == "диск" and query == "cut": + return candidate.startswith("cut_") + return query == candidate + + +def _dimensions_compatible( + query: tuple[Decimal, ...], candidate: ParsedAttributes +) -> bool: + if candidate.dimensions and len(query) <= len(candidate.dimensions): + return all(value == candidate.dimensions[index] for index, value in enumerate(query)) + if len(query) == 1: + value = query[0] + if candidate.mm_values and value in candidate.mm_values: + return True + if candidate.dimensions and candidate.dimensions[0] == value: + return True + return False + + +def compare_attributes( + product_type: str, + query: ParsedAttributes, + candidate: ParsedAttributes, + *, + candidate_unit: str, +) -> Compatibility: + matched = 0 + total = 0 + + def require(condition: bool, name: str) -> Compatibility | None: + nonlocal matched, total + total += 1 + if not condition: + return Compatibility(False, matched, total, name) + matched += 1 + return None + + if query.variant is not None: + conflict = require( + _variant_compatible(product_type, query.variant, candidate.variant), "variant" + ) + if conflict: + return conflict + + if query.dimensions: + conflict = require(_dimensions_compatible(query.dimensions, candidate), "dimensions") + if conflict: + return conflict + + for value in query.mm_values: + condition = ( + value in candidate.mm_values + or bool(candidate.dimensions and candidate.dimensions[0] == value) + or candidate.thread_length_mm == value + ) + conflict = require(condition, "millimetres") + if conflict: + return conflict + + for value in query.meter_values: + condition = value in candidate.meter_values or candidate.thread_length_mm == value * 1000 + conflict = require(condition, "metres") + if conflict: + return conflict + + for query_value, candidate_value, name in ( + (query.thread_size, candidate.thread_size, "thread"), + (query.profile, candidate.profile, "profile"), + (query.grit, candidate.grit, "grit"), + (query.voltage_v, candidate.voltage_v, "voltage"), + (query.power_w, candidate.power_w, "power"), + (query.teeth, candidate.teeth, "teeth"), + (query.pack_size, candidate.pack_size, "pack_size"), + ): + if query_value is not None: + conflict = require(query_value == candidate_value, name) + if conflict: + return conflict + + if query.unit_hint is not None: + conflict = require(normalize_text(candidate_unit) == query.unit_hint, "unit") + if conflict: + return conflict + + if query.brand is not None and query.brand_required: + conflict = require(query.brand == candidate.brand, "brand") + if conflict: + return conflict + + if query.model_codes: + conflict = require(query.model_codes.issubset(candidate.model_codes), "model") + if conflict: + return conflict + + if query.sds_type is not None: + condition = ( + candidate.sds_type is not None + if query.sds_type == "sds" + else query.sds_type == candidate.sds_type + ) + conflict = require(condition, "sds") + if conflict: + return conflict + + return Compatibility(True, matched, total) diff --git a/app/catalog.py b/app/catalog.py index 16ab977..e64299b 100644 --- a/app/catalog.py +++ b/app/catalog.py @@ -5,6 +5,10 @@ from dataclasses import dataclass from decimal import Decimal, InvalidOperation from pathlib import Path +from app.aliases import ProductAliasIndex +from app.attributes import BrandIndex, ParsedAttributes, parse_attributes +from app.normalization import meaningful_tokens, normalize_text, tokenize + @dataclass(frozen=True, slots=True) class RawCatalogItem: @@ -14,6 +18,18 @@ class RawCatalogItem: price: Decimal +@dataclass(frozen=True, slots=True) +class CatalogItem: + sku: str + name: str + unit: str + price: Decimal + normalized_name: str + search_tokens: tuple[str, ...] + product_type: str + attributes: ParsedAttributes + + def load_raw_catalog(path: Path) -> tuple[RawCatalogItem, ...]: if not path.is_file(): raise FileNotFoundError(f"Catalog file does not exist: {path}") @@ -52,3 +68,36 @@ def load_raw_catalog(path: Path) -> tuple[RawCatalogItem, ...]: if not items: raise ValueError("Catalog is empty") return tuple(items) + + +def build_catalog( + raw_items: tuple[RawCatalogItem, ...], + alias_index: ProductAliasIndex, + brand_index: BrandIndex, +) -> tuple[CatalogItem, ...]: + result: list[CatalogItem] = [] + for raw in raw_items: + normalized = normalize_text(raw.name) + product_type = alias_index.detect_catalog_type(raw.name) + attributes = parse_attributes( + raw.name, + product_type, + unit=raw.unit, + catalog_item=True, + brand_index=brand_index, + ) + result.append( + CatalogItem( + sku=raw.sku, + name=raw.name, + unit=raw.unit, + price=raw.price, + normalized_name=normalized, + search_tokens=meaningful_tokens( + tokenize(normalized, already_normalized=True) + ), + product_type=product_type, + attributes=attributes, + ) + ) + return tuple(result) diff --git a/app/main.py b/app/main.py index b441bf5..60e0caf 100644 --- a/app/main.py +++ b/app/main.py @@ -5,26 +5,24 @@ from pathlib import Path from fastapi import FastAPI -from app.catalog import load_raw_catalog -from app.models import MatchRequest, MatchResponse, MatchResult +from app.matcher import CatalogMatcher +from app.models import MatchRequest, MatchResponse BASE_DIR = Path(__file__).resolve().parent.parent CATALOG_PATH = Path(os.environ.get("CATALOG_PATH", BASE_DIR / "catalog_excel.csv")) -RAW_CATALOG = load_raw_catalog(CATALOG_PATH) +MATCHER = CatalogMatcher(CATALOG_PATH) -app = FastAPI(title="Offline catalog matcher", version="0.1.0") - - -@app.get("/health") -def health() -> dict[str, object]: - return {"status": "ok", "catalog_items": len(RAW_CATALOG)} +# The task asks for one application endpoint. OpenAPI/Swagger routes are disabled +# so the running service exposes only POST /match. +app = FastAPI( + title="Offline catalog matcher", + version="0.2.0", + docs_url=None, + redoc_url=None, + openapi_url=None, +) @app.post("/match", response_model=MatchResponse) def match(request: MatchRequest) -> MatchResponse: - return MatchResponse( - results=[ - MatchResult(message=message, status="not_found", candidates=[]) - for message in request.messages - ] - ) + return MatchResponse(results=[MATCHER.match(message) for message in request.messages]) diff --git a/app/matcher.py b/app/matcher.py new file mode 100644 index 0000000..0db3973 --- /dev/null +++ b/app/matcher.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +from app.aliases import ProductAliasIndex, ProductTypeMatch +from app.attributes import BrandIndex, Compatibility, compare_attributes, parse_attributes +from app.catalog import CatalogItem, build_catalog, load_raw_catalog +from app.intents import detect_service_intent +from app.models import Candidate, MatchResult +from app.search_index import LexicalIndex, LexicalScore + + +@dataclass(frozen=True, slots=True) +class _RankedCandidate: + item: CatalogItem + lexical: LexicalScore + compatibility: Compatibility + ranking_score: float + + +class CatalogMatcher: + def __init__(self, catalog_path: Path) -> None: + self.alias_index = ProductAliasIndex() + self.brand_index = BrandIndex() + self.items = build_catalog( + load_raw_catalog(catalog_path), self.alias_index, self.brand_index + ) + self.lexical_index = LexicalIndex(self.items) + + type_index: dict[str, list[int]] = defaultdict(list) + model_index: dict[str, set[int]] = defaultdict(set) + for index, item in enumerate(self.items): + type_index[item.product_type].append(index) + for model_code in item.attributes.model_codes: + model_index[model_code].add(index) + self.type_index = { + product_type: tuple(indexes) for product_type, indexes in type_index.items() + } + self.model_index = { + model: frozenset(indexes) for model, indexes in model_index.items() + } + + def match(self, message: str) -> MatchResult: + if not message or not message.strip(): + return MatchResult(message=message, status="not_found", candidates=[]) + + product_match = self.alias_index.detect(message) + provisional_attributes = parse_attributes( + message, + product_match.canonical if product_match else "", + brand_index=self.brand_index, + ) + + if product_match is None: + model_candidates = self._model_candidates(provisional_attributes.model_codes) + if model_candidates: + product_types = {self.items[index].product_type for index in model_candidates} + if len(product_types) == 1: + product_type = next(iter(product_types)) + product_match = ProductTypeMatch( + canonical=product_type, + score=1.0, + exact=True, + alias="model_code", + token_start=0, + ) + if product_match is None: + # Product evidence always has priority. Service intent detection + # is only a diagnostic fallback; both unknown and service + # messages share the required not_found response. + detect_service_intent(message) + return MatchResult(message=message, status="not_found", candidates=[]) + + product_type = product_match.canonical + query_attributes = parse_attributes( + message, + product_type, + brand_index=self.brand_index, + ) + candidate_indices = self.type_index.get(product_type, ()) + + model_candidates = self._model_candidates(query_attributes.model_codes) + if model_candidates: + candidate_indices = tuple( + index for index in candidate_indices if index in model_candidates + ) + + compatible: dict[int, Compatibility] = {} + for index in candidate_indices: + item = self.items[index] + comparison = compare_attributes( + product_type, + query_attributes, + item.attributes, + candidate_unit=item.unit, + ) + if comparison.compatible: + compatible[index] = comparison + + if not compatible: + return MatchResult(message=message, status="not_found", candidates=[]) + + lexical_scores = self.lexical_index.rank( + message, product_type, tuple(compatible.keys()) + ) + ranked = self._apply_preferences( + lexical_scores, compatible, cheap=query_attributes.cheap_preference + ) + if not ranked: + return MatchResult(message=message, status="not_found", candidates=[]) + + status = "matched" if len(ranked) == 1 else "ambiguous" + response_candidates = [ + Candidate( + sku=candidate.item.sku, + confidence=self._confidence( + candidate, + product_match=product_match, + candidate_count=len(ranked), + specificity=query_attributes.specificity, + status=status, + rank=rank, + ), + ) + for rank, candidate in enumerate(ranked[:3]) + ] + return MatchResult( + message=message, + status=status, + candidates=response_candidates, + ) + + def _model_candidates(self, model_codes: frozenset[str]) -> frozenset[int]: + if not model_codes: + return frozenset() + sets = [self.model_index.get(code, frozenset()) for code in model_codes] + if not sets or any(not indexes for indexes in sets): + return frozenset() + result = set(sets[0]) + for indexes in sets[1:]: + result.intersection_update(indexes) + return frozenset(result) + + def _apply_preferences( + self, + lexical_scores: list[LexicalScore], + compatible: dict[int, Compatibility], + *, + cheap: bool, + ) -> list[_RankedCandidate]: + if not lexical_scores: + return [] + + prices = [self.items[score.item_index].price for score in lexical_scores] + minimum_price = min(prices) + maximum_price = max(prices) + price_range = maximum_price - minimum_price + + ranked: list[_RankedCandidate] = [] + for lexical in lexical_scores: + item = self.items[lexical.item_index] + if cheap: + cheap_score = ( + 1.0 + if price_range == 0 + else float((maximum_price - item.price) / price_range) + ) + ranking_score = lexical.score * 0.68 + cheap_score * 0.32 + else: + ranking_score = lexical.score + ranked.append( + _RankedCandidate( + item=item, + lexical=lexical, + compatibility=compatible[lexical.item_index], + ranking_score=ranking_score, + ) + ) + + ranked.sort( + key=lambda candidate: ( + -candidate.ranking_score, + candidate.item.price if cheap else candidate.item.sku, + ) + ) + return ranked + + @staticmethod + def _confidence( + candidate: _RankedCandidate, + *, + product_match: ProductTypeMatch, + candidate_count: int, + specificity: int, + status: str, + rank: int, + ) -> float: + attribute_signal = ( + candidate.compatibility.score + if candidate.compatibility.total_fields > 0 + else 0.55 + ) + uniqueness_signal = 1.0 if candidate_count == 1 else max(0.35, 0.72 - rank * 0.10) + specificity_signal = min(1.0, 0.35 + specificity * 0.16) + value = ( + product_match.score * 0.30 + + candidate.lexical.score * 0.27 + + attribute_signal * 0.23 + + uniqueness_signal * 0.12 + + specificity_signal * 0.08 + ) + if status == "matched": + value = max(value, 0.82) + else: + value = min(value, 0.84) + return round(max(0.01, min(0.99, value)), 4)