Improve sparse and structured catalog matching
This commit is contained in:
+15
-2
@@ -30,7 +30,11 @@ _VOLTAGE_RE = re.compile(
|
||||
rf"(?<![a-zа-я0-9-])({_NUMBER})\s*(?:v|в|вольт\w*)\b", re.IGNORECASE
|
||||
)
|
||||
_POWER_RE = re.compile(rf"\b({_NUMBER})\s*(?:w|вт|ватт\w*)\b", re.IGNORECASE)
|
||||
_TEETH_RE = re.compile(r"\b(\d+)\s*(?:зуб(?:ьев|а|ов)?|z)\b", re.IGNORECASE)
|
||||
_TEETH_RE = re.compile(r"\b(\d+)\s*(?:зуб(?:ьев|а|ов)?|[zt])\b", re.IGNORECASE)
|
||||
_SAW_COMPACT_RE = re.compile(
|
||||
rf"(?<![a-zа-я0-9.])({_NUMBER})(?:x|\s+)(\d+)\s*[zt]\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PACK_SIZE_RE = re.compile(
|
||||
r"(?:пачк\w*|упаковк\w*|\bуп\.?)(?:\s+(?:по|на))?\s*(\d+)\s*(?:шт\w*)?",
|
||||
re.IGNORECASE,
|
||||
@@ -377,8 +381,15 @@ def parse_attributes(
|
||||
normalized = normalize_text(text)
|
||||
dimensions = _extract_dimensions(normalized)
|
||||
|
||||
compact_saw_match = _SAW_COMPACT_RE.search(normalized)
|
||||
teeth_match = _TEETH_RE.search(normalized)
|
||||
teeth = int(teeth_match.group(1)) if teeth_match else None
|
||||
teeth = (
|
||||
int(compact_saw_match.group(2))
|
||||
if compact_saw_match is not None
|
||||
else int(teeth_match.group(1))
|
||||
if teeth_match is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if not dimensions and product_type in _PAIR_DIMENSION_TYPES:
|
||||
pair_match = _PAIR_WITH_NA_RE.search(normalized)
|
||||
@@ -395,6 +406,8 @@ def parse_attributes(
|
||||
|
||||
mm_values = [_decimal(match.group(1)) for match in _MM_RE.finditer(normalized)]
|
||||
meter_values = [_decimal(match.group(1)) for match in _METER_RE.finditer(normalized)]
|
||||
if compact_saw_match is not None:
|
||||
mm_values.append(_decimal(compact_saw_match.group(1)))
|
||||
|
||||
# ``190 на 48 зубьев`` describes diameter and tooth count, not 190x48.
|
||||
saw_pair = re.search(
|
||||
|
||||
+188
-5
@@ -15,6 +15,7 @@ from app.attributes import (
|
||||
from app.catalog import CatalogItem, build_catalog, load_raw_catalog
|
||||
from app.intents import detect_service_intent
|
||||
from app.models import Candidate, MatchResult
|
||||
from app.normalization import ZERO_WEIGHT_TOKENS, normalize_text, tokenize
|
||||
from app.search_index import LexicalIndex, LexicalScore
|
||||
|
||||
|
||||
@@ -26,6 +27,43 @@ class _RankedCandidate:
|
||||
ranking_score: float
|
||||
|
||||
|
||||
_STRUCTURED_QUERY_WORDS = frozenset(
|
||||
{
|
||||
"x",
|
||||
"max",
|
||||
"plus",
|
||||
"sds",
|
||||
"t",
|
||||
"z",
|
||||
"v",
|
||||
"w",
|
||||
"в",
|
||||
"ватт",
|
||||
"ватта",
|
||||
"ваттов",
|
||||
"вольт",
|
||||
"вольта",
|
||||
"вольтов",
|
||||
"зуб",
|
||||
"зуба",
|
||||
"зубов",
|
||||
"зубьев",
|
||||
"м",
|
||||
"метр",
|
||||
"метра",
|
||||
"метров",
|
||||
"миллиметр",
|
||||
"миллиметра",
|
||||
"миллиметров",
|
||||
"мм",
|
||||
"размер",
|
||||
"размера",
|
||||
"размером",
|
||||
"вт",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class CatalogMatcher:
|
||||
def __init__(self, catalog_path: Path) -> None:
|
||||
self.alias_index = ProductAliasIndex()
|
||||
@@ -78,15 +116,46 @@ class CatalogMatcher:
|
||||
alias="model_code",
|
||||
token_start=0,
|
||||
)
|
||||
if product_match is None and provisional_attributes.teeth is not None:
|
||||
# A tooth count is type-specific evidence in this domain: only
|
||||
# saw blades carry it. This safely supports requests such as
|
||||
# "210 мм 24 зуба" without requiring the word "диск".
|
||||
product_match = ProductTypeMatch(
|
||||
canonical="диск",
|
||||
score=1.0,
|
||||
exact=True,
|
||||
alias="tooth_count",
|
||||
token_start=0,
|
||||
)
|
||||
if product_match is None:
|
||||
brand_result = self._match_brand_only(message, provisional_attributes)
|
||||
if brand_result is not None:
|
||||
return brand_result
|
||||
# Product evidence always has priority. Service intent detection
|
||||
# is only a diagnostic fallback; both unknown and service
|
||||
# messages share the required not_found response.
|
||||
detect_service_intent(message)
|
||||
return MatchResult(message=message, status="not_found", candidates=[])
|
||||
structured_result = self._match_structured_only(
|
||||
message, provisional_attributes
|
||||
)
|
||||
if structured_result is not None:
|
||||
return structured_result
|
||||
# Trigrams used to rank only inside a type selected by aliases.
|
||||
# Let them infer that missing type as well, but only when the
|
||||
# leading catalog rows agree and share strong word-level evidence.
|
||||
# This handles "удар" -> "ударная дрель" while
|
||||
# keeping numeric coincidences and arbitrary text out.
|
||||
if detect_service_intent(message) is None:
|
||||
lexical_inference = self.lexical_index.infer_product_type(message)
|
||||
if lexical_inference is not None:
|
||||
product_match = ProductTypeMatch(
|
||||
canonical=lexical_inference.product_type,
|
||||
score=lexical_inference.score,
|
||||
exact=lexical_inference.query_word
|
||||
== lexical_inference.catalog_word,
|
||||
alias=lexical_inference.catalog_word,
|
||||
token_start=0,
|
||||
)
|
||||
if product_match is None:
|
||||
return MatchResult(
|
||||
message=message, status="not_found", candidates=[]
|
||||
)
|
||||
|
||||
product_type = product_match.canonical
|
||||
query_attributes = parse_attributes(
|
||||
@@ -154,6 +223,7 @@ class CatalogMatcher:
|
||||
query_attributes.brand is None
|
||||
or not query_attributes.brand_required
|
||||
or query_attributes.model_codes
|
||||
or query_attributes.specificity != 1
|
||||
):
|
||||
return None
|
||||
|
||||
@@ -207,6 +277,119 @@ class CatalogMatcher:
|
||||
]
|
||||
return MatchResult(message=message, status=status, candidates=candidates)
|
||||
|
||||
def _match_structured_only(
|
||||
self, message: str, query_attributes: ParsedAttributes
|
||||
) -> MatchResult | None:
|
||||
if not self._is_structured_only_query(message, query_attributes):
|
||||
return None
|
||||
|
||||
compatible: dict[int, Compatibility] = {}
|
||||
for index, item in enumerate(self.items):
|
||||
comparison = compare_attributes(
|
||||
item.product_type,
|
||||
query_attributes,
|
||||
item.attributes,
|
||||
candidate_unit=item.unit,
|
||||
)
|
||||
if comparison.compatible and comparison.matched_fields > 0:
|
||||
compatible[index] = comparison
|
||||
|
||||
if not compatible:
|
||||
return None
|
||||
|
||||
lexical_scores = self.lexical_index.rank(
|
||||
message, "", tuple(compatible.keys())
|
||||
)
|
||||
ranked = self._apply_preferences(
|
||||
lexical_scores,
|
||||
compatible,
|
||||
cheap=query_attributes.cheap_preference,
|
||||
)
|
||||
if not ranked:
|
||||
return None
|
||||
|
||||
status = "matched" if len(ranked) == 1 else "ambiguous"
|
||||
selected = self._select_diverse_product_types(ranked, limit=3)
|
||||
structured_signal = ProductTypeMatch(
|
||||
canonical="",
|
||||
score=0.90,
|
||||
exact=True,
|
||||
alias="structured_attributes",
|
||||
token_start=0,
|
||||
)
|
||||
candidates = [
|
||||
Candidate(
|
||||
sku=candidate.item.sku,
|
||||
confidence=self._confidence(
|
||||
candidate,
|
||||
product_match=structured_signal,
|
||||
candidate_count=len(ranked),
|
||||
specificity=query_attributes.specificity,
|
||||
status=status,
|
||||
rank=rank,
|
||||
),
|
||||
)
|
||||
for rank, candidate in enumerate(selected)
|
||||
]
|
||||
return MatchResult(message=message, status=status, candidates=candidates)
|
||||
|
||||
def _is_structured_only_query(
|
||||
self, message: str, query_attributes: ParsedAttributes
|
||||
) -> bool:
|
||||
has_strong_evidence = any(
|
||||
(
|
||||
bool(query_attributes.dimensions),
|
||||
bool(query_attributes.mm_values),
|
||||
bool(query_attributes.meter_values),
|
||||
query_attributes.thread_size is not None,
|
||||
query_attributes.profile is not None,
|
||||
query_attributes.grit is not None,
|
||||
query_attributes.voltage_v is not None,
|
||||
query_attributes.power_w is not None,
|
||||
query_attributes.teeth is not None,
|
||||
query_attributes.brand is not None
|
||||
and query_attributes.brand_required,
|
||||
bool(query_attributes.model_codes),
|
||||
query_attributes.sds_type is not None,
|
||||
)
|
||||
)
|
||||
if not has_strong_evidence:
|
||||
return False
|
||||
|
||||
allowed_words = set(ZERO_WEIGHT_TOKENS) | set(_STRUCTURED_QUERY_WORDS)
|
||||
brand_match = self.brand_index.detect(message)
|
||||
if brand_match is not None:
|
||||
allowed_words.update(tokenize(brand_match.alias))
|
||||
|
||||
for token in tokenize(normalize_text(message), already_normalized=True):
|
||||
if any(character.isdigit() for character in token):
|
||||
continue
|
||||
if token in allowed_words:
|
||||
continue
|
||||
if token.startswith(("бюджет", "дешев", "недорог")):
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _select_diverse_product_types(
|
||||
ranked: list[_RankedCandidate], *, limit: int
|
||||
) -> list[_RankedCandidate]:
|
||||
if len({candidate.item.product_type for candidate in ranked}) == 1:
|
||||
return ranked[:limit]
|
||||
|
||||
selected: list[_RankedCandidate] = []
|
||||
product_types: set[str] = set()
|
||||
|
||||
for candidate in ranked:
|
||||
if candidate.item.product_type in product_types:
|
||||
continue
|
||||
selected.append(candidate)
|
||||
product_types.add(candidate.item.product_type)
|
||||
if len(selected) == limit:
|
||||
break
|
||||
return selected
|
||||
|
||||
def _model_candidates(self, model_codes: frozenset[str]) -> frozenset[int]:
|
||||
if not model_codes:
|
||||
return frozenset()
|
||||
|
||||
+95
-1
@@ -6,7 +6,12 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from app.normalization import meaningful_tokens, normalize_text, tokenize, word_tokens
|
||||
from app.typo import character_trigrams, trigram_dice, word_similarity
|
||||
from app.typo import (
|
||||
character_trigrams,
|
||||
damerau_levenshtein_distance,
|
||||
trigram_dice,
|
||||
word_similarity,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.catalog import CatalogItem
|
||||
@@ -22,6 +27,14 @@ class LexicalScore:
|
||||
exact_coverage: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProductTypeInference:
|
||||
product_type: str
|
||||
score: float
|
||||
query_word: str
|
||||
catalog_word: str
|
||||
|
||||
|
||||
class LexicalIndex:
|
||||
"""Small in-memory lexical index: BM25 + char trigrams + typo alignment."""
|
||||
|
||||
@@ -69,6 +82,87 @@ class LexicalIndex:
|
||||
counts.update(self._trigram_postings.get(trigram, ()))
|
||||
return tuple(index for index, _ in counts.most_common(limit))
|
||||
|
||||
def infer_product_type(self, query_text: str) -> ProductTypeInference | None:
|
||||
"""Infer a missing product type from stable lexical evidence.
|
||||
|
||||
Trigrams provide the shortlist, but are deliberately not sufficient on
|
||||
their own: sizes and random strings can share a few character triples
|
||||
with unrelated catalog rows. The leading rows must agree on one type
|
||||
and share a catalog word that is an exact match, a morphological prefix,
|
||||
or a tightly bounded typo of a query word.
|
||||
"""
|
||||
|
||||
candidate_indices = self.trigram_candidates(query_text, limit=60)
|
||||
if len(candidate_indices) < 3:
|
||||
return None
|
||||
|
||||
leaders = self.rank(query_text, "", candidate_indices)[:3]
|
||||
if len(leaders) < 3:
|
||||
return None
|
||||
|
||||
product_types = {
|
||||
self._items[leader.item_index].product_type for leader in leaders
|
||||
}
|
||||
if len(product_types) != 1:
|
||||
return None
|
||||
|
||||
common_words = set(self._document_words[leaders[0].item_index])
|
||||
for leader in leaders[1:]:
|
||||
common_words.intersection_update(self._document_words[leader.item_index])
|
||||
if not common_words:
|
||||
return None
|
||||
|
||||
normalized = normalize_text(query_text)
|
||||
query_words = word_tokens(
|
||||
meaningful_tokens(tokenize(normalized, already_normalized=True))
|
||||
)
|
||||
best: tuple[float, str, str] | None = None
|
||||
for query_word in query_words:
|
||||
if len(query_word) < 4:
|
||||
continue
|
||||
for catalog_word in common_words:
|
||||
score = self._type_inference_word_score(query_word, catalog_word)
|
||||
if score <= 0:
|
||||
continue
|
||||
evidence = (score, query_word, catalog_word)
|
||||
if best is None or evidence > best:
|
||||
best = evidence
|
||||
|
||||
if best is None:
|
||||
return None
|
||||
|
||||
score, query_word, catalog_word = best
|
||||
return ProductTypeInference(
|
||||
product_type=next(iter(product_types)),
|
||||
score=score,
|
||||
query_word=query_word,
|
||||
catalog_word=catalog_word,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _type_inference_word_score(query_word: str, catalog_word: str) -> float:
|
||||
if query_word == catalog_word:
|
||||
return 1.0
|
||||
|
||||
shorter_length = min(len(query_word), len(catalog_word))
|
||||
longer_length = max(len(query_word), len(catalog_word))
|
||||
if shorter_length >= 4 and (
|
||||
query_word.startswith(catalog_word)
|
||||
or catalog_word.startswith(query_word)
|
||||
):
|
||||
length_ratio = shorter_length / longer_length
|
||||
if length_ratio >= 0.55:
|
||||
return 0.72 + length_ratio * 0.18
|
||||
|
||||
distance = damerau_levenshtein_distance(query_word, catalog_word)
|
||||
if shorter_length >= 5 and distance <= 1:
|
||||
return 0.90
|
||||
if longer_length >= 8 and distance <= 2:
|
||||
edit_similarity = 1.0 - distance / longer_length
|
||||
if edit_similarity >= 0.75:
|
||||
return 0.82
|
||||
return 0.0
|
||||
|
||||
def rank(
|
||||
self,
|
||||
query_text: str,
|
||||
|
||||
+21
-1
@@ -98,4 +98,24 @@ SELECT * FROM catalog;
|
||||
мне два метра того красного
|
||||
хочу всё и сразу за сто рублей
|
||||
статус заказа 4512 и бур 8х160
|
||||
диск
|
||||
диск
|
||||
24 зуба
|
||||
210 мм 24 зуба
|
||||
125 мм
|
||||
12 в
|
||||
900 вт
|
||||
M10
|
||||
P120
|
||||
SDS-plus
|
||||
PH2
|
||||
20x20x2
|
||||
ToolKraft 125 мм
|
||||
99999 мм
|
||||
аквариум 125 мм
|
||||
удар
|
||||
ударная
|
||||
удар мощный
|
||||
удар мощно 900 хромированый анус долбить
|
||||
дрель удар мощно 900 хромированый анус долбить
|
||||
анус
|
||||
долбить
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from app.catalog import CatalogItem, RawCatalogItem, load_raw_catalog
|
||||
from app.matcher import CatalogMatcher
|
||||
|
||||
|
||||
CATALOG_ITEMS = load_raw_catalog(
|
||||
Path(__file__).resolve().parents[1] / "catalog_excel.csv"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("item", CATALOG_ITEMS, ids=lambda item: item.sku)
|
||||
def test_full_catalog_name_retrieves_its_own_sku(
|
||||
matcher: CatalogMatcher, item: RawCatalogItem
|
||||
) -> None:
|
||||
result = matcher.match(item.name)
|
||||
|
||||
assert item.sku in {candidate.sku for candidate in result.candidates}
|
||||
|
||||
|
||||
def test_every_canonical_product_type_returns_only_its_own_items(
|
||||
matcher: CatalogMatcher,
|
||||
) -> None:
|
||||
items_by_sku = {item.sku: item for item in matcher.items}
|
||||
product_types = {item.product_type for item in matcher.items}
|
||||
|
||||
for product_type in product_types:
|
||||
result = matcher.match(product_type)
|
||||
|
||||
assert result.candidates, product_type
|
||||
assert all(
|
||||
items_by_sku[candidate.sku].product_type == product_type
|
||||
for candidate in result.candidates
|
||||
), product_type
|
||||
|
||||
|
||||
def test_every_catalog_model_code_returns_only_matching_items(
|
||||
matcher: CatalogMatcher,
|
||||
) -> None:
|
||||
items_by_sku = {item.sku: item for item in matcher.items}
|
||||
model_codes = {
|
||||
model
|
||||
for item in matcher.items
|
||||
for model in item.attributes.model_codes
|
||||
}
|
||||
|
||||
for model_code in model_codes:
|
||||
result = matcher.match(model_code)
|
||||
|
||||
assert result.candidates, model_code
|
||||
assert all(
|
||||
model_code in items_by_sku[candidate.sku].attributes.model_codes
|
||||
for candidate in result.candidates
|
||||
), model_code
|
||||
|
||||
|
||||
def test_every_catalog_brand_returns_only_its_own_items(
|
||||
matcher: CatalogMatcher,
|
||||
) -> None:
|
||||
items_by_sku = {item.sku: item for item in matcher.items}
|
||||
brands = {
|
||||
item.attributes.brand
|
||||
for item in matcher.items
|
||||
if item.attributes.brand is not None
|
||||
}
|
||||
|
||||
for brand in brands:
|
||||
result = matcher.match(brand)
|
||||
|
||||
assert result.candidates, brand
|
||||
assert all(
|
||||
items_by_sku[candidate.sku].attributes.brand == brand
|
||||
for candidate in result.candidates
|
||||
), brand
|
||||
|
||||
|
||||
def _number_text(value: Decimal) -> str:
|
||||
return format(value, "f")
|
||||
|
||||
|
||||
def _assert_query_returns_only(
|
||||
matcher: CatalogMatcher,
|
||||
query: str,
|
||||
predicate: Callable[[CatalogItem], bool],
|
||||
) -> None:
|
||||
items_by_sku = {item.sku: item for item in matcher.items}
|
||||
result = matcher.match(query)
|
||||
|
||||
assert result.candidates, query
|
||||
assert all(
|
||||
predicate(items_by_sku[candidate.sku]) for candidate in result.candidates
|
||||
), query
|
||||
|
||||
|
||||
def test_every_catalog_millimetre_value_is_searchable_without_product_name(
|
||||
matcher: CatalogMatcher,
|
||||
) -> None:
|
||||
values = {
|
||||
value
|
||||
for item in matcher.items
|
||||
for value in (
|
||||
*item.attributes.mm_values,
|
||||
*((item.attributes.dimensions[0],) if item.attributes.dimensions else ()),
|
||||
*((item.attributes.thread_length_mm,) if item.attributes.thread_length_mm else ()),
|
||||
)
|
||||
}
|
||||
|
||||
for value in values:
|
||||
_assert_query_returns_only(
|
||||
matcher,
|
||||
f"{_number_text(value)} мм",
|
||||
lambda item, expected=value: (
|
||||
expected in item.attributes.mm_values
|
||||
or bool(
|
||||
item.attributes.dimensions
|
||||
and item.attributes.dimensions[0] == expected
|
||||
)
|
||||
or item.attributes.thread_length_mm == expected
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_every_catalog_dimension_is_searchable_without_product_name(
|
||||
matcher: CatalogMatcher,
|
||||
) -> None:
|
||||
dimensions = {
|
||||
item.attributes.dimensions
|
||||
for item in matcher.items
|
||||
if item.attributes.dimensions
|
||||
}
|
||||
|
||||
for dimension in dimensions:
|
||||
query = "x".join(_number_text(value) for value in dimension)
|
||||
_assert_query_returns_only(
|
||||
matcher,
|
||||
query,
|
||||
lambda item, expected=dimension: (
|
||||
len(item.attributes.dimensions) >= len(expected)
|
||||
and item.attributes.dimensions[: len(expected)] == expected
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "suffix"),
|
||||
[
|
||||
("voltage_v", " В"),
|
||||
("power_w", " Вт"),
|
||||
("thread_size", ""),
|
||||
("grit", ""),
|
||||
("profile", ""),
|
||||
("sds_type", ""),
|
||||
("teeth", " зубьев"),
|
||||
],
|
||||
)
|
||||
def test_every_scalar_catalog_attribute_is_searchable_without_product_name(
|
||||
matcher: CatalogMatcher,
|
||||
field: str,
|
||||
suffix: str,
|
||||
) -> None:
|
||||
values = {
|
||||
getattr(item.attributes, field)
|
||||
for item in matcher.items
|
||||
if getattr(item.attributes, field) is not None
|
||||
}
|
||||
|
||||
for value in values:
|
||||
if field == "thread_size":
|
||||
query = f"M{_number_text(value)}"
|
||||
elif field == "grit":
|
||||
query = f"P{value}"
|
||||
else:
|
||||
value_text = _number_text(value) if isinstance(value, Decimal) else str(value)
|
||||
query = f"{value_text}{suffix}"
|
||||
_assert_query_returns_only(
|
||||
matcher,
|
||||
query,
|
||||
lambda item, expected=value: getattr(item.attributes, field) == expected,
|
||||
)
|
||||
@@ -184,6 +184,55 @@ def test_product_type_survives_typos(matcher: CatalogMatcher, query: str) -> Non
|
||||
assert all(candidate.sku.startswith("INS-") for candidate in result.candidates)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "skus"),
|
||||
[
|
||||
("удар", ("INS-0008", "INS-0009", "INS-0022")),
|
||||
("ударная", ("INS-0008", "INS-0009", "INS-0022")),
|
||||
("удар мощный", ("INS-0008", "INS-0009", "INS-0022")),
|
||||
(
|
||||
"удар мощно 900 хромированый анус долбить",
|
||||
("INS-0009", "INS-0016", "INS-0023"),
|
||||
),
|
||||
(
|
||||
"дрель удар мощно 900 хромированый анус долбить",
|
||||
("INS-0009", "INS-0023", "INS-0002"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_trigram_fallback_finds_product_type_from_sparse_or_noisy_text(
|
||||
matcher: CatalogMatcher, query: str, skus: tuple[str, ...]
|
||||
) -> None:
|
||||
result = matcher.match(query)
|
||||
|
||||
assert result.status == "ambiguous"
|
||||
assert tuple(candidate.sku for candidate in result.candidates) == skus
|
||||
items_by_sku = {item.sku: item for item in matcher.items}
|
||||
assert {
|
||||
items_by_sku[candidate.sku].product_type for candidate in result.candidates
|
||||
} == {"дрель"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
"анус",
|
||||
"долбить",
|
||||
"какой адрес",
|
||||
"аквариум 125 мм",
|
||||
"<script>alert(1)</script>",
|
||||
"qwerty asdf zxcv",
|
||||
],
|
||||
)
|
||||
def test_trigram_fallback_rejects_weak_or_accidental_similarity(
|
||||
matcher: CatalogMatcher, query: str
|
||||
) -> None:
|
||||
result = matcher.match(query)
|
||||
|
||||
assert result.status == "not_found"
|
||||
assert result.candidates == []
|
||||
|
||||
|
||||
def test_bare_size_is_parsed_without_unit(matcher: CatalogMatcher) -> None:
|
||||
assert matcher.match("бита ph2 50").candidates[0].sku == "BIT-0005"
|
||||
assert matcher.match("диск пильный 190 48 зубьев").candidates[0].sku == "DSK-0034"
|
||||
@@ -195,3 +244,110 @@ def test_unknown_size_does_not_fall_back_to_nearest_product(
|
||||
result = matcher.match("ушм 150")
|
||||
assert result.status == "not_found"
|
||||
assert result.candidates == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "sku"),
|
||||
[
|
||||
("165 мм 24 зуба", "DSK-0029"),
|
||||
("165 мм 36 зубов", "DSK-0030"),
|
||||
("165 мм 48 зубьев", "DSK-0031"),
|
||||
("190 мм 24 зуба", "DSK-0032"),
|
||||
("190 мм 36 зубов", "DSK-0033"),
|
||||
("190 мм 48 зубьев", "DSK-0034"),
|
||||
("210 мм 24 зуба", "DSK-0035"),
|
||||
("210 мм 36 зубов", "DSK-0036"),
|
||||
("210 мм 48 зубьев", "DSK-0037"),
|
||||
],
|
||||
)
|
||||
def test_tooth_count_infers_saw_blade_without_product_name(
|
||||
matcher: CatalogMatcher, query: str, sku: str
|
||||
) -> None:
|
||||
result = matcher.match(query)
|
||||
|
||||
assert result.status == "matched"
|
||||
assert [candidate.sku for candidate in result.candidates] == [sku]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "skus"),
|
||||
[
|
||||
("24 зуба", ("DSK-0029", "DSK-0032", "DSK-0035")),
|
||||
("36 зубов", ("DSK-0030", "DSK-0033", "DSK-0036")),
|
||||
("48 зубьев", ("DSK-0031", "DSK-0034", "DSK-0037")),
|
||||
],
|
||||
)
|
||||
def test_bare_tooth_count_returns_all_matching_saw_blade_sizes(
|
||||
matcher: CatalogMatcher, query: str, skus: tuple[str, ...]
|
||||
) -> None:
|
||||
result = matcher.match(query)
|
||||
|
||||
assert result.status == "ambiguous"
|
||||
assert tuple(candidate.sku for candidate in result.candidates) == skus
|
||||
|
||||
|
||||
def test_unknown_tooth_count_does_not_return_nearest_saw_blade(
|
||||
matcher: CatalogMatcher,
|
||||
) -> None:
|
||||
result = matcher.match("210 мм 25 зубьев")
|
||||
|
||||
assert result.status == "not_found"
|
||||
assert result.candidates == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
"пильный диск 190x48t",
|
||||
"190x48T",
|
||||
"190x48z",
|
||||
"190 48t",
|
||||
"190/48T",
|
||||
],
|
||||
)
|
||||
def test_compact_saw_notation_parses_diameter_and_teeth(
|
||||
matcher: CatalogMatcher, query: str
|
||||
) -> None:
|
||||
result = matcher.match(query)
|
||||
|
||||
assert result.status == "matched"
|
||||
assert [candidate.sku for candidate in result.candidates] == ["DSK-0034"]
|
||||
|
||||
|
||||
def test_bare_t_suffix_infers_saw_blades(matcher: CatalogMatcher) -> None:
|
||||
result = matcher.match("24T")
|
||||
|
||||
assert result.status == "ambiguous"
|
||||
assert [candidate.sku for candidate in result.candidates] == [
|
||||
"DSK-0029",
|
||||
"DSK-0032",
|
||||
"DSK-0035",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "status", "skus"),
|
||||
[
|
||||
("125 мм", "ambiguous", ("INS-0013", "DSK-0025", "RAS-0054")),
|
||||
("12 В", "ambiguous", ("INS-0003", "INS-0010", "INS-0024")),
|
||||
("900 Вт", "ambiguous", ("INS-0009", "INS-0013")),
|
||||
("M10", "ambiguous", ("KRP-0030", "KRP-0020", "KRP-0009")),
|
||||
("P120", "ambiguous", ("DSK-0024", "RAS-0050", "RAS-0057")),
|
||||
("SDS-plus", "ambiguous", ("BIT-0073", "INS-0012")),
|
||||
("PH2", "ambiguous", ("BIT-0004", "RIN-0010")),
|
||||
("20x20x2", "matched", ("TRB-0003",)),
|
||||
("ToolKraft 125 мм", "matched", ("INS-0027",)),
|
||||
("99999 мм", "not_found", ()),
|
||||
("аквариум 125 мм", "not_found", ()),
|
||||
],
|
||||
)
|
||||
def test_attribute_only_queries_use_exact_catalog_values(
|
||||
matcher: CatalogMatcher,
|
||||
query: str,
|
||||
status: str,
|
||||
skus: tuple[str, ...],
|
||||
) -> None:
|
||||
result = matcher.match(query)
|
||||
|
||||
assert result.status == status
|
||||
assert tuple(candidate.sku for candidate in result.candidates) == skus
|
||||
|
||||
@@ -44,10 +44,17 @@ def _load_message_cases() -> tuple[MessageCase, ...]:
|
||||
|
||||
|
||||
MESSAGE_CASES = _load_message_cases()
|
||||
VALIDATED_MESSAGE_CASES = tuple(
|
||||
case for case in MESSAGE_CASES if case.source == "messages.txt"
|
||||
)
|
||||
MORE_MESSAGE_CASES = tuple(
|
||||
case for case in MESSAGE_CASES if case.source == "more_messages.txt"
|
||||
)
|
||||
|
||||
# Baseline approved for the current corpus. The key is the message text so adding
|
||||
# or moving unrelated lines does not invalidate every expectation below.
|
||||
EXPECTED_RESULTS: dict[str, ExpectedResult] = {
|
||||
# Hand-reviewed acceptance results for every required messages.txt query.
|
||||
# Do not regenerate this table from CatalogMatcher output: it is the independent
|
||||
# oracle that should catch semantic regressions in the implementation.
|
||||
VALIDATED_MESSAGE_RESULTS: dict[str, ExpectedResult] = {
|
||||
"здравствуйте, есть саморезы гкл 3.5х25?": ("ambiguous", ("SAM-0063", "SAM-0061", "SAM-0062")),
|
||||
"дрель ударная prowerk pw-750 в наличии?": ("matched", ("INS-0008",)),
|
||||
"кабель шввп 2х0.5 сколько за метр": ("matched", ("KAB-0017",)),
|
||||
@@ -88,6 +95,11 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
|
||||
"где находится ваш магазин": ("not_found", ()),
|
||||
"статус заказа 4512 подскажите": ("not_found", ()),
|
||||
"спасибо, заказ получил, все отлично": ("not_found", ()),
|
||||
}
|
||||
|
||||
# The larger exploratory corpus is useful as a regression snapshot, but it is
|
||||
# intentionally kept separate from the hand-reviewed acceptance set above.
|
||||
MORE_MESSAGE_BASELINE_RESULTS: dict[str, ExpectedResult] = {
|
||||
"добрый день, саморезы по дереву 3.5х25 пачка 200 штук": ("matched", ("SAM-0001",)),
|
||||
"саморез гкл 3,5 на 25 кг": ("matched", ("SAM-0063",)),
|
||||
"нужны черные саморезы 4.2x75 упаковка 1000": ("matched", ("SAM-0023",)),
|
||||
@@ -106,7 +118,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
|
||||
"ШВВП 2 X 0,75, цена за метр?": ("matched", ("KAB-0018",)),
|
||||
"кабель кг 3х2,5": ("not_found", ()),
|
||||
"кабель для чайника": ("ambiguous", ("KAB-0004", "KAB-0005", "KAB-0008")),
|
||||
"3х1.5": ("not_found", ()),
|
||||
"3х1.5": ("ambiguous", ("KAB-0022", "KAB-0002", "KAB-0010")),
|
||||
"ввгнг 20х20": ("not_found", ()),
|
||||
"труба профильная 40x20x2": ("not_found", ()),
|
||||
"профтруба 60 40 3": ("ambiguous", ("TRB-0009", "TRB-0010", "TRB-0015")),
|
||||
@@ -115,7 +127,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
|
||||
"нужна труба длиной до луны": ("ambiguous", ("TRB-0003", "TRB-0007", "TRB-0008")),
|
||||
"гипсокартон 12,5 влагостойкий": ("matched", ("GKL-0003",)),
|
||||
"гкл обычный 9.5 мм": ("matched", ("GKL-0001",)),
|
||||
"лист гипса зеленый": ("not_found", ()),
|
||||
"лист гипса зеленый": ("ambiguous", ("GKL-0001", "GKL-0002", "GKL-0004")),
|
||||
"гипсокортан 12.5": ("ambiguous", ("GKL-0002", "GKL-0004", "GKL-0003")),
|
||||
"фанера 18 мм": ("not_found", ()),
|
||||
"дрель prowerk pw-750": ("matched", ("INS-0008",)),
|
||||
@@ -140,7 +152,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
|
||||
"насадка на шуруповерт": ("ambiguous", ("INS-0010", "INS-0011", "INS-0024")),
|
||||
"диск отрезной 125х1,0 металл": ("matched", ("DSK-0003",)),
|
||||
"круг по металлу 230 на 2.5": ("not_found", ()),
|
||||
"пильный диск 190x48t": ("ambiguous", ("DSK-0034", "DSK-0032", "DSK-0033")),
|
||||
"пильный диск 190x48t": ("matched", ("DSK-0034",)),
|
||||
"алмазный диск 125 бетон": ("ambiguous", ("DSK-0025", "DSK-0026", "DSK-0027")),
|
||||
"диск на болгарку": ("ambiguous", ("DSK-0011", "DSK-0012", "DSK-0013")),
|
||||
"круг квадратный 12х34": ("not_found", ()),
|
||||
@@ -189,8 +201,30 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
|
||||
"хочу всё и сразу за сто рублей": ("not_found", ()),
|
||||
"статус заказа 4512 и бур 8х160": ("matched", ("BIT-0077",)),
|
||||
"диск": ("ambiguous", ("DSK-0011", "DSK-0012", "DSK-0013")),
|
||||
"24 зуба": ("ambiguous", ("DSK-0029", "DSK-0032", "DSK-0035")),
|
||||
"210 мм 24 зуба": ("matched", ("DSK-0035",)),
|
||||
"125 мм": ("ambiguous", ("INS-0013", "DSK-0025", "RAS-0054")),
|
||||
"12 в": ("ambiguous", ("INS-0003", "INS-0010", "INS-0024")),
|
||||
"900 вт": ("ambiguous", ("INS-0009", "INS-0013")),
|
||||
"M10": ("ambiguous", ("KRP-0030", "KRP-0020", "KRP-0009")),
|
||||
"P120": ("ambiguous", ("DSK-0024", "RAS-0050", "RAS-0057")),
|
||||
"SDS-plus": ("ambiguous", ("BIT-0073", "INS-0012")),
|
||||
"PH2": ("ambiguous", ("BIT-0004", "RIN-0010")),
|
||||
"20x20x2": ("matched", ("TRB-0003",)),
|
||||
"ToolKraft 125 мм": ("matched", ("INS-0027",)),
|
||||
"99999 мм": ("not_found", ()),
|
||||
"аквариум 125 мм": ("not_found", ()),
|
||||
"удар": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")),
|
||||
"ударная": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")),
|
||||
"удар мощный": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")),
|
||||
"удар мощно 900 хромированый анус долбить": ("ambiguous", ("INS-0009", "INS-0016", "INS-0023")),
|
||||
"дрель удар мощно 900 хромированый анус долбить": ("ambiguous", ("INS-0009", "INS-0023", "INS-0002")),
|
||||
"анус": ("not_found", ()),
|
||||
"долбить": ("not_found", ()),
|
||||
}
|
||||
|
||||
EXPECTED_RESULTS = VALIDATED_MESSAGE_RESULTS | MORE_MESSAGE_BASELINE_RESULTS
|
||||
|
||||
|
||||
def test_message_files_are_present_and_have_expected_size() -> None:
|
||||
counts = {
|
||||
@@ -206,10 +240,15 @@ def test_message_files_are_present_and_have_expected_size() -> None:
|
||||
messages = [case.message for case in MESSAGE_CASES]
|
||||
assert len(messages) == len(set(messages)), "message texts must be unique"
|
||||
assert set(EXPECTED_RESULTS) == set(messages)
|
||||
assert set(VALIDATED_MESSAGE_RESULTS) == {
|
||||
case.message for case in MESSAGE_CASES if case.source == "messages.txt"
|
||||
}
|
||||
assert set(MORE_MESSAGE_BASELINE_RESULTS) == {
|
||||
case.message for case in MESSAGE_CASES if case.source == "more_messages.txt"
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", MESSAGE_CASES, ids=lambda case: case.id)
|
||||
def test_every_message_can_be_matched(
|
||||
def _assert_expected_result(
|
||||
matcher: CatalogMatcher, case: MessageCase
|
||||
) -> None:
|
||||
result = matcher.match(case.message)
|
||||
@@ -232,3 +271,19 @@ def test_every_message_can_be_matched(
|
||||
expected_status, expected_skus = EXPECTED_RESULTS[case.message]
|
||||
assert result.status == expected_status
|
||||
assert tuple(candidate.sku for candidate in result.candidates) == expected_skus
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", VALIDATED_MESSAGE_CASES, ids=lambda case: case.id
|
||||
)
|
||||
def test_validated_messages_txt_result(
|
||||
matcher: CatalogMatcher, case: MessageCase
|
||||
) -> None:
|
||||
_assert_expected_result(matcher, case)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", MORE_MESSAGE_CASES, ids=lambda case: case.id)
|
||||
def test_more_messages_regression(
|
||||
matcher: CatalogMatcher, case: MessageCase
|
||||
) -> None:
|
||||
_assert_expected_result(matcher, case)
|
||||
|
||||
Reference in New Issue
Block a user