From 42f8f072076ed11d23d0514c93e47b3c61f0746d Mon Sep 17 00:00:00 2001 From: Fiden Date: Fri, 7 Aug 2026 18:31:59 +0300 Subject: [PATCH] Improve sparse and structured catalog matching --- README.md | 19 ++- app/attributes.py | 17 ++- app/matcher.py | 193 ++++++++++++++++++++++++++- app/search_index.py | 96 ++++++++++++- more_messages.txt | 22 ++- tests/test_catalog_self_retrieval.py | 185 +++++++++++++++++++++++++ tests/test_matcher.py | 156 ++++++++++++++++++++++ tests/test_message_corpus.py | 71 ++++++++-- 8 files changed, 739 insertions(+), 20 deletions(-) create mode 100644 tests/test_catalog_self_retrieval.py diff --git a/README.md b/README.md index f1b8d3c..f7afab7 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,8 @@ Unicode- и текстовая нормализация ↓ восстановление опечаток ↓ +вывод типа по триграммам, если алиаса нет + ↓ лексическое ранжирование кандидатов ↓ извлечение типизированных характеристик @@ -194,7 +196,13 @@ alias_to_canonical[normalize_text(alias)] = canonical отличаются одной соседней перестановкой. А повреждённое `шураыввавёрт` всё ещё может восстановиться до категории `шуруповерт`, но низкая уверенность в категории не превращает один из нескольких шуруповёртов в ложный `matched`. -### Лексическое ранжирование +### Лексический fallback и ранжирование + +Если алиас товарного типа не найден, триграммы собирают короткий список +ближайших строк каталога. Тип выводится только когда три лучшие строки относятся к +одному типу и имеют общее слово, которое точно, по префиксу или с малой опечаткой совпало со словом +запроса. Поэтому `удар` находит `ударная дрель`, а `аквариум 125 мм` не выбирает УШМ только +из-за числа `125`. После определения товарного типа поиск ограничивается соответствующей частью каталога. Внутри неё кандидаты получают совокупный лексический score: @@ -203,7 +211,8 @@ alias_to_canonical[normalize_text(alias)] = canonical - выравнивание слов с Damerau–Levenshtein; - покрытие точных токенов запроса. -BM25 и n-граммы отвечают только за порядок кандидатов. Они не имеют права отменять конфликт размеров или модели. +BM25 и n-граммы отвечают за выбор и порядок кандидатов, но не имеют права отменять +конфликт размеров или модели. ### Структурные характеристики @@ -441,6 +450,7 @@ python -m pytest -q - точные совпадения; - алиасы и разговорные названия; - Damerau–Levenshtein и тяжёлые опечатки; +- вывод товарного типа по одному слову и шумным фразам; - разные записи размеров; - отсутствующие размеры, профили и резьбы; - неоднозначные фасовки; @@ -451,7 +461,10 @@ python -m pytest -q - максимум три кандидата; - диапазон confidence; - строгий формат request/response; -- единственный endpoint `/match`. +- единственный endpoint `/match`; +- self-retrieval всех позиций каталога по полному названию; +- поиск по каждому каноническому типу, бренду и модельному коду; +- exact-only поиск без названия товара по размерам и структурным атрибутам. `tests/test_message_corpus.py` автоматически читает все непустые строки из `messages.txt` и `more_messages.txt`, прогоняет их через matcher и строго diff --git a/app/attributes.py b/app/attributes.py index 75a5449..1dbe5d7 100644 --- a/app/attributes.py +++ b/app/attributes.py @@ -30,7 +30,11 @@ _VOLTAGE_RE = re.compile( rf"(? None: self.alias_index = ProductAliasIndex() @@ -78,15 +116,46 @@ class CatalogMatcher: alias="model_code", token_start=0, ) + if product_match is None and provisional_attributes.teeth is not None: + # A tooth count is type-specific evidence in this domain: only + # saw blades carry it. This safely supports requests such as + # "210 мм 24 зуба" without requiring the word "диск". + product_match = ProductTypeMatch( + canonical="диск", + score=1.0, + exact=True, + alias="tooth_count", + token_start=0, + ) if product_match is None: brand_result = self._match_brand_only(message, provisional_attributes) if brand_result is not None: return brand_result - # Product evidence always has priority. Service intent detection - # is only a diagnostic fallback; both unknown and service - # messages share the required not_found response. - detect_service_intent(message) - return MatchResult(message=message, status="not_found", candidates=[]) + structured_result = self._match_structured_only( + message, provisional_attributes + ) + if structured_result is not None: + return structured_result + # Trigrams used to rank only inside a type selected by aliases. + # Let them infer that missing type as well, but only when the + # leading catalog rows agree and share strong word-level evidence. + # This handles "удар" -> "ударная дрель" while + # keeping numeric coincidences and arbitrary text out. + if detect_service_intent(message) is None: + lexical_inference = self.lexical_index.infer_product_type(message) + if lexical_inference is not None: + product_match = ProductTypeMatch( + canonical=lexical_inference.product_type, + score=lexical_inference.score, + exact=lexical_inference.query_word + == lexical_inference.catalog_word, + alias=lexical_inference.catalog_word, + token_start=0, + ) + if product_match is None: + return MatchResult( + message=message, status="not_found", candidates=[] + ) product_type = product_match.canonical query_attributes = parse_attributes( @@ -154,6 +223,7 @@ class CatalogMatcher: query_attributes.brand is None or not query_attributes.brand_required or query_attributes.model_codes + or query_attributes.specificity != 1 ): return None @@ -207,6 +277,119 @@ class CatalogMatcher: ] return MatchResult(message=message, status=status, candidates=candidates) + def _match_structured_only( + self, message: str, query_attributes: ParsedAttributes + ) -> MatchResult | None: + if not self._is_structured_only_query(message, query_attributes): + return None + + compatible: dict[int, Compatibility] = {} + for index, item in enumerate(self.items): + comparison = compare_attributes( + item.product_type, + query_attributes, + item.attributes, + candidate_unit=item.unit, + ) + if comparison.compatible and comparison.matched_fields > 0: + compatible[index] = comparison + + if not compatible: + return None + + lexical_scores = self.lexical_index.rank( + message, "", tuple(compatible.keys()) + ) + ranked = self._apply_preferences( + lexical_scores, + compatible, + cheap=query_attributes.cheap_preference, + ) + if not ranked: + return None + + status = "matched" if len(ranked) == 1 else "ambiguous" + selected = self._select_diverse_product_types(ranked, limit=3) + structured_signal = ProductTypeMatch( + canonical="", + score=0.90, + exact=True, + alias="structured_attributes", + token_start=0, + ) + candidates = [ + Candidate( + sku=candidate.item.sku, + confidence=self._confidence( + candidate, + product_match=structured_signal, + candidate_count=len(ranked), + specificity=query_attributes.specificity, + status=status, + rank=rank, + ), + ) + for rank, candidate in enumerate(selected) + ] + return MatchResult(message=message, status=status, candidates=candidates) + + def _is_structured_only_query( + self, message: str, query_attributes: ParsedAttributes + ) -> bool: + has_strong_evidence = any( + ( + bool(query_attributes.dimensions), + bool(query_attributes.mm_values), + bool(query_attributes.meter_values), + query_attributes.thread_size is not None, + query_attributes.profile is not None, + query_attributes.grit is not None, + query_attributes.voltage_v is not None, + query_attributes.power_w is not None, + query_attributes.teeth is not None, + query_attributes.brand is not None + and query_attributes.brand_required, + bool(query_attributes.model_codes), + query_attributes.sds_type is not None, + ) + ) + if not has_strong_evidence: + return False + + allowed_words = set(ZERO_WEIGHT_TOKENS) | set(_STRUCTURED_QUERY_WORDS) + brand_match = self.brand_index.detect(message) + if brand_match is not None: + allowed_words.update(tokenize(brand_match.alias)) + + for token in tokenize(normalize_text(message), already_normalized=True): + if any(character.isdigit() for character in token): + continue + if token in allowed_words: + continue + if token.startswith(("бюджет", "дешев", "недорог")): + continue + return False + return True + + @staticmethod + def _select_diverse_product_types( + ranked: list[_RankedCandidate], *, limit: int + ) -> list[_RankedCandidate]: + if len({candidate.item.product_type for candidate in ranked}) == 1: + return ranked[:limit] + + selected: list[_RankedCandidate] = [] + product_types: set[str] = set() + + for candidate in ranked: + if candidate.item.product_type in product_types: + continue + selected.append(candidate) + product_types.add(candidate.item.product_type) + if len(selected) == limit: + break + return selected + def _model_candidates(self, model_codes: frozenset[str]) -> frozenset[int]: if not model_codes: return frozenset() diff --git a/app/search_index.py b/app/search_index.py index 3a246ef..3406219 100644 --- a/app/search_index.py +++ b/app/search_index.py @@ -6,7 +6,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING from app.normalization import meaningful_tokens, normalize_text, tokenize, word_tokens -from app.typo import character_trigrams, trigram_dice, word_similarity +from app.typo import ( + character_trigrams, + damerau_levenshtein_distance, + trigram_dice, + word_similarity, +) if TYPE_CHECKING: from app.catalog import CatalogItem @@ -22,6 +27,14 @@ class LexicalScore: exact_coverage: float +@dataclass(frozen=True, slots=True) +class ProductTypeInference: + product_type: str + score: float + query_word: str + catalog_word: str + + class LexicalIndex: """Small in-memory lexical index: BM25 + char trigrams + typo alignment.""" @@ -69,6 +82,87 @@ class LexicalIndex: counts.update(self._trigram_postings.get(trigram, ())) return tuple(index for index, _ in counts.most_common(limit)) + def infer_product_type(self, query_text: str) -> ProductTypeInference | None: + """Infer a missing product type from stable lexical evidence. + + Trigrams provide the shortlist, but are deliberately not sufficient on + their own: sizes and random strings can share a few character triples + with unrelated catalog rows. The leading rows must agree on one type + and share a catalog word that is an exact match, a morphological prefix, + or a tightly bounded typo of a query word. + """ + + candidate_indices = self.trigram_candidates(query_text, limit=60) + if len(candidate_indices) < 3: + return None + + leaders = self.rank(query_text, "", candidate_indices)[:3] + if len(leaders) < 3: + return None + + product_types = { + self._items[leader.item_index].product_type for leader in leaders + } + if len(product_types) != 1: + return None + + common_words = set(self._document_words[leaders[0].item_index]) + for leader in leaders[1:]: + common_words.intersection_update(self._document_words[leader.item_index]) + if not common_words: + return None + + normalized = normalize_text(query_text) + query_words = word_tokens( + meaningful_tokens(tokenize(normalized, already_normalized=True)) + ) + best: tuple[float, str, str] | None = None + for query_word in query_words: + if len(query_word) < 4: + continue + for catalog_word in common_words: + score = self._type_inference_word_score(query_word, catalog_word) + if score <= 0: + continue + evidence = (score, query_word, catalog_word) + if best is None or evidence > best: + best = evidence + + if best is None: + return None + + score, query_word, catalog_word = best + return ProductTypeInference( + product_type=next(iter(product_types)), + score=score, + query_word=query_word, + catalog_word=catalog_word, + ) + + @staticmethod + def _type_inference_word_score(query_word: str, catalog_word: str) -> float: + if query_word == catalog_word: + return 1.0 + + shorter_length = min(len(query_word), len(catalog_word)) + longer_length = max(len(query_word), len(catalog_word)) + if shorter_length >= 4 and ( + query_word.startswith(catalog_word) + or catalog_word.startswith(query_word) + ): + length_ratio = shorter_length / longer_length + if length_ratio >= 0.55: + return 0.72 + length_ratio * 0.18 + + distance = damerau_levenshtein_distance(query_word, catalog_word) + if shorter_length >= 5 and distance <= 1: + return 0.90 + if longer_length >= 8 and distance <= 2: + edit_similarity = 1.0 - distance / longer_length + if edit_similarity >= 0.75: + return 0.82 + return 0.0 + def rank( self, query_text: str, diff --git a/more_messages.txt b/more_messages.txt index 8e0045b..e835dff 100644 --- a/more_messages.txt +++ b/more_messages.txt @@ -98,4 +98,24 @@ SELECT * FROM catalog; мне два метра того красного хочу всё и сразу за сто рублей статус заказа 4512 и бур 8х160 -диск \ No newline at end of file +диск +24 зуба +210 мм 24 зуба +125 мм +12 в +900 вт +M10 +P120 +SDS-plus +PH2 +20x20x2 +ToolKraft 125 мм +99999 мм +аквариум 125 мм +удар +ударная +удар мощный +удар мощно 900 хромированый анус долбить +дрель удар мощно 900 хромированый анус долбить +анус +долбить diff --git a/tests/test_catalog_self_retrieval.py b/tests/test_catalog_self_retrieval.py new file mode 100644 index 0000000..7ea9103 --- /dev/null +++ b/tests/test_catalog_self_retrieval.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from decimal import Decimal +from pathlib import Path +from typing import Callable + +import pytest + +from app.catalog import CatalogItem, RawCatalogItem, load_raw_catalog +from app.matcher import CatalogMatcher + + +CATALOG_ITEMS = load_raw_catalog( + Path(__file__).resolve().parents[1] / "catalog_excel.csv" +) + + +@pytest.mark.parametrize("item", CATALOG_ITEMS, ids=lambda item: item.sku) +def test_full_catalog_name_retrieves_its_own_sku( + matcher: CatalogMatcher, item: RawCatalogItem +) -> None: + result = matcher.match(item.name) + + assert item.sku in {candidate.sku for candidate in result.candidates} + + +def test_every_canonical_product_type_returns_only_its_own_items( + matcher: CatalogMatcher, +) -> None: + items_by_sku = {item.sku: item for item in matcher.items} + product_types = {item.product_type for item in matcher.items} + + for product_type in product_types: + result = matcher.match(product_type) + + assert result.candidates, product_type + assert all( + items_by_sku[candidate.sku].product_type == product_type + for candidate in result.candidates + ), product_type + + +def test_every_catalog_model_code_returns_only_matching_items( + matcher: CatalogMatcher, +) -> None: + items_by_sku = {item.sku: item for item in matcher.items} + model_codes = { + model + for item in matcher.items + for model in item.attributes.model_codes + } + + for model_code in model_codes: + result = matcher.match(model_code) + + assert result.candidates, model_code + assert all( + model_code in items_by_sku[candidate.sku].attributes.model_codes + for candidate in result.candidates + ), model_code + + +def test_every_catalog_brand_returns_only_its_own_items( + matcher: CatalogMatcher, +) -> None: + items_by_sku = {item.sku: item for item in matcher.items} + brands = { + item.attributes.brand + for item in matcher.items + if item.attributes.brand is not None + } + + for brand in brands: + result = matcher.match(brand) + + assert result.candidates, brand + assert all( + items_by_sku[candidate.sku].attributes.brand == brand + for candidate in result.candidates + ), brand + + +def _number_text(value: Decimal) -> str: + return format(value, "f") + + +def _assert_query_returns_only( + matcher: CatalogMatcher, + query: str, + predicate: Callable[[CatalogItem], bool], +) -> None: + items_by_sku = {item.sku: item for item in matcher.items} + result = matcher.match(query) + + assert result.candidates, query + assert all( + predicate(items_by_sku[candidate.sku]) for candidate in result.candidates + ), query + + +def test_every_catalog_millimetre_value_is_searchable_without_product_name( + matcher: CatalogMatcher, +) -> None: + values = { + value + for item in matcher.items + for value in ( + *item.attributes.mm_values, + *((item.attributes.dimensions[0],) if item.attributes.dimensions else ()), + *((item.attributes.thread_length_mm,) if item.attributes.thread_length_mm else ()), + ) + } + + for value in values: + _assert_query_returns_only( + matcher, + f"{_number_text(value)} мм", + lambda item, expected=value: ( + expected in item.attributes.mm_values + or bool( + item.attributes.dimensions + and item.attributes.dimensions[0] == expected + ) + or item.attributes.thread_length_mm == expected + ), + ) + + +def test_every_catalog_dimension_is_searchable_without_product_name( + matcher: CatalogMatcher, +) -> None: + dimensions = { + item.attributes.dimensions + for item in matcher.items + if item.attributes.dimensions + } + + for dimension in dimensions: + query = "x".join(_number_text(value) for value in dimension) + _assert_query_returns_only( + matcher, + query, + lambda item, expected=dimension: ( + len(item.attributes.dimensions) >= len(expected) + and item.attributes.dimensions[: len(expected)] == expected + ), + ) + + +@pytest.mark.parametrize( + ("field", "suffix"), + [ + ("voltage_v", " В"), + ("power_w", " Вт"), + ("thread_size", ""), + ("grit", ""), + ("profile", ""), + ("sds_type", ""), + ("teeth", " зубьев"), + ], +) +def test_every_scalar_catalog_attribute_is_searchable_without_product_name( + matcher: CatalogMatcher, + field: str, + suffix: str, +) -> None: + values = { + getattr(item.attributes, field) + for item in matcher.items + if getattr(item.attributes, field) is not None + } + + for value in values: + if field == "thread_size": + query = f"M{_number_text(value)}" + elif field == "grit": + query = f"P{value}" + else: + value_text = _number_text(value) if isinstance(value, Decimal) else str(value) + query = f"{value_text}{suffix}" + _assert_query_returns_only( + matcher, + query, + lambda item, expected=value: getattr(item.attributes, field) == expected, + ) diff --git a/tests/test_matcher.py b/tests/test_matcher.py index cf1fa37..cfc86eb 100644 --- a/tests/test_matcher.py +++ b/tests/test_matcher.py @@ -184,6 +184,55 @@ def test_product_type_survives_typos(matcher: CatalogMatcher, query: str) -> Non assert all(candidate.sku.startswith("INS-") for candidate in result.candidates) +@pytest.mark.parametrize( + ("query", "skus"), + [ + ("удар", ("INS-0008", "INS-0009", "INS-0022")), + ("ударная", ("INS-0008", "INS-0009", "INS-0022")), + ("удар мощный", ("INS-0008", "INS-0009", "INS-0022")), + ( + "удар мощно 900 хромированый анус долбить", + ("INS-0009", "INS-0016", "INS-0023"), + ), + ( + "дрель удар мощно 900 хромированый анус долбить", + ("INS-0009", "INS-0023", "INS-0002"), + ), + ], +) +def test_trigram_fallback_finds_product_type_from_sparse_or_noisy_text( + matcher: CatalogMatcher, query: str, skus: tuple[str, ...] +) -> None: + result = matcher.match(query) + + assert result.status == "ambiguous" + assert tuple(candidate.sku for candidate in result.candidates) == skus + items_by_sku = {item.sku: item for item in matcher.items} + assert { + items_by_sku[candidate.sku].product_type for candidate in result.candidates + } == {"дрель"} + + +@pytest.mark.parametrize( + "query", + [ + "анус", + "долбить", + "какой адрес", + "аквариум 125 мм", + "", + "qwerty asdf zxcv", + ], +) +def test_trigram_fallback_rejects_weak_or_accidental_similarity( + matcher: CatalogMatcher, query: str +) -> None: + result = matcher.match(query) + + assert result.status == "not_found" + assert result.candidates == [] + + def test_bare_size_is_parsed_without_unit(matcher: CatalogMatcher) -> None: assert matcher.match("бита ph2 50").candidates[0].sku == "BIT-0005" assert matcher.match("диск пильный 190 48 зубьев").candidates[0].sku == "DSK-0034" @@ -195,3 +244,110 @@ def test_unknown_size_does_not_fall_back_to_nearest_product( result = matcher.match("ушм 150") assert result.status == "not_found" assert result.candidates == [] + + +@pytest.mark.parametrize( + ("query", "sku"), + [ + ("165 мм 24 зуба", "DSK-0029"), + ("165 мм 36 зубов", "DSK-0030"), + ("165 мм 48 зубьев", "DSK-0031"), + ("190 мм 24 зуба", "DSK-0032"), + ("190 мм 36 зубов", "DSK-0033"), + ("190 мм 48 зубьев", "DSK-0034"), + ("210 мм 24 зуба", "DSK-0035"), + ("210 мм 36 зубов", "DSK-0036"), + ("210 мм 48 зубьев", "DSK-0037"), + ], +) +def test_tooth_count_infers_saw_blade_without_product_name( + matcher: CatalogMatcher, query: str, sku: str +) -> None: + result = matcher.match(query) + + assert result.status == "matched" + assert [candidate.sku for candidate in result.candidates] == [sku] + + +@pytest.mark.parametrize( + ("query", "skus"), + [ + ("24 зуба", ("DSK-0029", "DSK-0032", "DSK-0035")), + ("36 зубов", ("DSK-0030", "DSK-0033", "DSK-0036")), + ("48 зубьев", ("DSK-0031", "DSK-0034", "DSK-0037")), + ], +) +def test_bare_tooth_count_returns_all_matching_saw_blade_sizes( + matcher: CatalogMatcher, query: str, skus: tuple[str, ...] +) -> None: + result = matcher.match(query) + + assert result.status == "ambiguous" + assert tuple(candidate.sku for candidate in result.candidates) == skus + + +def test_unknown_tooth_count_does_not_return_nearest_saw_blade( + matcher: CatalogMatcher, +) -> None: + result = matcher.match("210 мм 25 зубьев") + + assert result.status == "not_found" + assert result.candidates == [] + + +@pytest.mark.parametrize( + "query", + [ + "пильный диск 190x48t", + "190x48T", + "190x48z", + "190 48t", + "190/48T", + ], +) +def test_compact_saw_notation_parses_diameter_and_teeth( + matcher: CatalogMatcher, query: str +) -> None: + result = matcher.match(query) + + assert result.status == "matched" + assert [candidate.sku for candidate in result.candidates] == ["DSK-0034"] + + +def test_bare_t_suffix_infers_saw_blades(matcher: CatalogMatcher) -> None: + result = matcher.match("24T") + + assert result.status == "ambiguous" + assert [candidate.sku for candidate in result.candidates] == [ + "DSK-0029", + "DSK-0032", + "DSK-0035", + ] + + +@pytest.mark.parametrize( + ("query", "status", "skus"), + [ + ("125 мм", "ambiguous", ("INS-0013", "DSK-0025", "RAS-0054")), + ("12 В", "ambiguous", ("INS-0003", "INS-0010", "INS-0024")), + ("900 Вт", "ambiguous", ("INS-0009", "INS-0013")), + ("M10", "ambiguous", ("KRP-0030", "KRP-0020", "KRP-0009")), + ("P120", "ambiguous", ("DSK-0024", "RAS-0050", "RAS-0057")), + ("SDS-plus", "ambiguous", ("BIT-0073", "INS-0012")), + ("PH2", "ambiguous", ("BIT-0004", "RIN-0010")), + ("20x20x2", "matched", ("TRB-0003",)), + ("ToolKraft 125 мм", "matched", ("INS-0027",)), + ("99999 мм", "not_found", ()), + ("аквариум 125 мм", "not_found", ()), + ], +) +def test_attribute_only_queries_use_exact_catalog_values( + matcher: CatalogMatcher, + query: str, + status: str, + skus: tuple[str, ...], +) -> None: + result = matcher.match(query) + + assert result.status == status + assert tuple(candidate.sku for candidate in result.candidates) == skus diff --git a/tests/test_message_corpus.py b/tests/test_message_corpus.py index 48e0ef7..c350927 100644 --- a/tests/test_message_corpus.py +++ b/tests/test_message_corpus.py @@ -44,10 +44,17 @@ def _load_message_cases() -> tuple[MessageCase, ...]: MESSAGE_CASES = _load_message_cases() +VALIDATED_MESSAGE_CASES = tuple( + case for case in MESSAGE_CASES if case.source == "messages.txt" +) +MORE_MESSAGE_CASES = tuple( + case for case in MESSAGE_CASES if case.source == "more_messages.txt" +) -# Baseline approved for the current corpus. The key is the message text so adding -# or moving unrelated lines does not invalidate every expectation below. -EXPECTED_RESULTS: dict[str, ExpectedResult] = { +# Hand-reviewed acceptance results for every required messages.txt query. +# Do not regenerate this table from CatalogMatcher output: it is the independent +# oracle that should catch semantic regressions in the implementation. +VALIDATED_MESSAGE_RESULTS: dict[str, ExpectedResult] = { "здравствуйте, есть саморезы гкл 3.5х25?": ("ambiguous", ("SAM-0063", "SAM-0061", "SAM-0062")), "дрель ударная prowerk pw-750 в наличии?": ("matched", ("INS-0008",)), "кабель шввп 2х0.5 сколько за метр": ("matched", ("KAB-0017",)), @@ -88,6 +95,11 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = { "где находится ваш магазин": ("not_found", ()), "статус заказа 4512 подскажите": ("not_found", ()), "спасибо, заказ получил, все отлично": ("not_found", ()), +} + +# The larger exploratory corpus is useful as a regression snapshot, but it is +# intentionally kept separate from the hand-reviewed acceptance set above. +MORE_MESSAGE_BASELINE_RESULTS: dict[str, ExpectedResult] = { "добрый день, саморезы по дереву 3.5х25 пачка 200 штук": ("matched", ("SAM-0001",)), "саморез гкл 3,5 на 25 кг": ("matched", ("SAM-0063",)), "нужны черные саморезы 4.2x75 упаковка 1000": ("matched", ("SAM-0023",)), @@ -106,7 +118,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = { "ШВВП 2 X 0,75, цена за метр?": ("matched", ("KAB-0018",)), "кабель кг 3х2,5": ("not_found", ()), "кабель для чайника": ("ambiguous", ("KAB-0004", "KAB-0005", "KAB-0008")), - "3х1.5": ("not_found", ()), + "3х1.5": ("ambiguous", ("KAB-0022", "KAB-0002", "KAB-0010")), "ввгнг 20х20": ("not_found", ()), "труба профильная 40x20x2": ("not_found", ()), "профтруба 60 40 3": ("ambiguous", ("TRB-0009", "TRB-0010", "TRB-0015")), @@ -115,7 +127,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = { "нужна труба длиной до луны": ("ambiguous", ("TRB-0003", "TRB-0007", "TRB-0008")), "гипсокартон 12,5 влагостойкий": ("matched", ("GKL-0003",)), "гкл обычный 9.5 мм": ("matched", ("GKL-0001",)), - "лист гипса зеленый": ("not_found", ()), + "лист гипса зеленый": ("ambiguous", ("GKL-0001", "GKL-0002", "GKL-0004")), "гипсокортан 12.5": ("ambiguous", ("GKL-0002", "GKL-0004", "GKL-0003")), "фанера 18 мм": ("not_found", ()), "дрель prowerk pw-750": ("matched", ("INS-0008",)), @@ -140,7 +152,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = { "насадка на шуруповерт": ("ambiguous", ("INS-0010", "INS-0011", "INS-0024")), "диск отрезной 125х1,0 металл": ("matched", ("DSK-0003",)), "круг по металлу 230 на 2.5": ("not_found", ()), - "пильный диск 190x48t": ("ambiguous", ("DSK-0034", "DSK-0032", "DSK-0033")), + "пильный диск 190x48t": ("matched", ("DSK-0034",)), "алмазный диск 125 бетон": ("ambiguous", ("DSK-0025", "DSK-0026", "DSK-0027")), "диск на болгарку": ("ambiguous", ("DSK-0011", "DSK-0012", "DSK-0013")), "круг квадратный 12х34": ("not_found", ()), @@ -189,8 +201,30 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = { "хочу всё и сразу за сто рублей": ("not_found", ()), "статус заказа 4512 и бур 8х160": ("matched", ("BIT-0077",)), "диск": ("ambiguous", ("DSK-0011", "DSK-0012", "DSK-0013")), + "24 зуба": ("ambiguous", ("DSK-0029", "DSK-0032", "DSK-0035")), + "210 мм 24 зуба": ("matched", ("DSK-0035",)), + "125 мм": ("ambiguous", ("INS-0013", "DSK-0025", "RAS-0054")), + "12 в": ("ambiguous", ("INS-0003", "INS-0010", "INS-0024")), + "900 вт": ("ambiguous", ("INS-0009", "INS-0013")), + "M10": ("ambiguous", ("KRP-0030", "KRP-0020", "KRP-0009")), + "P120": ("ambiguous", ("DSK-0024", "RAS-0050", "RAS-0057")), + "SDS-plus": ("ambiguous", ("BIT-0073", "INS-0012")), + "PH2": ("ambiguous", ("BIT-0004", "RIN-0010")), + "20x20x2": ("matched", ("TRB-0003",)), + "ToolKraft 125 мм": ("matched", ("INS-0027",)), + "99999 мм": ("not_found", ()), + "аквариум 125 мм": ("not_found", ()), + "удар": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")), + "ударная": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")), + "удар мощный": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")), + "удар мощно 900 хромированый анус долбить": ("ambiguous", ("INS-0009", "INS-0016", "INS-0023")), + "дрель удар мощно 900 хромированый анус долбить": ("ambiguous", ("INS-0009", "INS-0023", "INS-0002")), + "анус": ("not_found", ()), + "долбить": ("not_found", ()), } +EXPECTED_RESULTS = VALIDATED_MESSAGE_RESULTS | MORE_MESSAGE_BASELINE_RESULTS + def test_message_files_are_present_and_have_expected_size() -> None: counts = { @@ -206,10 +240,15 @@ def test_message_files_are_present_and_have_expected_size() -> None: messages = [case.message for case in MESSAGE_CASES] assert len(messages) == len(set(messages)), "message texts must be unique" assert set(EXPECTED_RESULTS) == set(messages) + assert set(VALIDATED_MESSAGE_RESULTS) == { + case.message for case in MESSAGE_CASES if case.source == "messages.txt" + } + assert set(MORE_MESSAGE_BASELINE_RESULTS) == { + case.message for case in MESSAGE_CASES if case.source == "more_messages.txt" + } -@pytest.mark.parametrize("case", MESSAGE_CASES, ids=lambda case: case.id) -def test_every_message_can_be_matched( +def _assert_expected_result( matcher: CatalogMatcher, case: MessageCase ) -> None: result = matcher.match(case.message) @@ -232,3 +271,19 @@ def test_every_message_can_be_matched( expected_status, expected_skus = EXPECTED_RESULTS[case.message] assert result.status == expected_status assert tuple(candidate.sku for candidate in result.candidates) == expected_skus + + +@pytest.mark.parametrize( + "case", VALIDATED_MESSAGE_CASES, ids=lambda case: case.id +) +def test_validated_messages_txt_result( + matcher: CatalogMatcher, case: MessageCase +) -> None: + _assert_expected_result(matcher, case) + + +@pytest.mark.parametrize("case", MORE_MESSAGE_CASES, ids=lambda case: case.id) +def test_more_messages_regression( + matcher: CatalogMatcher, case: MessageCase +) -> None: + _assert_expected_result(matcher, case)