Improve sparse and structured catalog matching

tests/test_catalog_self_retrieval.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+from decimal import Decimal
+from pathlib import Path
+from typing import Callable
+
+import pytest
+
+from app.catalog import CatalogItem, RawCatalogItem, load_raw_catalog
+from app.matcher import CatalogMatcher
+
+
+CATALOG_ITEMS = load_raw_catalog(
+    Path(__file__).resolve().parents[1] / "catalog_excel.csv"
+)
+
+
+@pytest.mark.parametrize("item", CATALOG_ITEMS, ids=lambda item: item.sku)
+def test_full_catalog_name_retrieves_its_own_sku(
+    matcher: CatalogMatcher, item: RawCatalogItem
+) -> None:
+    result = matcher.match(item.name)
+
+    assert item.sku in {candidate.sku for candidate in result.candidates}
+
+
+def test_every_canonical_product_type_returns_only_its_own_items(
+    matcher: CatalogMatcher,
+) -> None:
+    items_by_sku = {item.sku: item for item in matcher.items}
+    product_types = {item.product_type for item in matcher.items}
+
+    for product_type in product_types:
+        result = matcher.match(product_type)
+
+        assert result.candidates, product_type
+        assert all(
+            items_by_sku[candidate.sku].product_type == product_type
+            for candidate in result.candidates
+        ), product_type
+
+
+def test_every_catalog_model_code_returns_only_matching_items(
+    matcher: CatalogMatcher,
+) -> None:
+    items_by_sku = {item.sku: item for item in matcher.items}
+    model_codes = {
+        model
+        for item in matcher.items
+        for model in item.attributes.model_codes
+    }
+
+    for model_code in model_codes:
+        result = matcher.match(model_code)
+
+        assert result.candidates, model_code
+        assert all(
+            model_code in items_by_sku[candidate.sku].attributes.model_codes
+            for candidate in result.candidates
+        ), model_code
+
+
+def test_every_catalog_brand_returns_only_its_own_items(
+    matcher: CatalogMatcher,
+) -> None:
+    items_by_sku = {item.sku: item for item in matcher.items}
+    brands = {
+        item.attributes.brand
+        for item in matcher.items
+        if item.attributes.brand is not None
+    }
+
+    for brand in brands:
+        result = matcher.match(brand)
+
+        assert result.candidates, brand
+        assert all(
+            items_by_sku[candidate.sku].attributes.brand == brand
+            for candidate in result.candidates
+        ), brand
+
+
+def _number_text(value: Decimal) -> str:
+    return format(value, "f")
+
+
+def _assert_query_returns_only(
+    matcher: CatalogMatcher,
+    query: str,
+    predicate: Callable[[CatalogItem], bool],
+) -> None:
+    items_by_sku = {item.sku: item for item in matcher.items}
+    result = matcher.match(query)
+
+    assert result.candidates, query
+    assert all(
+        predicate(items_by_sku[candidate.sku]) for candidate in result.candidates
+    ), query
+
+
+def test_every_catalog_millimetre_value_is_searchable_without_product_name(
+    matcher: CatalogMatcher,
+) -> None:
+    values = {
+        value
+        for item in matcher.items
+        for value in (
+            *item.attributes.mm_values,
+            *((item.attributes.dimensions[0],) if item.attributes.dimensions else ()),
+            *((item.attributes.thread_length_mm,) if item.attributes.thread_length_mm else ()),
+        )
+    }
+
+    for value in values:
+        _assert_query_returns_only(
+            matcher,
+            f"{_number_text(value)} мм",
+            lambda item, expected=value: (
+                expected in item.attributes.mm_values
+                or bool(
+                    item.attributes.dimensions
+                    and item.attributes.dimensions[0] == expected
+                )
+                or item.attributes.thread_length_mm == expected
+            ),
+        )
+
+
+def test_every_catalog_dimension_is_searchable_without_product_name(
+    matcher: CatalogMatcher,
+) -> None:
+    dimensions = {
+        item.attributes.dimensions
+        for item in matcher.items
+        if item.attributes.dimensions
+    }
+
+    for dimension in dimensions:
+        query = "x".join(_number_text(value) for value in dimension)
+        _assert_query_returns_only(
+            matcher,
+            query,
+            lambda item, expected=dimension: (
+                len(item.attributes.dimensions) >= len(expected)
+                and item.attributes.dimensions[: len(expected)] == expected
+            ),
+        )
+
+
+@pytest.mark.parametrize(
+    ("field", "suffix"),
+    [
+        ("voltage_v", " В"),
+        ("power_w", " Вт"),
+        ("thread_size", ""),
+        ("grit", ""),
+        ("profile", ""),
+        ("sds_type", ""),
+        ("teeth", " зубьев"),
+    ],
+)
+def test_every_scalar_catalog_attribute_is_searchable_without_product_name(
+    matcher: CatalogMatcher,
+    field: str,
+    suffix: str,
+) -> None:
+    values = {
+        getattr(item.attributes, field)
+        for item in matcher.items
+        if getattr(item.attributes, field) is not None
+    }
+
+    for value in values:
+        if field == "thread_size":
+            query = f"M{_number_text(value)}"
+        elif field == "grit":
+            query = f"P{value}"
+        else:
+            value_text = _number_text(value) if isinstance(value, Decimal) else str(value)
+            query = f"{value_text}{suffix}"
+        _assert_query_returns_only(
+            matcher,
+            query,
+            lambda item, expected=value: getattr(item.attributes, field) == expected,
+        )

tests/test_matcher.py

@@ -184,6 +184,55 @@ def test_product_type_survives_typos(matcher: CatalogMatcher, query: str) -> Non
     assert all(candidate.sku.startswith("INS-") for candidate in result.candidates)
 
 
+@pytest.mark.parametrize(
+    ("query", "skus"),
+    [
+        ("удар", ("INS-0008", "INS-0009", "INS-0022")),
+        ("ударная", ("INS-0008", "INS-0009", "INS-0022")),
+        ("удар мощный", ("INS-0008", "INS-0009", "INS-0022")),
+        (
+            "удар мощно 900 хромированый анус долбить",
+            ("INS-0009", "INS-0016", "INS-0023"),
+        ),
+        (
+            "дрель удар мощно 900 хромированый анус долбить",
+            ("INS-0009", "INS-0023", "INS-0002"),
+        ),
+    ],
+)
+def test_trigram_fallback_finds_product_type_from_sparse_or_noisy_text(
+    matcher: CatalogMatcher, query: str, skus: tuple[str, ...]
+) -> None:
+    result = matcher.match(query)
+
+    assert result.status == "ambiguous"
+    assert tuple(candidate.sku for candidate in result.candidates) == skus
+    items_by_sku = {item.sku: item for item in matcher.items}
+    assert {
+        items_by_sku[candidate.sku].product_type for candidate in result.candidates
+    } == {"дрель"}
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "анус",
+        "долбить",
+        "какой адрес",
+        "аквариум 125 мм",
+        "<script>alert(1)</script>",
+        "qwerty asdf zxcv",
+    ],
+)
+def test_trigram_fallback_rejects_weak_or_accidental_similarity(
+    matcher: CatalogMatcher, query: str
+) -> None:
+    result = matcher.match(query)
+
+    assert result.status == "not_found"
+    assert result.candidates == []
+
+
 def test_bare_size_is_parsed_without_unit(matcher: CatalogMatcher) -> None:
     assert matcher.match("бита ph2 50").candidates[0].sku == "BIT-0005"
     assert matcher.match("диск пильный 190 48 зубьев").candidates[0].sku == "DSK-0034"
@@ -195,3 +244,110 @@ def test_unknown_size_does_not_fall_back_to_nearest_product(
     result = matcher.match("ушм 150")
     assert result.status == "not_found"
     assert result.candidates == []
+
+
+@pytest.mark.parametrize(
+    ("query", "sku"),
+    [
+        ("165 мм 24 зуба", "DSK-0029"),
+        ("165 мм 36 зубов", "DSK-0030"),
+        ("165 мм 48 зубьев", "DSK-0031"),
+        ("190 мм 24 зуба", "DSK-0032"),
+        ("190 мм 36 зубов", "DSK-0033"),
+        ("190 мм 48 зубьев", "DSK-0034"),
+        ("210 мм 24 зуба", "DSK-0035"),
+        ("210 мм 36 зубов", "DSK-0036"),
+        ("210 мм 48 зубьев", "DSK-0037"),
+    ],
+)
+def test_tooth_count_infers_saw_blade_without_product_name(
+    matcher: CatalogMatcher, query: str, sku: str
+) -> None:
+    result = matcher.match(query)
+
+    assert result.status == "matched"
+    assert [candidate.sku for candidate in result.candidates] == [sku]
+
+
+@pytest.mark.parametrize(
+    ("query", "skus"),
+    [
+        ("24 зуба", ("DSK-0029", "DSK-0032", "DSK-0035")),
+        ("36 зубов", ("DSK-0030", "DSK-0033", "DSK-0036")),
+        ("48 зубьев", ("DSK-0031", "DSK-0034", "DSK-0037")),
+    ],
+)
+def test_bare_tooth_count_returns_all_matching_saw_blade_sizes(
+    matcher: CatalogMatcher, query: str, skus: tuple[str, ...]
+) -> None:
+    result = matcher.match(query)
+
+    assert result.status == "ambiguous"
+    assert tuple(candidate.sku for candidate in result.candidates) == skus
+
+
+def test_unknown_tooth_count_does_not_return_nearest_saw_blade(
+    matcher: CatalogMatcher,
+) -> None:
+    result = matcher.match("210 мм 25 зубьев")
+
+    assert result.status == "not_found"
+    assert result.candidates == []
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "пильный диск 190x48t",
+        "190x48T",
+        "190x48z",
+        "190 48t",
+        "190/48T",
+    ],
+)
+def test_compact_saw_notation_parses_diameter_and_teeth(
+    matcher: CatalogMatcher, query: str
+) -> None:
+    result = matcher.match(query)
+
+    assert result.status == "matched"
+    assert [candidate.sku for candidate in result.candidates] == ["DSK-0034"]
+
+
+def test_bare_t_suffix_infers_saw_blades(matcher: CatalogMatcher) -> None:
+    result = matcher.match("24T")
+
+    assert result.status == "ambiguous"
+    assert [candidate.sku for candidate in result.candidates] == [
+        "DSK-0029",
+        "DSK-0032",
+        "DSK-0035",
+    ]
+
+
+@pytest.mark.parametrize(
+    ("query", "status", "skus"),
+    [
+        ("125 мм", "ambiguous", ("INS-0013", "DSK-0025", "RAS-0054")),
+        ("12 В", "ambiguous", ("INS-0003", "INS-0010", "INS-0024")),
+        ("900 Вт", "ambiguous", ("INS-0009", "INS-0013")),
+        ("M10", "ambiguous", ("KRP-0030", "KRP-0020", "KRP-0009")),
+        ("P120", "ambiguous", ("DSK-0024", "RAS-0050", "RAS-0057")),
+        ("SDS-plus", "ambiguous", ("BIT-0073", "INS-0012")),
+        ("PH2", "ambiguous", ("BIT-0004", "RIN-0010")),
+        ("20x20x2", "matched", ("TRB-0003",)),
+        ("ToolKraft 125 мм", "matched", ("INS-0027",)),
+        ("99999 мм", "not_found", ()),
+        ("аквариум 125 мм", "not_found", ()),
+    ],
+)
+def test_attribute_only_queries_use_exact_catalog_values(
+    matcher: CatalogMatcher,
+    query: str,
+    status: str,
+    skus: tuple[str, ...],
+) -> None:
+    result = matcher.match(query)
+
+    assert result.status == status
+    assert tuple(candidate.sku for candidate in result.candidates) == skus

tests/test_message_corpus.py

@@ -44,10 +44,17 @@ def _load_message_cases() -> tuple[MessageCase, ...]:
 
 
 MESSAGE_CASES = _load_message_cases()
+VALIDATED_MESSAGE_CASES = tuple(
+    case for case in MESSAGE_CASES if case.source == "messages.txt"
+)
+MORE_MESSAGE_CASES = tuple(
+    case for case in MESSAGE_CASES if case.source == "more_messages.txt"
+)
 
-# Baseline approved for the current corpus. The key is the message text so adding
-# or moving unrelated lines does not invalidate every expectation below.
-EXPECTED_RESULTS: dict[str, ExpectedResult] = {
+# Hand-reviewed acceptance results for every required messages.txt query.
+# Do not regenerate this table from CatalogMatcher output: it is the independent
+# oracle that should catch semantic regressions in the implementation.
+VALIDATED_MESSAGE_RESULTS: dict[str, ExpectedResult] = {
     "здравствуйте, есть саморезы гкл 3.5х25?": ("ambiguous", ("SAM-0063", "SAM-0061", "SAM-0062")),
     "дрель ударная prowerk pw-750 в наличии?": ("matched", ("INS-0008",)),
     "кабель шввп 2х0.5 сколько за метр": ("matched", ("KAB-0017",)),
@@ -88,6 +95,11 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
     "где находится ваш магазин": ("not_found", ()),
     "статус заказа 4512 подскажите": ("not_found", ()),
     "спасибо, заказ получил, все отлично": ("not_found", ()),
+}
+
+# The larger exploratory corpus is useful as a regression snapshot, but it is
+# intentionally kept separate from the hand-reviewed acceptance set above.
+MORE_MESSAGE_BASELINE_RESULTS: dict[str, ExpectedResult] = {
     "добрый день, саморезы по дереву 3.5х25 пачка 200 штук": ("matched", ("SAM-0001",)),
     "саморез гкл 3,5 на 25 кг": ("matched", ("SAM-0063",)),
     "нужны черные саморезы 4.2x75 упаковка 1000": ("matched", ("SAM-0023",)),
@@ -106,7 +118,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
     "ШВВП 2 X 0,75, цена за метр?": ("matched", ("KAB-0018",)),
     "кабель кг 3х2,5": ("not_found", ()),
     "кабель для чайника": ("ambiguous", ("KAB-0004", "KAB-0005", "KAB-0008")),
-    "3х1.5": ("not_found", ()),
+    "3х1.5": ("ambiguous", ("KAB-0022", "KAB-0002", "KAB-0010")),
     "ввгнг 20х20": ("not_found", ()),
     "труба профильная 40x20x2": ("not_found", ()),
     "профтруба 60 40 3": ("ambiguous", ("TRB-0009", "TRB-0010", "TRB-0015")),
@@ -115,7 +127,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
     "нужна труба длиной до луны": ("ambiguous", ("TRB-0003", "TRB-0007", "TRB-0008")),
     "гипсокартон 12,5 влагостойкий": ("matched", ("GKL-0003",)),
     "гкл обычный 9.5 мм": ("matched", ("GKL-0001",)),
-    "лист гипса зеленый": ("not_found", ()),
+    "лист гипса зеленый": ("ambiguous", ("GKL-0001", "GKL-0002", "GKL-0004")),
     "гипсокортан 12.5": ("ambiguous", ("GKL-0002", "GKL-0004", "GKL-0003")),
     "фанера 18 мм": ("not_found", ()),
     "дрель prowerk pw-750": ("matched", ("INS-0008",)),
@@ -140,7 +152,7 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
     "насадка на шуруповерт": ("ambiguous", ("INS-0010", "INS-0011", "INS-0024")),
     "диск отрезной 125х1,0 металл": ("matched", ("DSK-0003",)),
     "круг по металлу 230 на 2.5": ("not_found", ()),
-    "пильный диск 190x48t": ("ambiguous", ("DSK-0034", "DSK-0032", "DSK-0033")),
+    "пильный диск 190x48t": ("matched", ("DSK-0034",)),
     "алмазный диск 125 бетон": ("ambiguous", ("DSK-0025", "DSK-0026", "DSK-0027")),
     "диск на болгарку": ("ambiguous", ("DSK-0011", "DSK-0012", "DSK-0013")),
     "круг квадратный 12х34": ("not_found", ()),
@@ -189,8 +201,30 @@ EXPECTED_RESULTS: dict[str, ExpectedResult] = {
     "хочу всё и сразу за сто рублей": ("not_found", ()),
     "статус заказа 4512 и бур 8х160": ("matched", ("BIT-0077",)),
     "диск": ("ambiguous", ("DSK-0011", "DSK-0012", "DSK-0013")),
+    "24 зуба": ("ambiguous", ("DSK-0029", "DSK-0032", "DSK-0035")),
+    "210 мм 24 зуба": ("matched", ("DSK-0035",)),
+    "125 мм": ("ambiguous", ("INS-0013", "DSK-0025", "RAS-0054")),
+    "12 в": ("ambiguous", ("INS-0003", "INS-0010", "INS-0024")),
+    "900 вт": ("ambiguous", ("INS-0009", "INS-0013")),
+    "M10": ("ambiguous", ("KRP-0030", "KRP-0020", "KRP-0009")),
+    "P120": ("ambiguous", ("DSK-0024", "RAS-0050", "RAS-0057")),
+    "SDS-plus": ("ambiguous", ("BIT-0073", "INS-0012")),
+    "PH2": ("ambiguous", ("BIT-0004", "RIN-0010")),
+    "20x20x2": ("matched", ("TRB-0003",)),
+    "ToolKraft 125 мм": ("matched", ("INS-0027",)),
+    "99999 мм": ("not_found", ()),
+    "аквариум 125 мм": ("not_found", ()),
+    "удар": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")),
+    "ударная": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")),
+    "удар мощный": ("ambiguous", ("INS-0008", "INS-0009", "INS-0022")),
+    "удар мощно 900 хромированый анус долбить": ("ambiguous", ("INS-0009", "INS-0016", "INS-0023")),
+    "дрель удар мощно 900 хромированый анус долбить": ("ambiguous", ("INS-0009", "INS-0023", "INS-0002")),
+    "анус": ("not_found", ()),
+    "долбить": ("not_found", ()),
 }
 
+EXPECTED_RESULTS = VALIDATED_MESSAGE_RESULTS | MORE_MESSAGE_BASELINE_RESULTS
+
 
 def test_message_files_are_present_and_have_expected_size() -> None:
     counts = {
@@ -206,10 +240,15 @@ def test_message_files_are_present_and_have_expected_size() -> None:
     messages = [case.message for case in MESSAGE_CASES]
     assert len(messages) == len(set(messages)), "message texts must be unique"
     assert set(EXPECTED_RESULTS) == set(messages)
+    assert set(VALIDATED_MESSAGE_RESULTS) == {
+        case.message for case in MESSAGE_CASES if case.source == "messages.txt"
+    }
+    assert set(MORE_MESSAGE_BASELINE_RESULTS) == {
+        case.message for case in MESSAGE_CASES if case.source == "more_messages.txt"
+    }
 
 
-@pytest.mark.parametrize("case", MESSAGE_CASES, ids=lambda case: case.id)
-def test_every_message_can_be_matched(
+def _assert_expected_result(
     matcher: CatalogMatcher, case: MessageCase
 ) -> None:
     result = matcher.match(case.message)
@@ -232,3 +271,19 @@ def test_every_message_can_be_matched(
     expected_status, expected_skus = EXPECTED_RESULTS[case.message]
     assert result.status == expected_status
     assert tuple(candidate.sku for candidate in result.candidates) == expected_skus
+
+
+@pytest.mark.parametrize(
+    "case", VALIDATED_MESSAGE_CASES, ids=lambda case: case.id
+)
+def test_validated_messages_txt_result(
+    matcher: CatalogMatcher, case: MessageCase
+) -> None:
+    _assert_expected_result(matcher, case)
+
+
+@pytest.mark.parametrize("case", MORE_MESSAGE_CASES, ids=lambda case: case.id)
+def test_more_messages_regression(
+    matcher: CatalogMatcher, case: MessageCase
+) -> None:
+    _assert_expected_result(matcher, case)