feat: add text_match option to HybridSearchConfig for OR keyword matching

examples/pg_vectorstore_how_to.ipynb

@@ -784,15 +784,16 @@
     "### Building the config\n",
     "\n",
     "Here are the parameters to the hybrid search config:\n",
-    "* **tsv_column:** The column name for TSV column. Default: `<content_column>_tsv`\n",
+    "* **tsv_column:** The column name for TSV column. Default: `\"\"` (no TSV column; the TSV vector is computed from the content column at query time). When the config is passed to `init_vectorstore_table` with no `tsv_column`, the new table gets a TSV column named `<content_column>_tsv`.\n",
     "* **tsv_lang:** Value representing a supported language. Default: `pg_catalog.english`\n",
     "* **fts_query:** If provided, this would be used for secondary retrieval instead of user provided query.\n",
     "* **fusion_function:** Determines how the results are to be merged, default is equal weighted sum ranking.\n",
     "* **fusion_function_parameters:** Parameters for the fusion function\n",
     "* **primary_top_k:** Max results fetched for primary retrieval. Default: `4`\n",
     "* **secondary_top_k:** Max results fetched for secondary retrieval. Default: `4`\n",
     "* **index_name:** Name of the index built on the `tsv_column`\n",
-    "* **index_type:** GIN or GIST. Default: `GIN`"
+    "* **index_type:** GIN or GIST. Default: `GIN`\n",
+    "* **text_match:** How the keyword search matches query terms. `\"all\"` matches documents that contain every term of the query, `\"any\"` matches documents that contain at least one. Default: `\"all\"`"
    ]
   },
   {
@@ -842,6 +843,35 @@
     "* fetch_top_k: The number of documents to fetch after merging the results. Defaults to 4\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Matching any keyword\n",
+    "\n",
+    "By default (`text_match=\"all\"`) the keyword search only returns documents that contain every word of the query. Natural-language questions rarely share every word with the passage that answers them, so the keyword search can return no results and the hybrid search falls back to vector results only.\n",
+    "\n",
+    "Set `text_match=\"any\"` to return documents that contain at least one of the query words, ranked by how well they match. Stop words such as \"the\" or \"is\" are ignored."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "any_match_config = HybridSearchConfig(\n",
+    "    tsv_column=\"hybrid_description\",\n",
+    "    tsv_lang=\"pg_catalog.english\",\n",
+    "    text_match=\"any\",\n",
+    "    fusion_function=reciprocal_rank_fusion,\n",
+    "    fusion_function_parameters={\n",
+    "        \"rrf_k\": 60,\n",
+    "        \"fetch_top_k\": 10,\n",
+    "    },\n",
+    ")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

langchain_postgres/v2/async_vectorstore.py

@@ -723,7 +723,17 @@ async def __query_collection(
                 if hybrid_search_config.tsv_lang
                 else ""
             )
-            query_tsv = f"plainto_tsquery({lang} :fts_query)"
+            if hybrid_search_config.text_match == "any":
+                # OR together the lexemes of the query. Each lexeme is quoted and
+                # cast directly to tsquery so it is never parsed as tsquery syntax.
+                query_tsv = (
+                    r"CAST(array_to_string(ARRAY(SELECT '''' || replace(replace("
+                    r"lexeme, E'\\', E'\\\\'), '''', '''''') || '''' FROM unnest("
+                    f"tsvector_to_array(to_tsvector({lang} :fts_query))) AS lexeme"
+                    r"), ' | ') AS tsquery)"
+                )
+            else:
+                query_tsv = f"plainto_tsquery({lang} :fts_query)"
             param_dict["fts_query"] = fts_query
             if hybrid_search_config.tsv_column:
                 content_tsv = f'"{hybrid_search_config.tsv_column}"'

langchain_postgres/v2/hybrid_search_config.py

@@ -1,6 +1,6 @@
 from abc import ABC
 from dataclasses import dataclass, field
-from typing import Any, Callable, Optional, Sequence
+from typing import Any, Callable, Literal, Optional, Sequence
 
 from sqlalchemy import RowMapping
 
@@ -210,3 +210,12 @@ class HybridSearchConfig(ABC):
     secondary_top_k: int = 4
     index_name: str = "langchain_tsv_index"
     index_type: str = "GIN"
+    # "all": keyword search matches documents containing every query term.
+    # "any": keyword search matches documents containing at least one term.
+    text_match: Literal["all", "any"] = "all"
+
+    def __post_init__(self) -> None:
+        if self.text_match not in ("all", "any"):
+            raise ValueError(
+                f'text_match must be "all" or "any", got {self.text_match!r}.'
+            )

tests/unit_tests/v2/test_async_pg_vectorstore_search.py

@@ -1,6 +1,6 @@
 import os
 import uuid
-from typing import AsyncIterator
+from typing import Any, AsyncIterator
 
 import pytest
 import pytest_asyncio
@@ -26,6 +26,7 @@
 CUSTOM_TABLE = "custom" + str(uuid.uuid4()).replace("-", "_")
 HYBRID_SEARCH_TABLE1 = "test_table_hybrid1" + str(uuid.uuid4()).replace("-", "_")
 HYBRID_SEARCH_TABLE2 = "test_table_hybrid2" + str(uuid.uuid4()).replace("-", "_")
+TEXT_MATCH_TABLE = "test_table_text_match" + str(uuid.uuid4()).replace("-", "_")
 CUSTOM_FILTER_TABLE = "custom_filter" + str(uuid.uuid4()).replace("-", "_")
 CUSTOM_METADATA_JSON_TABLE = "custom_metadata_json" + str(uuid.uuid4()).replace(
     "-", "_"
@@ -63,6 +64,16 @@
     Document(page_content=content, metadata={"doc_id_key": key})
     for key, content in hybrid_docs_content.items()
 ]
+# Documents for keyword-only checks of HybridSearchConfig.text_match
+text_match_docs_content = {
+    "tm_doc_vendor": "We onboarded a third-party vendor last year.",
+    "tm_doc_orange": "The orange is the fruit of various citrus species.",
+    "tm_doc_cat": "A fluffy cat sat on a mat.",
+}
+text_match_docs = [
+    Document(page_content=content, metadata={"doc_id_key": key})
+    for key, content in text_match_docs_content.items()
+]
 
 
 def get_env_var(key: str, desc: str) -> str:
@@ -508,6 +519,133 @@ async def test_asimilarity_hybrid_search_rrk(self, vs: AsyncPGVectorStore) -> No
         )
         assert results == [Document(page_content="bar", id=ids[1])]
 
+    @pytest_asyncio.fixture(scope="class")
+    async def vs_text_match(
+        self, engine: PGEngine
+    ) -> AsyncIterator[AsyncPGVectorStore]:
+        hybrid_search_config = HybridSearchConfig(tsv_column="mycontent_tsv")
+        await engine._ainit_vectorstore_table(
+            TEXT_MATCH_TABLE,
+            VECTOR_SIZE,
+            id_column=Column("myid", "TEXT"),
+            content_column="mycontent",
+            embedding_column="myembedding",
+            metadata_columns=[Column("doc_id_key", "TEXT")],
+            store_metadata=False,
+            hybrid_search_config=hybrid_search_config,
+        )
+        vs = await AsyncPGVectorStore.create(
+            engine,
+            embedding_service=embeddings_service,
+            table_name=TEXT_MATCH_TABLE,
+            id_column="myid",
+            content_column="mycontent",
+            embedding_column="myembedding",
+            metadata_columns=["doc_id_key"],
+            hybrid_search_config=hybrid_search_config,
+        )
+        await vs.aadd_documents(text_match_docs)
+        yield vs
+        await engine.adrop_table(TEXT_MATCH_TABLE)
+
+    async def _keyword_matches(
+        self, vs: AsyncPGVectorStore, query: str, **config_kwargs: Any
+    ) -> list[str]:
+        """Return the doc_id_key of every keyword match, with vector search disabled."""
+        config = HybridSearchConfig(
+            fts_query=query, primary_top_k=0, secondary_top_k=10, **config_kwargs
+        )
+        results = await vs.asimilarity_search(query, k=10, hybrid_search_config=config)
+        return sorted(doc.metadata["doc_id_key"] for doc in results)
+
+    @pytest.mark.parametrize("tsv_column", ["mycontent_tsv", ""])
+    async def test_hybrid_search_text_match_any_matches_single_term(
+        self, vs_text_match: AsyncPGVectorStore, tsv_column: str
+    ) -> None:
+        """Test that "any" matches a document containing only one query term."""
+        query = "which citrus grove"  # only "citrus" appears in a document
+
+        assert (
+            await self._keyword_matches(
+                vs_text_match, query, tsv_column=tsv_column, text_match="all"
+            )
+            == []
+        )
+        assert await self._keyword_matches(
+            vs_text_match, query, tsv_column=tsv_column, text_match="any"
+        ) == ["tm_doc_orange"]
+
+    @pytest.mark.parametrize("text_match", ["all", "any"])
+    async def test_hybrid_search_text_match_stopwords_only(
+        self, vs_text_match: AsyncPGVectorStore, text_match: str
+    ) -> None:
+        """Test that a query made only of stopwords returns no keyword matches."""
+        assert (
+            await self._keyword_matches(
+                vs_text_match,
+                "how is the",
+                tsv_column="mycontent_tsv",
+                text_match=text_match,
+            )
+            == []
+        )
+
+    @pytest.mark.parametrize("text_match", ["all", "any"])
+    async def test_hybrid_search_text_match_hyphen_and_apostrophe(
+        self, vs_text_match: AsyncPGVectorStore, text_match: str
+    ) -> None:
+        """Test that hyphens and apostrophes in the query still match."""
+        assert await self._keyword_matches(
+            vs_text_match,
+            "third-party vendor's",
+            tsv_column="mycontent_tsv",
+            text_match=text_match,
+        ) == ["tm_doc_vendor"]
+
+    async def test_hybrid_search_text_match_any_tsquery_syntax_characters(
+        self, vs_text_match: AsyncPGVectorStore
+    ) -> None:
+        """Test that "any" handles terms containing tsquery operators, like URLs."""
+        # The URL produces the term "/a:b?x=1", which is a tsquery syntax error
+        # if it is not quoted.
+        assert await self._keyword_matches(
+            vs_text_match,
+            "vendor at http://example.com/a:b?x=1 (c) & !d | 'e'",
+            tsv_column="mycontent_tsv",
+            text_match="any",
+        ) == ["tm_doc_vendor"]
+
+    async def test_hybrid_search_text_match_default_is_all(
+        self, engine: PGEngine, vs_text_match: AsyncPGVectorStore
+    ) -> None:
+        """Test that the default config matches exactly what plainto_tsquery matches."""
+        assert HybridSearchConfig().text_match == "all"
+
+        for query in [
+            "orange fruit",
+            "which citrus grove",
+            "third-party vendor's",
+            "how is the",
+            "fluffy cat",
+        ]:
+            async with engine._pool.connect() as conn:
+                result = await conn.execute(
+                    text(
+                        f'SELECT doc_id_key FROM "{TEXT_MATCH_TABLE}" '
+                        "WHERE mycontent_tsv @@ "
+                        "plainto_tsquery('pg_catalog.english', :query)"
+                    ),
+                    {"query": query},
+                )
+                expected = sorted(row[0] for row in result.fetchall())
+
+            assert (
+                await self._keyword_matches(
+                    vs_text_match, query, tsv_column="mycontent_tsv"
+                )
+                == expected
+            ), query
+
     async def test_hybrid_search_weighted_sum_default(
         self,
         vs_hybrid_search_with_tsv_column: AsyncPGVectorStore,

tests/unit_tests/v2/test_hybrid_search_config.py

@@ -4,6 +4,7 @@
 from sqlalchemy import RowMapping
 
 from langchain_postgres.v2.hybrid_search_config import (
+    HybridSearchConfig,
     reciprocal_rank_fusion,
     weighted_sum_ranking,
 )
@@ -312,3 +313,16 @@ def test_rrf_with_identical_scores(self) -> None:
         assert results[0]["distance"] == pytest.approx(1 / 60)
         assert results[1]["id_val"] == "p2"
         assert results[1]["distance"] == pytest.approx(1 / 61)
+
+
+class TestHybridSearchConfig:
+    @pytest.mark.parametrize("text_match", ["all", "any"])
+    def test_text_match_valid(self, text_match: str) -> None:
+        config = HybridSearchConfig(text_match=text_match)  # type: ignore[arg-type]
+        assert config.text_match == text_match
+
+    @pytest.mark.parametrize("text_match", ["ANY", "or", "", None])
+    def test_text_match_invalid(self, text_match: object) -> None:
+        """Tests that an unsupported text_match value is rejected."""
+        with pytest.raises(ValueError, match="text_match must be"):
+            HybridSearchConfig(text_match=text_match)  # type: ignore[arg-type]