add langchain vector sample

examples/llm-banking/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+.chroma/
+.dozer/
+__pycache__/
+*.pyc

examples/llm-banking/README.md

@@ -0,0 +1,68 @@
+# LLM banking sample (Dozer + LangChain + vector store)
+
+Bank-ish credit card advisor. Dozer pulls profile + txn CSVs, SQL builds a
+unified `customer_profile`, sinks it (Dummy by default, ClickHouse optional).
+The Python app does retrieval over card docs with Chroma and picks a card.
+
+Matches the old blog use case, against **current** Dozer (sources → sql → sinks).
+Dozer no longer ships REST table APIs — the app owns that layer.
+
+## Layout
+
+```
+dozer-config.yaml   # LocalStorage CSVs + SQL + Dummy sinks
+data/               # profiles, transactions, card products
+app.py              # CLI: profile → rank → vector retrieve → answer
+load_data.py        # CSV loaders + profile builder (mirrors the SQL)
+recommend.py        # offline ranking
+vector_rag.py       # Chroma + local hash embeddings
+docker-compose.yml  # optional ClickHouse
+```
+
+## Quick path (no Dozer binary needed)
+
+```bash
+python3 -m venv .venv && . .venv/bin/activate
+pip install -r requirements.txt
+# optional for real LangChain/Chroma/OpenAI — uncomment lines in requirements.txt
+python app.py --customer-id cust_002
+python -m unittest discover -s tests -v
+```
+
+`cust_002` is travel-heavy — expect Dozer Travel Signature near the top.
+
+## With Dozer
+
+Needs a Dozer build/install from this repo (`cargo build -p dozer-cli` or a
+release binary).
+
+```bash
+# from this directory
+dozer -c dozer-config.yaml build
+dozer -c dozer-config.yaml run
+```
+
+Dummy sinks just prove the pipeline. To actually query the sink:
+
+```bash
+docker compose up -d
+# swap the Dummy sinks for the commented ClickHouse block in dozer-config.yaml
+dozer run -c dozer-config.yaml
+python app.py --customer-id cust_002 --use-clickhouse
+```
+
+## LLM
+
+Set `OPENAI_API_KEY` if you want ChatOpenAI for the final blurb. Without it the
+local ranker prints the answer. Fine for CI.
+
+## Why this shape
+
+Issue #1690 asked for Dozer + vector DB + LangChain on a credit-card scenario.
+Current main moves data into sinks (see root README) — so this sample wires
+Dozer that way instead of inventing `/customer_profiles` REST endpoints that
+the binary no longer serves.
+
+## Demo
+
+`demo.mp4` is a short capture of `scripts/demo.sh` / `app.py` on cust_002 and cust_003. Or just run the script yourself.

examples/llm-banking/app.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Dozer-shaped banking sample: unified profile + vector retrieval + card pick.
+
+Offline by default (reads the same CSVs Dozer would ingest). Pass --use-clickhouse
+if you sank customer_profile into ClickHouse via dozer-config.yaml.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+from load_data import (
+    build_customer_profile,
+    load_products,
+    product_as_text,
+    profile_as_text,
+)
+from recommend import local_answer, rank_cards
+from vector_rag import build_docs, retrieve
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--customer-id", default="cust_002")
+    parser.add_argument(
+        "--question",
+        default="Which credit card should we offer this customer and why?",
+    )
+    parser.add_argument(
+        "--use-clickhouse",
+        action="store_true",
+        help="try reading customer_profile from ClickHouse first",
+    )
+    parser.add_argument(
+        "--no-vector",
+        action="store_true",
+        help="skip chroma; just print ranked cards",
+    )
+    args = parser.parse_args(argv)
+
+    profile = None
+    if args.use_clickhouse:
+        from clickhouse_client import fetch_profile
+
+        ch = fetch_profile(args.customer_id)
+        if ch:
+            # ClickHouse row is the SQL aggregate; fill spend detail from CSV.
+            profile = build_customer_profile(args.customer_id)
+            profile.update({k: ch[k] for k in ch if k in profile or k == "total_spend"})
+            print("loaded profile from ClickHouse + local txn detail")
+        else:
+            print("clickhouse miss; falling back to CSV profile", file=sys.stderr)
+
+    if profile is None:
+        profile = build_customer_profile(args.customer_id)
+
+    products = load_products()
+    ranked = rank_cards(profile, products)
+
+    print("--- profile ---")
+    print(profile_as_text(profile))
+    print("--- ranked ---")
+    for i, p in enumerate(ranked, 1):
+        print(f"{i}. {p['name']} score={p['score']} ({p['why']})")
+
+    if not args.no_vector:
+        docs = build_docs(profile_as_text(profile), products, product_as_text)
+        # bake profile terms into the query so hash/chroma retrieval isn't random
+        q = f"{args.question} {profile['goals']} {' '.join(profile['top_categories'])}"
+        hits = retrieve(docs, q, k=3)
+        print("--- vector hits ---")
+        for h in hits:
+            print(f"* [{h.metadata.get('kind')}] {h.page_content[:120]}...")
+
+    if os.getenv("OPENAI_API_KEY"):
+        try:
+            from langchain_openai import ChatOpenAI
+            from langchain_core.prompts import ChatPromptTemplate
+
+            ctx = profile_as_text(profile) + "\n" + "\n".join(
+                f"- {p['name']}: {p['why']} (score {p['score']})" for p in ranked[:3]
+            )
+            prompt = ChatPromptTemplate.from_template(
+                "You are a bank product advisor. Context:\n{ctx}\n\nQuestion: {q}\n"
+                "Pick one card and say why in 3 sentences."
+            )
+            llm = ChatOpenAI(model=os.getenv("OPENAI_MODEL", "gpt-4o-mini"), temperature=0)
+            msg = (prompt | llm).invoke({"ctx": ctx, "q": args.question})
+            print("--- llm ---")
+            print(msg.content)
+            return 0
+        except Exception as exc:
+            print(f"llm failed ({exc}); using local answer", file=sys.stderr)
+
+    print("--- answer ---")
+    print(local_answer(profile, ranked, args.question))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/llm-banking/clickhouse_client.py

@@ -0,0 +1,27 @@
+"""Optional reader for the ClickHouse sink path."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+
+def fetch_profile(customer_id: str) -> dict[str, Any] | None:
+    host = os.getenv("CLICKHOUSE_HOST", "localhost")
+    port = int(os.getenv("CLICKHOUSE_HTTP_PORT", "8123"))
+    try:
+        import clickhouse_connect
+    except ImportError:
+        return None
+
+    try:
+        client = clickhouse_connect.get_client(host=host, port=port)
+        rows = client.query(
+            "SELECT customer_id, segment, income_band, goals, risk_tolerance, total_spend "
+            "FROM customer_profile WHERE customer_id = {cid:String}",
+            parameters={"cid": customer_id},
+        ).named_results()
+        row = next(iter(rows), None)
+        return dict(row) if row else None
+    except Exception:
+        return None

examples/llm-banking/data/credit_card_products/credit_card_products.csv

@@ -0,0 +1,5 @@
+product_id,name,annual_fee_usd,rewards_focus,intro_offer,eligibility,caveat
+card_cash,Dozer Cash Everyday,0,cashback groceries gas dining,2% groceries 1% other,fair credit ok,no lounge
+card_travel,Dozer Travel Signature,95,travel flights hotels dining,60k points after 3k spend,good credit,fee waives with 15k spend
+card_starter,Dozer Starter Secure,0,build credit no annual fee,no intro bonus,thin credit file ok,security deposit may apply
+card_premium,Dozer Reserve,450,travel lounge dining hotels,100k points after 5k spend,excellent credit,high fee

examples/llm-banking/data/customer_profiles/customer_profiles.csv

@@ -0,0 +1,5 @@
+customer_id,age,income_band,segment,goals,risk_tolerance
+cust_001,34,mid,everyday,cashback groceries dining,medium
+cust_002,41,premium,travel,travel hotels flights,medium
+cust_003,29,starter,student,build credit no annual fee,low
+cust_004,52,premium,business,travel dining lounge access,high

examples/llm-banking/data/transactions/transactions.csv

@@ -0,0 +1,16 @@
+transaction_id,customer_id,merchant_category,amount_usd,txn_date
+txn_001,cust_001,groceries,86.40,2024-11-02
+txn_002,cust_001,dining,42.10,2024-11-05
+txn_003,cust_001,groceries,61.25,2024-11-12
+txn_004,cust_001,gas,38.00,2024-11-15
+txn_005,cust_002,flights,620.00,2024-11-01
+txn_006,cust_002,hotels,310.50,2024-11-03
+txn_007,cust_002,dining,95.00,2024-11-08
+txn_008,cust_002,flights,240.00,2024-11-18
+txn_009,cust_003,groceries,28.90,2024-11-04
+txn_010,cust_003,streaming,15.99,2024-11-07
+txn_011,cust_003,transit,22.50,2024-11-14
+txn_012,cust_004,flights,890.00,2024-11-02
+txn_013,cust_004,dining,180.00,2024-11-06
+txn_014,cust_004,hotels,450.00,2024-11-09
+txn_015,cust_004,dining,95.50,2024-11-16

examples/llm-banking/demo-output.txt

@@ -0,0 +1,48 @@
+$ scripts/demo.sh
+
+test_config_text_shape (test_config.ConfigTests.test_config_text_shape) ... ok
+test_csvs_exist_and_have_rows (test_config.ConfigTests.test_csvs_exist_and_have_rows) ... ok
+test_schema_has_sinks_not_endpoints (test_config.ConfigTests.test_schema_has_sinks_not_endpoints) ... ok
+test_sql_mentions_join (test_config.ConfigTests.test_sql_mentions_join) ... ok
+test_local_answer_mentions_customer (test_recommend.RankTests.test_local_answer_mentions_customer) ... ok
+test_profile_has_spend_breakdown (test_recommend.RankTests.test_profile_has_spend_breakdown) ... ok
+test_starter_prefers_no_fee (test_recommend.RankTests.test_starter_prefers_no_fee) ... ok
+test_travel_customer_gets_travel_card (test_recommend.RankTests.test_travel_customer_gets_travel_card) ... ok
+test_travel_query_surfaces_travel_card (test_vector.VectorTests.test_travel_query_surfaces_travel_card) ... ok
+
+----------------------------------------------------------------------
+Ran 9 tests in 0.002s
+
+OK
+
+$ python app.py --customer-id cust_002
+--- profile ---
+Customer cust_002 (travel, premium income). Goals: travel hotels flights. Risk: medium. Total recent spend $1265.50. Breakdown: flights $860.00, hotels $310.50, dining $95.00.
+--- ranked ---
+1. Dozer Travel Signature score=25 (flights spend fits rewards; hotels spend fits rewards; dining spend fits rewards; goal 'travel'; premium ok with fee)
+2. Dozer Reserve score=17 (hotels spend fits rewards; dining spend fits rewards; goal 'travel'; premium ok with fee)
+3. Dozer Cash Everyday score=3 (dining spend fits rewards)
+4. Dozer Starter Secure score=0 (baseline)
+--- vector hits ---
+* [product] Dozer Travel Signature (fee $95): rewards on travel flights hotels dining. Intro: 60k points after 3k spend. Eligibility...
+* [customer] Customer cust_002 (travel, premium income). Goals: travel hotels flights. Risk: medium. Total recent spend $1265.50. Bre...
+* [product] Dozer Reserve (fee $450): rewards on travel lounge dining hotels. Intro: 100k points after 5k spend. Eligibility: excell...
+--- answer ---
+Q: Which credit card should we offer this customer and why?
+For cust_002 I'd lead with Dozer Travel Signature (score 25). Reason: flights spend fits rewards; hotels spend fits rewards; dining spend fits rewards; goal 'travel'; premium ok with fee. Profile spend $1265.50 skewed to flights, hotels, dining.
+
+$ python app.py --customer-id cust_003
+--- profile ---
+Customer cust_003 (student, starter income). Goals: build credit no annual fee. Risk: low. Total recent spend $67.39. Breakdown: groceries $28.90, transit $22.50, streaming $15.99.
+--- ranked ---
+1. Dozer Starter Secure score=9 (goal 'build credit'; no fee; starter-friendly)
+2. Dozer Cash Everyday score=5 (groceries spend fits rewards; no fee)
+3. Dozer Travel Signature score=0 (baseline)
+4. Dozer Reserve score=-3 (eligibility stretch)
+--- vector hits ---
+* [product] Dozer Starter Secure (fee $0): rewards on build credit no annual fee. Intro: no intro bonus. Eligibility: thin credit fi...
+* [customer] Customer cust_003 (student, starter income). Goals: build credit no annual fee. Risk: low. Total recent spend $67.39. Br...
+* [product] Dozer Cash Everyday (fee $0): rewards on cashback groceries gas dining. Intro: 2% groceries 1% other. Eligibility: fair ...
+--- answer ---
+Q: Which credit card should we offer this customer and why?
+For cust_003 I'd lead with Dozer Starter Secure (score 9). Reason: goal 'build credit'; no fee; starter-friendly. Profile spend $67.39 skewed to groceries, transit, streaming.

examples/llm-banking/docker-compose.yml

@@ -0,0 +1,11 @@
+# Optional. Only needed if you want Dozer to sink into ClickHouse.
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.8
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144

examples/llm-banking/dozer-config.yaml

@@ -0,0 +1,98 @@
+# Current Dozer shape: connections -> sources -> sql -> sinks.
+# No REST endpoints here — those went away. App owns the API / LLM side.
+app_name: llm-banking-sample
+version: 1
+
+connections:
+  - name: bank_csvs
+    config: !LocalStorage
+      details:
+        path: ./data
+      tables:
+        - name: customer_profiles
+          config: !CSV
+            path: customer_profiles
+            extension: .csv
+        - name: transactions
+          config: !CSV
+            path: transactions
+            extension: .csv
+        - name: credit_card_products
+          config: !CSV
+            path: credit_card_products
+            extension: .csv
+
+sources:
+  - name: customer_profiles
+    table_name: customer_profiles
+    connection: bank_csvs
+  - name: transactions
+    table_name: transactions
+    connection: bank_csvs
+  - name: credit_card_products
+    table_name: credit_card_products
+    connection: bank_csvs
+
+# Unified customer profile — same idea as the blog post.
+sql: |
+  SELECT
+    c.customer_id,
+    c.segment,
+    c.income_band,
+    c.goals,
+    c.risk_tolerance,
+    SUM(t.amount_usd) AS total_spend
+  INTO customer_profile
+  FROM customer_profiles c
+  JOIN transactions t ON c.customer_id = t.customer_id
+  GROUP BY c.customer_id, c.segment, c.income_band, c.goals, c.risk_tolerance;
+
+  SELECT
+    product_id,
+    name,
+    annual_fee_usd,
+    rewards_focus,
+    intro_offer,
+    eligibility,
+    caveat
+  INTO card_catalog
+  FROM credit_card_products;
+
+sinks:
+  - name: customer_profile
+    config: !Dummy
+      table_name: customer_profile
+  - name: card_catalog
+    config: !Dummy
+      table_name: card_catalog
+
+# Optional ClickHouse path (uncomment + docker compose up):
+# sinks:
+#   - name: customer_profile_ch
+#     config: !Clickhouse
+#       host: localhost
+#       port: 9000
+#       user: default
+#       password: ""
+#       database: default
+#       options: []
+#       source_table_name: customer_profile
+#       sink_table_name: customer_profile
+#       create_table_options:
+#         engine: MergeTree
+#         order_by: [customer_id]
+#         primary_keys: [customer_id]
+#   - name: card_catalog_ch
+#     config: !Clickhouse
+#       host: localhost
+#       port: 9000
+#       user: default
+#       password: ""
+#       database: default
+#       options: []
+#       source_table_name: card_catalog
+#       sink_table_name: card_catalog
+#       create_table_options:
+#         engine: MergeTree
+#         order_by: [product_id]
+#         primary_keys: [product_id]