Feat/acronym search 04 detection and retrieval

.env.sample

@@ -9,10 +9,16 @@ QDRANT_CHECK_COMPATIBILITY=false
 # ── Redis ─────────────────────────────────────────────────────────────────────
 REDIS_HOST=localhost
 REDIS_PORT=6379
+REDIS_DB=0
 REDIS_PASSWORD=
 REDIS_CACHE_TTL=86400
 REDIS_MAX_CACHE_SIZE=1000
 
+# ── Internal API ──────────────────────────────────────────────────────────────
+# Shared secret checked against the X-Internal-Token header on internal-only
+# endpoints (e.g. POST /api/acronyms/bulk). Set to a real random value per env.
+INTERNAL_API_TOKEN=changeme-generate-a-random-token-per-environment
+
 # ── Embedding & LLM Models ────────────────────────────────────────────────────
 EMBEDDING_MODEL=all-MiniLM-L6-v2
 # AWS Bedrock LLM (summarisation / tagging) and the region its endpoint lives in
@@ -28,6 +34,8 @@ MARKDOWN_CHUNK_OVERLAP=800
 # ── Ingestion / Upload ────────────────────────────────────────────────────────
 # Max size (in MB) of a single uploaded document accepted for processing (1GB default)
 MAX_FILE_SIZE_MB=1024
+# Max size (in MB) of an acronym bulk-upload CSV (small tabular text, not a document)
+ACRONYM_BULK_UPLOAD_MAX_SIZE_MB=5
 
 # ── Search ────────────────────────────────────────────────────────────────────
 SIMILARITY_THRESHOLD=0.40
@@ -106,6 +114,12 @@ INCLUDE_SCORING_DEBUG=false
 SEARCH_CANDIDATE_FANOUT=8
 SEARCH_CANDIDATE_MAX=2000
 
+# ── Acronym Search ────────────────────────────────────────────────────────────
+# Master switch for the acronym detect -> expand -> tiered-rank search path.
+# Non-acronym queries are unaffected either way — this only gates the
+# acronym-detected path. Validated on real traffic (see ACRONYM_SEARCH_PLAN.md).
+ACRONYM_SEARCH_ENABLED=true
+
 # ── Database ──────────────────────────────────────────────────────────────────
 POSTGRES_DATABASE_URI=postgresql://username:password@localhost:5432/ai_vector_service
 

alembic.ini

@@ -0,0 +1,150 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts.
+# this is typically a path given in POSIX (e.g. forward slashes)
+# format, relative to the token %(here)s which refers to the location of this
+# ini file
+script_location = %(here)s/migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+# Or organize into date-based subdirectories (requires recursive_version_locations = true)
+# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.  for multiple paths, the path separator
+# is defined by "path_separator" below.
+prepend_sys_path = .
+
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the tzdata library which can be installed by adding
+# `alembic[tz]` to the pip requirements.
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to <script_location>/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "path_separator"
+# below.
+# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
+
+# path_separator; This indicates what character is used to split lists of file
+# paths, including version_locations and prepend_sys_path within configparser
+# files such as alembic.ini.
+# The default rendered in new alembic.ini files is "os", which uses os.pathsep
+# to provide os-dependent path splitting.
+#
+# Note that in order to support legacy alembic.ini files, this default does NOT
+# take place if path_separator is not present in alembic.ini.  If this
+# option is omitted entirely, fallback logic is as follows:
+#
+# 1. Parsing of the version_locations option falls back to using the legacy
+#    "version_path_separator" key, which if absent then falls back to the legacy
+#    behavior of splitting on spaces and/or commas.
+# 2. Parsing of the prepend_sys_path option falls back to the legacy
+#    behavior of splitting on spaces, commas, or colons.
+#
+# Valid values for path_separator are:
+#
+# path_separator = :
+# path_separator = ;
+# path_separator = space
+# path_separator = newline
+#
+# Use os.pathsep. Default configuration used for new projects.
+path_separator = os
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# database URL.  This is consumed by the user-maintained env.py script only.
+# other means of configuring database URLs may be customized within the env.py
+# file.
+# Overridden at runtime by migrations/env.py from settings.DATABASE_URL (env-driven).
+# sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
+# hooks = ruff
+# ruff.type = module
+# ruff.module = ruff
+# ruff.options = check --fix REVISION_SCRIPT_FILENAME
+
+# Alternatively, use the exec runner to execute a binary found on your PATH
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = ruff
+# ruff.options = check --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration.  This is also consumed by the user-maintained
+# env.py script only.
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

app/api/deps.py

@@ -0,0 +1,25 @@
+import secrets
+from typing import Optional
+
+from fastapi import Header, HTTPException
+
+from app.config import settings
+
+
+def verify_internal_token(x_internal_token: Optional[str] = Header(None)) -> None:
+    """Gate internal-only endpoints behind a shared secret (X-Internal-Token header).
+
+    Header is optional so a missing one fails with the same 401 as a wrong one,
+    not a 422 that leaks the auth mechanism. compare_digest on utf-8/
+    surrogateescape-encoded bytes gives constant-time comparison (no timing
+    side-channel) without crashing on non-ASCII header bytes.
+    """
+    if (
+        not settings.INTERNAL_API_TOKEN
+        or not x_internal_token
+        or not secrets.compare_digest(
+            x_internal_token.encode("utf-8", "surrogateescape"),
+            settings.INTERNAL_API_TOKEN.encode("utf-8", "surrogateescape"),
+        )
+    ):
+        raise HTTPException(status_code=401, detail="Invalid or missing internal token")

app/api/v1/api.py

@@ -1,8 +1,9 @@
 from fastapi import APIRouter
-from app.api.v1.endpoints import documents, query, cache
+from app.api.v1.endpoints import documents, query, cache, acronyms
 
 api_router = APIRouter()
 
 api_router.include_router(documents.router, prefix="", tags=["documents"])
 api_router.include_router(query.router, prefix="/query", tags=["query"])
 api_router.include_router(cache.router, prefix="/cache", tags=["cache"])
+api_router.include_router(acronyms.router, prefix="/acronyms", tags=["acronyms"])

app/api/v1/endpoints/acronyms.py

@@ -0,0 +1,85 @@
+import csv
+import io
+import logging
+
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
+from starlette.concurrency import run_in_threadpool
+
+from app.api.deps import verify_internal_token
+from app.config import settings
+from app.constants import ACRONYM_CSV_COLUMN_ACRONYM, ACRONYM_CSV_COLUMN_EXPANSIONS
+from app.models.api_models import AcronymBulkUploadResponse
+from app.services.acronym_service import bulk_upsert, invalidate_cache, refresh_cache
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+@router.post("/bulk", response_model=AcronymBulkUploadResponse, dependencies=[Depends(verify_internal_token)])
+async def bulk_upload_acronyms(file: UploadFile = File(...)):
+    """Internal-only: upsert acronym -> expansions rows from a CSV upload (spec
+    §7). Columns: acronym, expansions (pipe-separated), description (optional),
+    is_active (optional, "true"/"false" — defaults to active if omitted).
+    A bad row is reported in `errors`, not a batch failure — the rest commits.
+    """
+    raw = await file.read()
+    max_bytes = settings.ACRONYM_BULK_UPLOAD_MAX_SIZE_MB * 1024 * 1024
+    if len(raw) > max_bytes:
+        raise HTTPException(
+            status_code=413,
+            detail=f"File exceeds max size of {settings.ACRONYM_BULK_UPLOAD_MAX_SIZE_MB}MB",
+        )
+
+    try:
+        # utf-8-sig strips a leading BOM if present (common in CSVs exported
+        # from Excel/Sheets) and is otherwise identical to plain utf-8 — a
+        # BOM left in place would silently become part of the first header
+        # name ("acronym"), making every row fail acronym validation.
+        text = raw.decode("utf-8-sig")
+    except UnicodeDecodeError:
+        raise HTTPException(status_code=400, detail="File must be UTF-8 encoded")
+
+    reader = csv.DictReader(io.StringIO(text))
+    required_columns = {ACRONYM_CSV_COLUMN_ACRONYM, ACRONYM_CSV_COLUMN_EXPANSIONS}
+    if not reader.fieldnames or not required_columns.issubset(reader.fieldnames):
+        raise HTTPException(
+            status_code=400,
+            detail=f"CSV must have columns {sorted(required_columns)}, found {reader.fieldnames}",
+        )
+
+    rows = list(reader)
+    if not rows:
+        raise HTTPException(status_code=400, detail="CSV file has no data rows")
+
+    # bulk_upsert/refresh_cache/invalidate_cache are all synchronous, blocking
+    # Postgres/Redis I/O — run_in_threadpool keeps them off the event loop.
+    created, updated, deactivated, errors = await run_in_threadpool(bulk_upsert, rows)
+
+    # Spec §7: commit first (bulk_upsert already did), then refresh the cache.
+    # deactivated rows go straight to invalidate_cache (no DB round-trip needed,
+    # bulk_upsert already knows their status) — refresh_cache's own query filters
+    # to is_active=true, so it would silently skip them and leave their old
+    # cached expansion in place. If anything in this block fails, invalidate the
+    # whole batch instead so the next lookup reloads from Postgres rather than
+    # serving stale data.
+    if created or updated:
+        active_batch = [a for a in (created + updated) if a not in deactivated]
+        try:
+            if active_batch:
+                await run_in_threadpool(refresh_cache, active_batch)
+            if deactivated:
+                await run_in_threadpool(invalidate_cache, deactivated)
+        except Exception as e:
+            logger.warning(f"Cache refresh failed after bulk upload, invalidating instead: {e}")
+            await run_in_threadpool(invalidate_cache, created + updated)
+
+    logger.info(
+        f"Acronym bulk upload: {len(created)} created, {len(updated)} updated, "
+        f"{len(errors)} error(s)"
+    )
+    return AcronymBulkUploadResponse(
+        received=len(rows),
+        created=len(created),
+        updated=len(updated),
+        errors=errors,
+    )

app/config.py

@@ -42,19 +42,48 @@ class Settings(BaseSettings):
     MAX_CACHE_RESULTS: int = 1
     REDIS_HOST: str = os.getenv("REDIS_HOST", "localhost")
     REDIS_PORT: int = int(os.getenv("REDIS_PORT", 6379))
+    REDIS_DB: int = int(os.getenv("REDIS_DB", 0))
     REDIS_PASSWORD: str = os.getenv("REDIS_PASSWORD", "")
+    # Small on purpose: redis-py has no default timeout, and a blackholed
+    # connection (packets silently dropped) would hang cache reads for minutes,
+    # stalling search instead of falling back to Postgres.
+    REDIS_SOCKET_CONNECT_TIMEOUT: float = float(os.getenv("REDIS_SOCKET_CONNECT_TIMEOUT", 1))
+    REDIS_SOCKET_TIMEOUT: float = float(os.getenv("REDIS_SOCKET_TIMEOUT", 1))
     REDIS_CACHE_TTL: int = int(os.getenv("REDIS_CACHE_TTL", 86400))  # 24 hours in seconds
+    # Shorter than REDIS_CACHE_TTL: caches "this word isn't an acronym" so ordinary
+    # non-acronym words in a query don't re-hit Postgres on every request. A newly
+    # bulk-uploaded acronym overwrites any stale negative entry immediately (warm_cache()
+    # runs after every upload), so this TTL only bounds staleness for the rare case that
+    # invariant doesn't hold — not load-bearing correctness.
+    REDIS_NEGATIVE_CACHE_TTL: int = int(os.getenv("REDIS_NEGATIVE_CACHE_TTL", 3600))  # 1 hour
+    # Shorter than REDIS_NEGATIVE_CACHE_TTL: this caches a DB-error miss, not a
+    # genuine "not an acronym" miss. DB outages are often transient — caching
+    # "not found" for a full hour would hide real acronyms after Postgres recovers.
+    REDIS_DB_ERROR_CACHE_TTL: int = int(os.getenv("REDIS_DB_ERROR_CACHE_TTL", 30))
     REDIS_MAX_CACHE_SIZE: int = int(os.getenv("REDIS_MAX_CACHE_SIZE", 1000))
     DATABASE_URL: str = os.getenv("POSTGRES_DATABASE_URI", "postgresql://anuj:1234@localhost:5432/ai_vector_service")
+    # psycopg2 defaults to no connect timeout at all — a blackholed Postgres host
+    # (network silently drops packets, unlike a clean "connection refused") makes
+    # a connection attempt hang until the OS-level TCP timeout, which can be
+    # minutes. Confirmed live with a real local "black hole" TCP server: an
+    # unbounded psycopg2.connect() hung past 15s with no sign of returning.
+    POSTGRES_CONNECT_TIMEOUT: int = int(os.getenv("POSTGRES_CONNECT_TIMEOUT", 3))
     REDIS_CACHE_ENABLED: bool = False
 
+    # Shared secret for internal-only endpoints (e.g. acronym bulk upload), checked
+    # against the X-Internal-Token request header. No default — must be set explicitly.
+    INTERNAL_API_TOKEN: str = os.getenv("INTERNAL_API_TOKEN", "")
+
     # URL extraction settings
     URL_EXTRACTION_CHUNK_SIZE: int = 1500
     URL_EXTRACTION_CHUNK_OVERLAP: int = 300  # 20% overlap
     URL_REQUEST_TIMEOUT: int = 30  # seconds
 
     # File upload settings
     MAX_FILE_SIZE_MB: int = int(os.getenv("MAX_FILE_SIZE_MB", 1024))  # 1GB default (in MB)
+    # Acronym bulk-upload CSVs are small tabular text, not documents —
+    # a much lower cap than MAX_FILE_SIZE_MB.
+    ACRONYM_BULK_UPLOAD_MAX_SIZE_MB: int = int(os.getenv("ACRONYM_BULK_UPLOAD_MAX_SIZE_MB", 5))
 
 
     # Prioritized Search Configuration
@@ -84,6 +113,10 @@ class Settings(BaseSettings):
     SHORT_QUERY_THRESHOLD: int = int(os.getenv("SHORT_QUERY_THRESHOLD", "3"))
     RRF_K: int = int(os.getenv("RRF_K", "60"))  # standard Reciprocal Rank Fusion constant
 
+    # Acronym Search — default false until the full detect -> expand -> tiered-rank
+    # path lands and passes validation (see ACRONYM_SEARCH_PLAN.md).
+    ACRONYM_SEARCH_ENABLED: bool = os.getenv("ACRONYM_SEARCH_ENABLED", "false").lower() == "true"
+
     # Sparse Vector Configuration (Phase 2 — requires qdrant-client>=1.9.0)
     SPARSE_VECTOR_NAME: str = os.getenv("SPARSE_VECTOR_NAME", "bm25")
     SPARSE_SEARCH_ENABLED: bool = os.getenv("SPARSE_SEARCH_ENABLED", "false").lower() == "true"

app/constants.py

@@ -0,0 +1,7 @@
+ACRONYM_CACHE_KEY_PREFIX = "acronym"
+
+# Bulk-upload CSV column names (spec §7)
+ACRONYM_CSV_COLUMN_ACRONYM = "acronym"
+ACRONYM_CSV_COLUMN_EXPANSIONS = "expansions"
+ACRONYM_CSV_COLUMN_DESCRIPTION = "description"
+ACRONYM_CSV_COLUMN_IS_ACTIVE = "is_active"