refactor(metrics): move each metric beside the code that increments it

src/gateway/api/deps.py

@@ -15,7 +15,7 @@
 from gateway.core.database import DATABASE_ERRORS, create_session, get_db
 from gateway.core.feature import CoreFeature
 from gateway.log_config import logger
-from gateway.metrics import record_auth_failure
+from gateway.metrics import REGISTRY, Counter
 from gateway.models.entities import APIKey
 from gateway.models.tenancy import User as TenancyUser
 from gateway.ports.billing_port import BillingPort
@@ -39,6 +39,18 @@
 _config: GatewayConfig | None = None
 _LAST_USED_UPDATE_INTERVAL_SECONDS = 300
 
+AUTH_FAILURES = Counter(
+    "gateway_auth_failures",
+    "Total number of authentication failures",
+    ["reason"],
+    registry=REGISTRY,
+)
+
+
+def record_auth_failure(reason: str) -> None:
+    """Record an authentication failure."""
+    AUTH_FAILURES.labels(reason=reason).inc()
+
 
 def _as_utc(value: datetime | None) -> datetime | None:
     """Return ``value`` as a timezone-aware datetime in UTC.

src/gateway/api/routes/_attempts.py

@@ -38,10 +38,10 @@
 from gateway.api.routes._platform import (
     _provider_failure_http_exc,
     is_provider_billing_error,
+    record_abandoned_attempt,
     upstream_exception_shape,
 )
 from gateway.log_config import logger
-from gateway.metrics import record_abandoned_attempt
 from gateway.services.mcp_loop import MaxToolIterationsExceeded
 from gateway.services.sandbox_backend import SandboxNotReachableError
 from gateway.services.web_retrieval_backend import WebSearchNotReachableError

src/gateway/api/routes/_pipeline.py

@@ -85,6 +85,7 @@
     _resolve_platform_mcp_servers,
     _resolve_platform_web_search,
     is_provider_billing_error,
+    record_abandoned_attempt,
     run_platform_attempts,
     upstream_error_message,
     upstream_exception_chain,
@@ -117,7 +118,8 @@
 )
 from gateway.inflight import track_request
 from gateway.log_config import logger
-from gateway.metrics import record_abandoned_attempt, record_cost, record_inline_cost_settlement, record_tokens
+from gateway.metrics import REGISTRY, Histogram
+from gateway.metrics import Counter as PrometheusCounter
 from gateway.model_labeling import relabel_model
 from gateway.models.entities import APIKey, ModelPricing, UsageLog
 from gateway.models.guardrails import GuardrailConfig
@@ -220,6 +222,46 @@
 ResultT = TypeVar("ResultT")
 ChunkT = TypeVar("ChunkT")
 
+TOKENS = PrometheusCounter(
+    "gateway_tokens",
+    "Total number of tokens processed",
+    ["provider", "model", "type"],
+    registry=REGISTRY,
+)
+
+REQUEST_COST_DOLLARS = Histogram(
+    "gateway_request_cost_dollars",
+    "Request cost in USD",
+    ["provider", "model"],
+    registry=REGISTRY,
+)
+
+INLINE_COST_SETTLEMENTS = PrometheusCounter(
+    "gateway_inline_cost_settlements",
+    "Inline platform cost settlement outcomes on the hybrid response path",
+    ["outcome"],
+    registry=REGISTRY,
+)
+
+
+def record_tokens(provider: str, model: str, prompt_tokens: int, completion_tokens: int) -> None:
+    """Record token usage metrics."""
+    if prompt_tokens:
+        TOKENS.labels(provider=provider, model=model, type="input").inc(prompt_tokens)
+    if completion_tokens:
+        TOKENS.labels(provider=provider, model=model, type="output").inc(completion_tokens)
+
+
+def record_cost(provider: str, model: str, cost: float) -> None:
+    """Record request cost."""
+    REQUEST_COST_DOLLARS.labels(provider=provider, model=model).observe(cost)
+
+
+def record_inline_cost_settlement(outcome: str) -> None:
+    """Record an attached, unattached, or timed-out inline settlement."""
+    INLINE_COST_SETTLEMENTS.labels(outcome=outcome).inc()
+
+
 # ---------------------------------------------------------------------------
 # Shared wire-level detail strings. These are client-visible API contract
 # values; do not edit them without a deprecation plan.

src/gateway/api/routes/_platform.py

@@ -34,7 +34,7 @@
     cache_write_tokens_of,
 )
 from gateway.log_config import logger
-from gateway.metrics import record_abandoned_attempt
+from gateway.metrics import REGISTRY, Counter
 from gateway.models.mcp import McpServerConfig, ResolvedMcpServer
 from gateway.services.bedrock_gateway_auth import build_bedrock_client_args
 from gateway.services.mcp_loop import MaxToolIterationsExceeded
@@ -49,6 +49,26 @@
 
 T = TypeVar("T")
 
+ABANDONED_ATTEMPTS = Counter(
+    "gateway_abandoned_attempts",
+    "Total upstream attempts abandoned before their first chunk (provider fallback / timeout waste)",
+    ["provider", "model", "reason", "position"],
+    registry=REGISTRY,
+)
+
+
+def record_abandoned_attempt(provider: str, model: str, reason: str, position: int) -> None:
+    """Record an upstream attempt abandoned before it produced its first chunk.
+
+    ``reason`` is one of ``timeout`` (the first-chunk wait elapsed),
+    ``build_error`` (opening the upstream stream failed), or ``upstream_error``
+    (the upstream raised before yielding a chunk). ``position`` is the attempt's
+    index in the resolved routing plan; label cardinality stays bounded by the
+    plan length.
+    """
+    ABANDONED_ATTEMPTS.labels(provider=provider, model=model, reason=reason, position=str(position)).inc()
+
+
 # Status codes returned by the platform's usage-report endpoint that the
 # gateway should NOT retry. Auth, payment-required, not-found, conflict, gone,
 # and unprocessable are all permanent rejection signals: retrying would just

src/gateway/api/routes/auth_oauth.py

@@ -46,7 +46,7 @@
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from gateway.api.deps import IdentityProviderPortDep, get_config, get_db
+from gateway.api.deps import IdentityProviderPortDep, get_config, get_db, record_auth_failure
 from gateway.api.routes._public_auth import throttle_public_auth
 
 # The same refusal the password and passkey sign-ins carry, imported rather than
@@ -55,7 +55,6 @@
 from gateway.api.routes.auth_session import MAINTENANCE_MODE_REFUSAL
 from gateway.core.config import OAUTH_PROVIDERS, GatewayConfig
 from gateway.log_config import logger
-from gateway.metrics import record_auth_failure
 from gateway.services.dashboard_session_service import (
     apply_session_cookie,
     create_dashboard_session,

src/gateway/api/routes/auth_session.py

@@ -48,10 +48,9 @@
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from gateway.api.deps import get_config, get_db, is_valid_master_key
+from gateway.api.deps import get_config, get_db, is_valid_master_key, record_auth_failure
 from gateway.core.config import GatewayConfig
 from gateway.log_config import logger
-from gateway.metrics import record_auth_failure
 from gateway.models.tenancy import User as TenancyUser
 from gateway.rate_limit import RateLimiter
 from gateway.services.dashboard_session_service import (

src/gateway/api/routes/auth_webauthn.py

@@ -36,7 +36,7 @@
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from gateway.api.deps import CurrentIdentity, get_config, get_db
+from gateway.api.deps import CurrentIdentity, get_config, get_db, record_auth_failure
 from gateway.api.routes._public_auth import throttle_public_auth
 
 # The refusal is the same refusal, so it is imported rather than restated: the
@@ -45,7 +45,6 @@
 from gateway.api.routes.auth_session import MAINTENANCE_MODE_REFUSAL
 from gateway.core.config import GatewayConfig
 from gateway.log_config import logger
-from gateway.metrics import record_auth_failure
 from gateway.models.tenancy import (
     MAX_WEBAUTHN_CREDENTIAL_NAME,
     WebAuthnCredentialPublic,

src/gateway/metrics.py

@@ -1,4 +1,7 @@
-"""Prometheus metrics for the gateway."""
+"""Prometheus registry, metric types, and HTTP request instrumentation for the gateway.
+
+The metric types are re-exported so that code declaring a metric need not depend on ``prometheus_client`` directly.
+"""
 
 from __future__ import annotations
 
@@ -21,6 +24,8 @@
     from starlette.requests import Request
     from starlette.types import ASGIApp, Message, Receive, Scope, Send
 
+__all__ = ["Counter", "Gauge", "Histogram", "MetricsMiddleware", "REGISTRY", "metrics_endpoint"]
+
 REGISTRY = CollectorRegistry()
 
 # process_resident_memory_bytes and friends. The gateway keeps its own registry
@@ -49,80 +54,6 @@
     registry=REGISTRY,
 )
 
-TOKENS = Counter(
-    "gateway_tokens",
-    "Total number of tokens processed",
-    ["provider", "model", "type"],
-    registry=REGISTRY,
-)
-
-REQUEST_COST_DOLLARS = Histogram(
-    "gateway_request_cost_dollars",
-    "Request cost in USD",
-    ["provider", "model"],
-    registry=REGISTRY,
-)
-
-ABANDONED_ATTEMPTS = Counter(
-    "gateway_abandoned_attempts",
-    "Total upstream attempts abandoned before their first chunk (provider fallback / timeout waste)",
-    ["provider", "model", "reason", "position"],
-    registry=REGISTRY,
-)
-
-INLINE_COST_SETTLEMENTS = Counter(
-    "gateway_inline_cost_settlements",
-    "Inline platform cost settlement outcomes on the hybrid response path",
-    ["outcome"],
-    registry=REGISTRY,
-)
-
-RATE_LIMIT_HITS = Counter(
-    "gateway_rate_limit_hits",
-    "Total number of rate limit hits",
-    registry=REGISTRY,
-)
-
-BUDGET_EXCEEDED = Counter(
-    "gateway_budget_exceeded",
-    "Total number of budget exceeded events",
-    registry=REGISTRY,
-)
-
-AUTH_FAILURES = Counter(
-    "gateway_auth_failures",
-    "Total number of authentication failures",
-    ["reason"],
-    registry=REGISTRY,
-)
-
-LOG_WRITER_QUEUE_DEPTH = Gauge(
-    "gateway_usage_log_queue_depth",
-    "Number of usage log entries waiting to be written",
-    registry=REGISTRY,
-)
-
-LOG_WRITER_BATCH_SIZE = Histogram(
-    "gateway_usage_log_batch_size",
-    "Number of rows per flush batch",
-    ["writer"],
-    registry=REGISTRY,
-)
-
-LOG_WRITER_FLUSH_DURATION = Histogram(
-    "gateway_usage_log_flush_duration_seconds",
-    "Time spent flushing usage log batches",
-    ["writer", "result"],
-    registry=REGISTRY,
-)
-
-LOG_WRITER_ROWS = Counter(
-    "gateway_usage_log_rows",
-    "Total usage log rows by outcome",
-    ["writer", "result"],
-    registry=REGISTRY,
-)
-
 
 _PROMETHEUS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
 
@@ -202,54 +133,3 @@ async def send_wrapper(message: Message) -> None:
             REQUEST_DURATION_SECONDS.labels(method=method, endpoint=endpoint, api_version=api_version).observe(
                 duration
             )
-
-
-def record_tokens(provider: str, model: str, prompt_tokens: int, completion_tokens: int) -> None:
-    """Record token usage metrics."""
-    if prompt_tokens:
-        TOKENS.labels(provider=provider, model=model, type="input").inc(prompt_tokens)
-    if completion_tokens:
-        TOKENS.labels(provider=provider, model=model, type="output").inc(completion_tokens)
-
-
-def record_cost(provider: str, model: str, cost: float) -> None:
-    """Record request cost."""
-    REQUEST_COST_DOLLARS.labels(provider=provider, model=model).observe(cost)
-
-
-def record_abandoned_attempt(provider: str, model: str, reason: str, position: int) -> None:
-    """Record an upstream attempt abandoned before it produced its first chunk.
-
-    ``reason`` is one of ``timeout`` (the first-chunk wait elapsed),
-    ``build_error`` (opening the upstream stream failed), or ``upstream_error``
-    (the upstream raised before yielding a chunk). ``position`` is the attempt's
-    index in the resolved routing plan; label cardinality stays bounded by the
-    plan length.
-    """
-    ABANDONED_ATTEMPTS.labels(provider=provider, model=model, reason=reason, position=str(position)).inc()
-
-
-def record_inline_cost_settlement(outcome: str) -> None:
-    """Record an attached, unattached, or timed-out inline settlement."""
-    INLINE_COST_SETTLEMENTS.labels(outcome=outcome).inc()
-
-
-def record_rate_limit_hit() -> None:
-    """Record a rate limit hit."""
-    RATE_LIMIT_HITS.inc()
-
-
-def record_budget_exceeded() -> None:
-    """Record a budget exceeded event."""
-    BUDGET_EXCEEDED.inc()
-
-
-def record_auth_failure(reason: str) -> None:
-    """Record an authentication failure."""
-    AUTH_FAILURES.labels(reason=reason).inc()
-
-
-log_writer_queue_depth = LOG_WRITER_QUEUE_DEPTH
-log_writer_batch_size = LOG_WRITER_BATCH_SIZE
-log_writer_flush_duration = LOG_WRITER_FLUSH_DURATION
-log_writer_rows = LOG_WRITER_ROWS

src/gateway/rate_limit.py

@@ -7,7 +7,13 @@
 
 from fastapi import HTTPException, Request, status
 
-from gateway.metrics import record_rate_limit_hit
+from gateway.metrics import REGISTRY, Counter
+
+RATE_LIMIT_HITS = Counter(
+    "gateway_rate_limit_hits",
+    "Total number of rate limit hits",
+    registry=REGISTRY,
+)
 
 
 @dataclass
@@ -59,7 +65,7 @@ def check(self, user_id: str) -> RateLimitInfo:
         if len(timestamps) >= self._rpm:
             oldest = timestamps[0]
             retry_after = math.ceil(oldest - cutoff)
-            record_rate_limit_hit()
+            RATE_LIMIT_HITS.inc()
             raise HTTPException(
                 status_code=status.HTTP_429_TOO_MANY_REQUESTS,
                 detail="Rate limit exceeded",