diff --git a/.env.example b/.env.example index aba18bf..07b33db 100644 --- a/.env.example +++ b/.env.example @@ -304,3 +304,11 @@ STELLAR_RPC_MAX_KEYS=100 # Verify public-network endpoints resolve to a globally routable address (DNS) # before each request. Keep enabled; disable only for isolated networks. STELLAR_STRICT_HOST_VALIDATION=1 + +# Webhooks and event delivery (Phase 12, #674). +# WEBHOOK_MAX_PAYLOAD_BYTES bounds payload size to prevent unbounded memory buffering (default: 256 KB). +# WEBHOOK_OVERSIZED_POLICY defines behavior for oversized payloads: "reject" (default, HTTP 413) or "truncate". +WEBHOOK_MAX_PAYLOAD_BYTES=262144 +WEBHOOK_OVERSIZED_POLICY=reject +# Optional secret for incoming webhook validation via X-Webhook-Secret or Authorization header. +WEBHOOK_INGESTION_SECRET= diff --git a/app/__init__.py b/app/__init__.py index 8ea610b..2616308 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -101,6 +101,7 @@ def create_app(config_name: str | None = None) -> Flask: from app.reviews import bp as reviews_bp from app.stellar import bp as stellar_bp from app.tools import bp as tools_bp + from app.webhooks import bp as webhooks_bp from app.workspaces import bp as workspaces_bp app.register_blueprint(auth_bp) @@ -118,6 +119,7 @@ def create_app(config_name: str | None = None) -> Flask: app.register_blueprint(stellar_bp) app.register_blueprint(billing_bp) app.register_blueprint(admin_bp) + app.register_blueprint(webhooks_bp) # Register Socket.IO event handlers (import has side effects by design). from app.collaboration import realtime # noqa: F401 diff --git a/app/config.py b/app/config.py index 59d6177..2c2f747 100644 --- a/app/config.py +++ b/app/config.py @@ -381,6 +381,13 @@ class Config: # (defense-in-depth against SSRF through a misconfigured endpoint). STELLAR_STRICT_HOST_VALIDATION = os.getenv("STELLAR_STRICT_HOST_VALIDATION", "1") == "1" + # Webhooks and event delivery limits (Phase 12, #674). + # Bounds payload size to prevent unbounded memory buffering and DoS. + # Supported policies for oversized payloads: "reject" (default, fail closed) + # or "truncate" (explicit non-silent truncation of non-essential fields). + WEBHOOK_MAX_PAYLOAD_BYTES = _positive_int_env("WEBHOOK_MAX_PAYLOAD_BYTES", 256 * 1024) + WEBHOOK_OVERSIZED_POLICY = os.getenv("WEBHOOK_OVERSIZED_POLICY", "reject").strip().lower() + class DevelopmentConfig(Config): """Local development configuration.""" diff --git a/app/services/webhook_payloads.py b/app/services/webhook_payloads.py new file mode 100644 index 0000000..5eb4666 --- /dev/null +++ b/app/services/webhook_payloads.py @@ -0,0 +1,545 @@ +"""Webhook payload size limit and validation service (Phase 12, #674). + +Enforces deterministic payload bounds and handling policies for webhooks: +- Rejection policy (default): Oversized payloads fail closed with HTTP 413 + and structured error details. +- Truncation policy: Non-essential voluminous fields (such as diffs, patches, + or raw logs) are safely truncated with explicit metadata; required event + identity and envelope fields are NEVER silently truncated. +- Pre-buffering protection: Evaluates ``Content-Length`` headers and reads + streams in bounded chunks, preventing memory exhaustion before reading + oversized requests into memory. +- Privacy guarantee: Payloads, credentials, and authorization headers are + never leaked into server logs or unhandled exception traces. +""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Callable +from dataclasses import dataclass, field +from functools import wraps +from typing import Any + +logger = logging.getLogger(__name__) + +#: Default maximum webhook payload size in bytes (256 KB). +DEFAULT_WEBHOOK_MAX_PAYLOAD_BYTES = 256 * 1024 + +#: Default policy for handling payloads exceeding the size limit ("reject" or "truncate"). +DEFAULT_OVERSIZED_POLICY = "reject" +ALLOWED_OVERSIZED_POLICIES = frozenset({"reject", "truncate"}) + +#: Structural / envelope fields that must never be silently truncated. +REQUIRED_IMMUTABLE_FIELDS = frozenset( + { + "id", + "event", + "event_type", + "timestamp", + "workspace_id", + "user_id", + "version", + "delivery_id", + "signature", + } +) + +#: Known non-essential fields eligible for safe truncation under the truncate policy. +TRUNCATABLE_FIELD_NAMES = ("diff", "raw", "content", "output", "patch", "body", "findings", "items") + + +@dataclass(frozen=True) +class WebhookLimits: + """Resolved webhook limits and policies for the active context.""" + + max_payload_bytes: int + policy: str + + +@dataclass +class ValidationResult: + """Outcome of validating and parsing a webhook payload.""" + + is_valid: bool + payload: dict[str, Any] | None = None + raw_bytes: bytes | None = None + size: int = 0 + status_code: int = 200 + error: str | None = None + code: str | None = None + truncated: bool = False + truncated_fields: list[str] = field(default_factory=list) + + def to_error_response(self) -> dict[str, Any]: + """Return standardized structured error response dictionary.""" + response: dict[str, Any] = { + "error": self.error or "Webhook request failed validation.", + "code": self.code or "invalid_request", + } + if self.code == "payload_too_large": + response["limit_bytes"] = self.size + return response + + +class WebhookError(Exception): + """Base exception for webhook processing errors.""" + + def __init__(self, message: str, code: str = "webhook_error", status_code: int = 400): + super().__init__(message) + self.message = message + self.code = code + self.status_code = status_code + + +class WebhookPayloadTooLargeError(WebhookError): + """Raised when a payload exceeds the configured byte ceiling.""" + + def __init__( + self, + message: str, + limit_bytes: int, + actual_bytes: int, + code: str = "payload_too_large", + ): + super().__init__(message, code=code, status_code=413) + self.limit_bytes = limit_bytes + self.actual_bytes = actual_bytes + + +class WebhookMalformedPayloadError(WebhookError): + """Raised when the payload is not valid JSON.""" + + def __init__(self, message: str = "Malformed JSON payload.", code: str = "malformed_payload"): + super().__init__(message, code=code, status_code=400) + + +class WebhookInvalidPayloadError(WebhookError): + """Raised when required event fields are missing or invalid.""" + + def __init__(self, message: str, code: str = "invalid_payload"): + super().__init__(message, code=code, status_code=422) + + +class WebhookUnauthorizedError(WebhookError): + """Raised when webhook credentials or signatures are missing/invalid.""" + + def __init__(self, message: str = "Unauthorized webhook request.", code: str = "unauthorized"): + super().__init__(message, code=code, status_code=401) + + +class WebhookForbiddenError(WebhookError): + """Raised when the webhook subscriber lacks permissions for the event.""" + + def __init__(self, message: str = "Forbidden.", code: str = "forbidden"): + super().__init__(message, code=code, status_code=403) + + +def get_webhook_limits() -> WebhookLimits: + """Return effective webhook size limit and policy from current Flask app or defaults.""" + max_bytes = DEFAULT_WEBHOOK_MAX_PAYLOAD_BYTES + policy = DEFAULT_OVERSIZED_POLICY + + try: + from flask import current_app, has_app_context + + if has_app_context(): + configured_bytes = current_app.config.get("WEBHOOK_MAX_PAYLOAD_BYTES") + if configured_bytes is not None: + try: + parsed = int(configured_bytes) + if parsed > 0: + max_bytes = parsed + except (TypeError, ValueError): + pass + + configured_policy = current_app.config.get("WEBHOOK_OVERSIZED_POLICY") + if isinstance(configured_policy, str): + normalized = configured_policy.strip().lower() + if normalized in ALLOWED_OVERSIZED_POLICIES: + policy = normalized + except Exception: + pass + + return WebhookLimits(max_payload_bytes=max_bytes, policy=policy) + + +def read_webhook_stream( + stream: Any, + content_length: int | None = None, + max_bytes: int | None = None, + chunk_size: int = 8192, +) -> bytes: + """Read a request body stream enforcing size bounds before full buffering. + + Args: + stream: Byte stream (such as ``request.stream``). + content_length: Declared ``Content-Length`` from headers, if present. + max_bytes: Maximum allowed bytes. Defaults to configured limit. + chunk_size: Incremental read buffer size. + + Returns: + Full raw bytes of the request. + + Raises: + WebhookPayloadTooLargeError: If Content-Length or stream exceeds max_bytes. + """ + if max_bytes is None: + max_bytes = get_webhook_limits().max_payload_bytes + + # Pre-buffering defense: fail immediately if declared size exceeds maximum + if content_length is not None and content_length > max_bytes: + logger.warning( + "Rejecting webhook request: Content-Length %d exceeds limit %d bytes", + content_length, + max_bytes, + ) + raise WebhookPayloadTooLargeError( + f"Webhook payload length {content_length} bytes exceeds " + f"maximum limit of {max_bytes} bytes.", + limit_bytes=max_bytes, + actual_bytes=content_length, + ) + + chunks: list[bytes] = [] + total_read = 0 + + while True: + # Request read in chunks + chunk = stream.read(chunk_size) + if not chunk: + break + total_read += len(chunk) + if total_read > max_bytes: + logger.warning( + "Aborting webhook stream read: received %d bytes exceeding limit %d bytes", + total_read, + max_bytes, + ) + raise WebhookPayloadTooLargeError( + f"Webhook payload stream exceeded maximum limit of {max_bytes} bytes.", + limit_bytes=max_bytes, + actual_bytes=total_read, + ) + chunks.append(chunk) + + return b"".join(chunks) + + +def enforce_payload_size( + payload: dict[str, Any], + max_bytes: int | None = None, + policy: str | None = None, +) -> tuple[dict[str, Any], bool, list[str]]: + """Enforce size limit on a dictionary payload using the specified policy. + + Args: + payload: Event dictionary to evaluate. + max_bytes: Maximum allowed serialized byte size. + policy: "reject" or "truncate". Defaults to configured policy. + + Returns: + tuple (processed_payload, was_truncated, list_of_truncated_fields) + + Raises: + WebhookPayloadTooLargeError: If payload exceeds limit and policy is reject, + or if truncation cannot reduce size below limit without dropping required fields. + """ + limits = get_webhook_limits() + if max_bytes is None: + max_bytes = limits.max_payload_bytes + if policy is None: + policy = limits.policy + + # Calculate serialized UTF-8 size + raw_json = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + actual_size = len(raw_json) + + if actual_size <= max_bytes: + return payload, False, [] + + if policy != "truncate": + logger.warning( + "Webhook payload rejected: size %d bytes exceeds maximum limit " + "of %d bytes (policy: reject)", + actual_size, + max_bytes, + ) + raise WebhookPayloadTooLargeError( + f"Webhook payload size {actual_size} bytes exceeds maximum limit of {max_bytes} bytes.", + limit_bytes=max_bytes, + actual_bytes=actual_size, + ) + + # Safe Truncation Policy: + # 1. Required fields must NEVER be truncated silently. + # Extract immutable fields to verify they fit. + required_data: dict[str, Any] = { + k: v for k, v in payload.items() if k in REQUIRED_IMMUTABLE_FIELDS + } + required_bytes = len( + json.dumps(required_data, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ) + + if required_bytes >= max_bytes: + logger.warning( + "Webhook payload cannot be safely truncated: required fields alone consume " + "%d bytes >= limit %d bytes", + required_bytes, + max_bytes, + ) + raise WebhookPayloadTooLargeError( + f"Webhook required envelope fields consume {required_bytes} bytes, " + f"exceeding limit of {max_bytes} bytes.", + limit_bytes=max_bytes, + actual_bytes=actual_size, + ) + + # 2. Candidate fields for truncation + working_payload = dict(payload) + truncated_fields: list[str] = [] + + # Look inside 'data' or top-level for truncatable fields + data_dict = working_payload.get("data") + target_dicts: list[tuple[dict[str, Any], str]] = [] + if isinstance(data_dict, dict): + target_dicts.append((data_dict, "data.")) + target_dicts.append((working_payload, "")) + + for container, prefix in target_dicts: + for field_name in TRUNCATABLE_FIELD_NAMES: + if field_name in container and isinstance(container[field_name], (str, list, dict)): + val = container[field_name] + orig_len = len(val) + # Truncate string or list + if isinstance(val, str) and orig_len > 100: + # Truncate string with marker + container[field_name] = val[:100] + "... [truncated]" + truncated_fields.append(f"{prefix}{field_name}") + elif isinstance(val, list) and orig_len > 3: + container[field_name] = val[:3] + truncated_fields.append(f"{prefix}{field_name}") + elif isinstance(val, dict) and orig_len > 3: + truncated_keys = list(val.keys())[:3] + container[field_name] = {k: val[k] for k in truncated_keys} + truncated_fields.append(f"{prefix}{field_name}") + + current_bytes = len( + json.dumps(working_payload, separators=(",", ":"), ensure_ascii=False).encode( + "utf-8" + ) + ) + if current_bytes <= max_bytes: + break + current_bytes = len( + json.dumps(working_payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ) + if current_bytes <= max_bytes: + break + + # Re-check size after truncating non-essential fields + final_bytes = len( + json.dumps(working_payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ) + + if final_bytes > max_bytes: + # If still over limit, truncate larger values further + for container, prefix in target_dicts: + for k in list(container.keys()): + if ( + k not in REQUIRED_IMMUTABLE_FIELDS + and isinstance(container[k], str) + and len(container[k]) > 30 + ): + container[k] = container[k][:20] + "... [truncated]" + if f"{prefix}{k}" not in truncated_fields: + truncated_fields.append(f"{prefix}{k}") + current_bytes = len( + json.dumps( + working_payload, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + ) + if current_bytes <= max_bytes: + break + + final_bytes = len( + json.dumps(working_payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ) + if final_bytes > max_bytes: + raise WebhookPayloadTooLargeError( + f"Webhook payload remains oversized ({final_bytes} bytes) " + "after safe truncation attempt.", + limit_bytes=max_bytes, + actual_bytes=final_bytes, + ) + + # Documented truncation marker: never silent! + working_payload["_truncated"] = True + working_payload["_original_size"] = actual_size + working_payload["_truncated_fields"] = truncated_fields + + logger.info( + "Safely truncated oversized webhook payload from %d to %d bytes (fields: %s)", + actual_size, + final_bytes, + ", ".join(truncated_fields), + ) + return working_payload, True, truncated_fields + + +def validate_webhook_request( + request: Any, + max_bytes: int | None = None, + policy: str | None = None, + required_fields: list[str] | None = None, +) -> ValidationResult: + """Validate and read an incoming Flask webhook request with size limit enforcement. + + Args: + request: Flask request object. + max_bytes: Byte limit (falls back to configured limit). + policy: "reject" or "truncate". + required_fields: Expected top-level keys in JSON payload. + + Returns: + ValidationResult with parsed payload or structured error details. + """ + limits = get_webhook_limits() + if max_bytes is None: + max_bytes = limits.max_payload_bytes + if policy is None: + policy = limits.policy + + # When policy is "truncate", allow reading up to a bounded ingestion ceiling + # (up to 4x max_bytes or 2MB) so oversized non-essential fields can be received + # and safely truncated down to max_bytes. Under "reject" policy, stream fails fast at max_bytes. + stream_read_limit = max_bytes if policy != "truncate" else max(max_bytes * 4, 2 * 1024 * 1024) + + # 1. Read stream with pre-buffering check + try: + raw_bytes = read_webhook_stream( + request.stream, + content_length=request.content_length, + max_bytes=stream_read_limit, + ) + except WebhookPayloadTooLargeError as exc: + return ValidationResult( + is_valid=False, + size=exc.actual_bytes, + status_code=413, + error=exc.message, + code="payload_too_large", + ) + except Exception: + return ValidationResult( + is_valid=False, + status_code=400, + error="Failed to read request body stream.", + code="stream_read_error", + ) + + # 2. Check for empty body + if not raw_bytes: + return ValidationResult( + is_valid=False, + size=0, + status_code=400, + error="Empty webhook payload.", + code="empty_payload", + ) + + # 3. Parse JSON + try: + data = json.loads(raw_bytes.decode("utf-8")) + except Exception: + return ValidationResult( + is_valid=False, + raw_bytes=raw_bytes, + size=len(raw_bytes), + status_code=400, + error="Invalid JSON payload.", + code="malformed_payload", + ) + + if not isinstance(data, dict): + return ValidationResult( + is_valid=False, + raw_bytes=raw_bytes, + size=len(raw_bytes), + status_code=400, + error="Webhook payload must be a JSON object.", + code="malformed_payload", + ) + + # 4. Check required fields if specified + if required_fields: + for field_name in required_fields: + if field_name not in data or data[field_name] is None: + return ValidationResult( + is_valid=False, + payload=data, + raw_bytes=raw_bytes, + size=len(raw_bytes), + status_code=422, + error=f"Missing required payload field: '{field_name}'.", + code="invalid_payload", + ) + + # 5. Apply oversized policy to the payload + try: + processed, truncated, truncated_fields = enforce_payload_size( + data, + max_bytes=max_bytes, + policy=policy, + ) + return ValidationResult( + is_valid=True, + payload=processed, + raw_bytes=raw_bytes, + size=len(raw_bytes), + status_code=200, + truncated=truncated, + truncated_fields=truncated_fields, + ) + except WebhookPayloadTooLargeError as exc: + return ValidationResult( + is_valid=False, + payload=data, + raw_bytes=raw_bytes, + size=exc.actual_bytes, + status_code=413, + error=exc.message, + code="payload_too_large", + ) + + +def require_webhook_payload_limit( + max_bytes: int | None = None, + policy: str | None = None, + required_fields: list[str] | None = None, +) -> Callable: + """Flask view decorator enforcing webhook payload size limits and structured errors.""" + + def decorator(fn: Callable) -> Callable: + @wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + from flask import jsonify, request + + result = validate_webhook_request( + request, + max_bytes=max_bytes, + policy=policy, + required_fields=required_fields, + ) + if not result.is_valid: + return jsonify(result.to_error_response()), result.status_code + + # Pass parsed payload as kwarg 'payload' if accepted, or attach to request + request.webhook_payload = result.payload + request.webhook_validation = result + return fn(*args, **kwargs) + + return wrapper + + return decorator diff --git a/app/webhooks/__init__.py b/app/webhooks/__init__.py new file mode 100644 index 0000000..ba9ff27 --- /dev/null +++ b/app/webhooks/__init__.py @@ -0,0 +1,7 @@ +"""Webhooks blueprint (Phase 12, #674).""" + +from flask import Blueprint + +bp = Blueprint("webhooks", __name__, url_prefix="/api/webhooks") + +from app.webhooks import routes # noqa: E402, F401 diff --git a/app/webhooks/routes.py b/app/webhooks/routes.py new file mode 100644 index 0000000..c82c1fb --- /dev/null +++ b/app/webhooks/routes.py @@ -0,0 +1,98 @@ +"""Webhook ingestion and policy endpoints (Phase 12, #674).""" + +from __future__ import annotations + +import logging +from typing import Any + +from flask import current_app, jsonify, request + +from app.services.webhook_payloads import ( + ALLOWED_OVERSIZED_POLICIES, + get_webhook_limits, + validate_webhook_request, +) +from app.webhooks import bp + +logger = logging.getLogger(__name__) + + +@bp.route("/policy", methods=["GET"]) +def get_policy() -> tuple[Any, int]: + """Return the currently enforced webhook payload policy and size limits.""" + limits = get_webhook_limits() + return ( + jsonify( + { + "max_payload_bytes": limits.max_payload_bytes, + "policy": limits.policy, + "supported_policies": sorted(ALLOWED_OVERSIZED_POLICIES), + } + ), + 200, + ) + + +@bp.route("/events", methods=["POST"]) +def ingest_event() -> tuple[Any, int]: + """Ingest and validate an incoming webhook event. + + Enforces payload size limits, checks JSON structure, verifies required fields, + and returns consistent structured error responses. + """ + # 1. Authorization check: if a webhook secret is configured in the environment, + # require a matching X-Webhook-Secret or Authorization header. + configured_secret = current_app.config.get("WEBHOOK_INGESTION_SECRET") + if configured_secret: + auth_header = request.headers.get("X-Webhook-Secret") or request.headers.get( + "Authorization" + ) + if not auth_header: + return ( + jsonify( + {"error": "Missing webhook authentication credentials.", "code": "unauthorized"} + ), + 401, + ) + expected = ( + f"Bearer {configured_secret}" + if not auth_header.startswith("Bearer ") + else configured_secret + ) + if auth_header != configured_secret and auth_header != expected: + return ( + jsonify( + {"error": "Invalid webhook authentication credentials.", "code": "unauthorized"} + ), + 401, + ) + + # 2. Size limit enforcement & pre-buffering stream read + result = validate_webhook_request( + request, + required_fields=["event_type"], + ) + if not result.is_valid: + return jsonify(result.to_error_response()), result.status_code + + payload = result.payload or {} + event_type = payload.get("event_type") + + # Sanitized logging: never log full raw payload or sensitive tokens + logger.info( + "Accepted webhook event %s (size: %d bytes, truncated: %s)", + event_type, + result.size, + result.truncated, + ) + + response_data: dict[str, Any] = { + "status": "accepted", + "event_type": event_type, + "size_bytes": result.size, + } + if result.truncated: + response_data["truncated"] = True + response_data["truncated_fields"] = result.truncated_fields + + return jsonify(response_data), 202 diff --git a/docs/webhooks.md b/docs/webhooks.md new file mode 100644 index 0000000..1c313cc --- /dev/null +++ b/docs/webhooks.md @@ -0,0 +1,99 @@ +# Webhooks and Event Delivery Architecture + +This document describes the webhook payload size limits, oversized payload policies, pre-buffering defenses, and error response conventions for AI Code Assistant (Phase 12, #674). + +--- + +## 1. Overview + +AI Code Assistant delivers and ingests real-time events across projects, reviews, workspaces, and integrations. To prevent memory exhaustion attacks, unbounded buffering, and denial-of-service conditions, all webhook payloads are subject to strict byte-level size limits and pre-buffering enforcement. + +--- + +## 2. Configuration Settings + +Webhook payload bounds are environment-driven and can be tuned in `.env` or container settings: + +| Setting | Default | Description | +|---|---|---| +| `WEBHOOK_MAX_PAYLOAD_BYTES` | `262144` (256 KB) | Maximum allowed size of an event payload in bytes. | +| `WEBHOOK_OVERSIZED_POLICY` | `reject` | Handling policy when a payload exceeds the byte ceiling (`reject` or `truncate`). | +| `WEBHOOK_INGESTION_SECRET` | *(empty)* | Optional shared secret required via `X-Webhook-Secret` or `Authorization: Bearer `. | + +--- + +## 3. Oversized Payload Policies + +### 3.1 Rejection Policy (`reject`, Default) +The rejection policy is the recommended, fail-closed default. When any payload's serialized byte length exceeds `WEBHOOK_MAX_PAYLOAD_BYTES`: +- The request or delivery is immediately rejected. +- The HTTP server responds with status **413 Request Entity Too Large**. +- A structured error response is returned: + ```json + { + "error": "Webhook payload size 350000 bytes exceeds maximum limit of 262144 bytes.", + "code": "payload_too_large", + "limit_bytes": 262144 + } + ``` + +### 3.2 Safe Truncation Policy (`truncate`) +When the operator explicitly enables `WEBHOOK_OVERSIZED_POLICY=truncate`: +- **Core Envelope Invariance**: Structural and identity fields are **NEVER** silently truncated or dropped: + - `id` + - `event` + - `event_type` + - `timestamp` + - `workspace_id` + - `user_id` + - `version` + - `delivery_id` + - `signature` +- **Safe Truncation Targets**: Voluminous optional fields (such as `data.diff`, `data.raw`, `data.findings`, `data.output`, `data.patch`, `data.content`) are truncated to bring the total serialized size within the configured ceiling. +- **Explicit Truncation Markers**: Truncated payloads explicitly advertise metadata to ensure downstream consumers are aware of truncated fields: + ```json + { + "event_type": "project.updated", + "id": 42, + "timestamp": "2026-10-28T12:00:00Z", + "data": { + "diff": "diff --git a/... [truncated]" + }, + "_truncated": true, + "_original_size": 420000, + "_truncated_fields": ["data.diff"] + } + ``` +- **Fail-Closed Floor**: If the immutable required fields alone exceed `WEBHOOK_MAX_PAYLOAD_BYTES`, the system refuses to corrupt the event and fails closed with HTTP 413. + +--- + +## 4. Pre-Buffering Protection + +To prevent attackers from sending multi-gigabyte streams that exhaust server memory before validation occurs: +1. **Header Evaluation**: If the incoming HTTP request specifies a `Content-Length` header exceeding the permitted ceiling, the request is aborted immediately before reading bytes from the socket stream. +2. **Chunked Stream Bounds**: If `Content-Length` is omitted or chunked transfer encoding is used, the stream is read incrementally in 8 KB chunks. As soon as cumulative bytes exceed the maximum threshold, stream reading is halted and an HTTP 413 error is returned. + +--- + +## 5. Structured Error Responses + +All error responses adhere to the repository's API error convention: + +| Status Code | Error Code | Condition | +|---|---|---| +| `400 Bad Request` | `malformed_payload` | The request body is not valid JSON or is not a JSON object. | +| `400 Bad Request` | `empty_payload` | The request body is empty. | +| `401 Unauthorized` | `unauthorized` | Missing or invalid webhook secret or signature. | +| `403 Forbidden` | `forbidden` | Subscriber lacks capability or grant to receive the specified event. | +| `413 Payload Too Large` | `payload_too_large` | Payload exceeds `WEBHOOK_MAX_PAYLOAD_BYTES` under the `reject` policy. | +| `422 Unprocessable` | `invalid_payload` | Payload is valid JSON but lacks required fields (e.g. `event_type`). | + +--- + +## 6. Privacy and Sanitization + +Webhook logs and audit trails strictly sanitize all request contents: +- Raw request bodies and full payloads are never written to disk or logs. +- Webhook secrets, bearer tokens, and HMAC signatures are scrubbed. +- Logs capture only safe metadata: `event_type`, byte count, delivery outcome, and truncation status. diff --git a/tests/test_webhook_payload_limits.py b/tests/test_webhook_payload_limits.py new file mode 100644 index 0000000..b57a623 --- /dev/null +++ b/tests/test_webhook_payload_limits.py @@ -0,0 +1,240 @@ +"""Tests for webhook payload size limits, pre-buffering defense, and policies (Phase 12, #674).""" + +import io +import json +import logging + +import pytest + +from app import create_app +from app.services.webhook_payloads import ( + WebhookPayloadTooLargeError, + enforce_payload_size, + get_webhook_limits, + read_webhook_stream, +) + + +@pytest.fixture +def app(): + """Create test application context with testing config.""" + app = create_app("testing") + app.config["WEBHOOK_MAX_PAYLOAD_BYTES"] = 1024 # 1 KB limit for testing + app.config["WEBHOOK_OVERSIZED_POLICY"] = "reject" + return app + + +@pytest.fixture +def client(app): + return app.test_client() + + +class TestWebhookPayloadService: + """Unit tests for payload size enforcement service functions.""" + + def test_default_limits_fallback(self): + limits = get_webhook_limits() + assert limits.max_payload_bytes > 0 + assert limits.policy in {"reject", "truncate"} + + def test_under_limit_payload_passes(self): + payload = {"event_type": "project.created", "id": 1, "data": {"name": "test"}} + result, truncated, fields = enforce_payload_size(payload, max_bytes=1024, policy="reject") + assert result == payload + assert truncated is False + assert fields == [] + + def test_at_limit_payload_passes(self): + base_payload = {"event_type": "project.created", "id": 1, "padding": ""} + serialized_len = len(json.dumps(base_payload, separators=(",", ":")).encode("utf-8")) + padding_needed = 200 - serialized_len + base_payload["padding"] = "a" * padding_needed + + exact_bytes = len(json.dumps(base_payload, separators=(",", ":")).encode("utf-8")) + result, truncated, _ = enforce_payload_size( + base_payload, max_bytes=exact_bytes, policy="reject" + ) + assert result == base_payload + assert truncated is False + + def test_over_limit_payload_rejected_by_default(self): + payload = {"event_type": "project.created", "data": {"large_blob": "x" * 2000}} + with pytest.raises(WebhookPayloadTooLargeError) as exc_info: + enforce_payload_size(payload, max_bytes=500, policy="reject") + assert exc_info.value.status_code == 413 + assert exc_info.value.code == "payload_too_large" + assert exc_info.value.actual_bytes > 500 + + def test_over_limit_payload_safe_truncation(self): + payload = { + "event_type": "project.updated", + "id": 42, + "timestamp": "2026-10-28T00:00:00Z", + "data": { + "diff": "diff --git a/file.txt b/file.txt\n" + ("+line\n" * 500), + "author": "dev", + }, + } + truncated_payload, was_truncated, truncated_fields = enforce_payload_size( + payload, max_bytes=800, policy="truncate" + ) + assert was_truncated is True + assert "data.diff" in truncated_fields + # Required / immutable fields are preserved intact + assert truncated_payload["event_type"] == "project.updated" + assert truncated_payload["id"] == 42 + assert truncated_payload["timestamp"] == "2026-10-28T00:00:00Z" + assert truncated_payload["_truncated"] is True + assert truncated_payload["_original_size"] > 800 + + # Serialized size now fits within limit + final_size = len(json.dumps(truncated_payload, separators=(",", ":")).encode("utf-8")) + assert final_size <= 800 + + def test_truncation_fails_closed_if_required_fields_exceed_limit(self): + # Even under truncate policy, required fields are NEVER dropped or corrupted + large_required_id = "req-" + ("9" * 600) + payload = { + "event_type": "project.created", + "id": large_required_id, + "timestamp": "2026-10-28T00:00:00Z", + "data": {"diff": "short diff"}, + } + with pytest.raises(WebhookPayloadTooLargeError) as exc_info: + enforce_payload_size(payload, max_bytes=300, policy="truncate") + assert exc_info.value.status_code == 413 + assert "required" in exc_info.value.message.lower() + + def test_pre_buffering_rejects_content_length_immediately(self): + stream = io.BytesIO(b"dummy") + with pytest.raises(WebhookPayloadTooLargeError) as exc: + read_webhook_stream(stream, content_length=500000, max_bytes=1024) + assert exc.value.status_code == 413 + # Stream was not read + assert stream.tell() == 0 + + def test_stream_reading_aborts_when_exceeding_max_bytes(self): + stream = io.BytesIO(b"a" * 2048) + with pytest.raises(WebhookPayloadTooLargeError) as exc: + read_webhook_stream(stream, content_length=None, max_bytes=1024, chunk_size=512) + assert exc.value.status_code == 413 + assert exc.value.actual_bytes > 1024 + + +class TestWebhookEndpoints: + """Integration tests for webhook HTTP endpoints and boundary checks.""" + + def test_get_policy_endpoint(self, client): + resp = client.get("/api/webhooks/policy") + assert resp.status_code == 200 + data = resp.get_json() + assert data["max_payload_bytes"] == 1024 + assert data["policy"] == "reject" + assert "reject" in data["supported_policies"] + assert "truncate" in data["supported_policies"] + + def test_under_limit_request_accepted(self, client): + payload = {"event_type": "project.created", "project_id": 123} + resp = client.post("/api/webhooks/events", json=payload) + assert resp.status_code == 202 + data = resp.get_json() + assert data["status"] == "accepted" + assert data["event_type"] == "project.created" + + def test_over_limit_request_rejected_with_413(self, client): + payload = {"event_type": "project.created", "large_data": "z" * 2000} + resp = client.post("/api/webhooks/events", json=payload) + assert resp.status_code == 413 + data = resp.get_json() + assert data["code"] == "payload_too_large" + assert "exceeds" in data["error"].lower() + + def test_over_limit_request_with_truncate_policy(self, app, client): + app.config["WEBHOOK_OVERSIZED_POLICY"] = "truncate" + payload = { + "event_type": "review.completed", + "review_id": 99, + "data": { + "findings": "f" * 1500, + "summary": "completed", + }, + } + resp = client.post("/api/webhooks/events", json=payload) + assert resp.status_code == 202 + data = resp.get_json() + assert data["status"] == "accepted" + assert data["truncated"] is True + assert len(data["truncated_fields"]) > 0 + + def test_malformed_json_request(self, client): + resp = client.post( + "/api/webhooks/events", + data="{invalid_json:", + content_type="application/json", + ) + assert resp.status_code == 400 + data = resp.get_json() + assert data["code"] == "malformed_payload" + + def test_empty_request_body(self, client): + resp = client.post( + "/api/webhooks/events", + data="", + content_type="application/json", + ) + assert resp.status_code == 400 + data = resp.get_json() + assert data["code"] == "empty_payload" + + def test_missing_required_event_type(self, client): + resp = client.post( + "/api/webhooks/events", + json={"data": "no event_type key here"}, + ) + assert resp.status_code == 422 + data = resp.get_json() + assert data["code"] == "invalid_payload" + assert "event_type" in data["error"] + + def test_unauthorized_request_when_secret_configured(self, app, client): + app.config["WEBHOOK_INGESTION_SECRET"] = "super-secret-token" + + # Missing auth header -> 401 + resp = client.post( + "/api/webhooks/events", + json={"event_type": "project.created"}, + ) + assert resp.status_code == 401 + assert resp.get_json()["code"] == "unauthorized" + + # Wrong auth header -> 401 + resp2 = client.post( + "/api/webhooks/events", + json={"event_type": "project.created"}, + headers={"X-Webhook-Secret": "wrong-secret"}, + ) + assert resp2.status_code == 401 + assert resp2.get_json()["code"] == "unauthorized" + + # Valid auth header -> 202 + resp3 = client.post( + "/api/webhooks/events", + json={"event_type": "project.created"}, + headers={"X-Webhook-Secret": "super-secret-token"}, + ) + assert resp3.status_code == 202 + + def test_no_sensitive_data_in_logs(self, app, client, caplog): + app.config["WEBHOOK_INGESTION_SECRET"] = "top-secret-password-123" + secret_payload = {"event_type": "project.created", "user_password": "super-confidential"} + + with caplog.at_level(logging.INFO): + client.post( + "/api/webhooks/events", + json=secret_payload, + headers={"X-Webhook-Secret": "top-secret-password-123"}, + ) + + log_text = caplog.text + assert "top-secret-password-123" not in log_text + assert "super-confidential" not in log_text