From f14ba093ebae7334d8285e552e7bc9fdd6978746 Mon Sep 17 00:00:00 2001 From: Cameron Taylor <50385537+ct3685@users.noreply.github.com> Date: Mon, 4 May 2026 13:51:12 -0400 Subject: [PATCH 1/2] feat(guardrails): failure semantics, diagnostics, and UI parity (#1060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Implements configurable Fiddler guardrails **failure posture** (fail-open vs fail-closed), **observability-only (shadow)** mode, honest **degraded** health when the API cannot be evaluated, and full **config hierarchy** (environment → organization → chatflow). Adds **plan-tier awareness** so freemium Fiddler accounts no longer surface phantom 404 banners, plus a comprehensive UI polish pass for both light and dark themes. Closes the-answerai/theanswer#1059 ## What changed ### Runtime - New typed errors (`FiddlerAuthError`, `FiddlerUpstreamError`, `FiddlerTimeoutError`, `FiddlerNetworkError`, `FiddlerCircuitOpenError`, **`FiddlerUnsupportedError`**) replace silent fake-safe fallbacks. - Shared `runStage` helper centralizes fail-open / fail-closed branching across all 5 input/output call sites. - `FiddlerUnsupportedError` (HTTP 404 + "not supported by the freemium guardrails API") is treated as `ok: true, degraded: false` — does not block requests, does not show degraded banners, does not trip the circuit breaker. - Process-wide cache of unsupported endpoints keyed by `(sha256(apiKey), endpoint)` — freemium accounts pay the round-trip once per endpoint, and rotating to a paid key invalidates automatically. - Faithfulness payload corrected to Fiddler's spec: `{data: {prompt, response, context}}` (was `{data: {input, context}}` → 400). User prompt is plumbed through `runOutputStage` from `buildChatflow.ts` and `buildAgentflow.ts`; faithfulness is skipped (not thrown) when no prompt is available. - Org / Platform credential resolution: the runtime no longer filters strictly by `workspaceId`, so org-visible credentials reach sibling-workspace chatflows. - AgentFlow V2 output validation parity (was missing entirely; previously only chain and legacy multi-agent flows ran output validation). - Structured `logger.*` everywhere — failures land in Datadog / log aggregation. ### API - `GET /api/v1/guardrails/selftest` for operators — resolved config, credential source, live `healthCheck` latency/status, circuit-breaker state. - `FiddlerGuardrailsService.capabilityMatrix()` probes safety / PII / faithfulness independently. Selftest now returns a `capabilities` block plus human-readable `notes` when PII or faithfulness aren't on the connected plan. ### Admin UI (Guardrails Settings) - MasterConfig: failure-mode toggle, observability-only switch, **Plan capabilities** chip row (Safety / PII / Faithfulness — green available, amber not-included, red degraded) with a Recheck button. - Credential change (select-from-dropdown **and** add-new) is now async-aware — `onSave` is awaited before the capabilities `useEffect` re-fetches, so the chip row reflects the new key immediately instead of one click later. Optimistic local state with rollback on save failure. - SimpleMode: Strict / Balanced / Lenient presets now set `failureMode` and `observabilityOnly` so degraded behavior matches preset intent. - AdvancedMode: per-stage Enable switches, sliders, sub-tabs, and accordion summaries restyled for theme parity via shared `sx` constants. - Comprehensive light / dark theme pass: - Outlined action buttons (Edit / Change / Recheck) anchor on `text.primary` + `divider` instead of translucent `primary.main` (were nearly invisible in dark mode). - Page-level switches (Enable Guardrails, Observability-only) and section switches (Safety, PII, Faithfulness) use `info.main` (Material blue) when on — same look in both themes. - Selected toggle button (Fail open / Fail closed) uses `action.selected` + `text.primary` for legible contrast in dark mode. - Selected preset cards use `action.selected` + theme-aware border (was a malformed rgba opacity suffix). - Sliders, inner tabs, accordion summaries restyled with shared `sx` constants for consistent rhythm. - Save Configuration buttons now show a solid disabled state (the global `contained` gradient was bleeding through the disabled state). ### Chatflow UI - Per-chatflow `failureMode` override in chatflow guardrails dialog. - Per-chatflow **Observability-only (override)** switch mirrors org admin controls. - Saves merge partial updates and re-read `chatbotConfig` from the save response (no more dropped fields on partial saves). ### Chat message UI - Amber degraded-state banner renders only on real outages (auth, timeout, upstream) — never on plan-tier `unsupported`. - In-accordion Health detail block (mode, per-stage reason, HTTP status, latency). - "✓ Clean" rendering for Input / Output Validation sections when no violations / redactions / faithfulness signals exist (previously left an empty section under the header). - `reason: "unsupported"` rendered as **"Not included in Fiddler plan (skipped silently)"** instead of generic `api_error`. ### Docs - `packages/docs/docs/integrations/fiddler.mdx` — new "Plan Tiers (Freemium vs Paid)" section, updated `selftest` example with `capabilities` and `notes`, refreshed troubleshooting FAQ. - `CHANGELOG.md` — full Features / Bug Fixes / Internal / Plan-Tier Awareness entries under `[Unreleased]`. - `.alphaAgent/spec/env-vars.json` — new `FIDDLER_FAILURE_MODE`, `FIDDLER_OBSERVABILITY_ONLY` registered. - `apps/web/e2e/tests/GUARDRAILS_TESTS.md` — E2E coverage notes. ## Schema impact - New `health` field on `chat_message.guardrails_metadata` JSON column — **no DB migration** (reuses existing TEXT column). ## Tests - New `packages/server/test/services/guardrails/index.test.ts` — 8 passing unit tests covering `FiddlerUnsupportedError` handling, the unsupported-endpoint cache, circuit-breaker interaction, and the corrected faithfulness payload. ## Testing checklist - [ ] `/api/v1/guardrails/selftest` returns a `capabilities` block reflecting connected plan tier - [ ] Freemium key: chat works; no amber banner; admin chip row shows amber on PII / Faithfulness - [ ] Paid key: all three capability chips green; safety / PII / faithfulness all run end-to-end - [ ] Failure mode = `closed` + bad API key → chat blocked with clear message - [ ] Failure mode = `open` + bad API key → chat delivered with amber degraded banner - [ ] Observability-only ON → real violations recorded in `guardrailsMetadata`, request not blocked - [ ] Per-chatflow override beats org default for both `failureMode` and `observabilityOnly` - [ ] Change credential in admin → Plan capabilities row refreshes without manual Recheck - [ ] Light + dark theme: outlined buttons, switches, sliders, preset cards, toggle group all legible --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- .alphaAgent/spec/env-vars.json | 35 + .claude/plans/fiddler-guardrails-spec.md | 5 +- .claude/plans/fiddler-guardrails-status.md | 94 +- .claude/skills/error-handling.md | 148 ++-- CHANGELOG.md | 40 + apps/web/e2e/tests/GUARDRAILS_TESTS.md | 44 +- .../ui/src/GuardrailsSettings.tsx | 5 +- .../src/GuardrailsSettings/AdvancedMode.tsx | 154 +++- .../src/GuardrailsSettings/MasterConfig.tsx | 438 +++++++++- .../ui/src/GuardrailsSettings/SimpleMode.tsx | 138 ++- packages-answers/ui/src/Message/Message.tsx | 351 ++++++-- .../ui/src/theme/components/muiOverrides.ts | 16 + packages/components/src/Interface.ts | 1 + packages/docs/docs/integrations/fiddler.mdx | 149 +++- packages/server/src/Interface.ts | 37 + .../src/controllers/guardrails/index.ts | 37 + .../server/src/routes/guardrails/index.ts | 13 + packages/server/src/routes/index.ts | 2 + .../guardrails/FiddlerGuardrailsService.ts | 811 +++++++++++------- .../server/src/services/guardrails/config.ts | 31 +- .../server/src/services/guardrails/errors.ts | 120 +++ .../src/services/guardrails/extractContext.ts | 22 + .../server/src/services/guardrails/index.ts | 6 +- .../src/services/guardrails/runStage.ts | 329 +++++++ .../src/services/guardrails/selftest.ts | 140 +++ packages/server/src/types/guardrails.ts | 62 ++ packages/server/src/utils/buildAgentflow.ts | 69 +- packages/server/src/utils/buildChatflow.ts | 300 +++---- .../test/services/guardrails/index.test.ts | 226 +++++ packages/ui/src/api/guardrails.js | 17 +- .../extended/ChatflowGuardrails.jsx | 86 +- .../views/organizations/guardrails/presets.ts | 12 +- 32 files changed, 3186 insertions(+), 752 deletions(-) create mode 100644 packages/server/src/controllers/guardrails/index.ts create mode 100644 packages/server/src/routes/guardrails/index.ts create mode 100644 packages/server/src/services/guardrails/errors.ts create mode 100644 packages/server/src/services/guardrails/extractContext.ts create mode 100644 packages/server/src/services/guardrails/runStage.ts create mode 100644 packages/server/src/services/guardrails/selftest.ts create mode 100644 packages/server/test/services/guardrails/index.test.ts diff --git a/.alphaAgent/spec/env-vars.json b/.alphaAgent/spec/env-vars.json index 05931ff808a..30fb583a45e 100644 --- a/.alphaAgent/spec/env-vars.json +++ b/.alphaAgent/spec/env-vars.json @@ -624,6 +624,23 @@ "sensitive": true, "category": "integrations" }, + { + "name": "FIDDLER_API_KEY", + "description": "Fiddler Guardrails API key (priority-3 fallback when no workspace/org credential is configured)", + "required": false, + "example": "fiddler_api_key_...", + "sensitive": true, + "category": "guardrails" + }, + { + "name": "FIDDLER_API_URL", + "description": "Fiddler Guardrails API base URL (used together with FIDDLER_API_KEY env fallback)", + "required": false, + "default": "https://guardrails.cloud.fiddler.ai", + "example": "https://guardrails.cloud.fiddler.ai", + "sensitive": false, + "category": "guardrails" + }, { "name": "FIDDLER_GUARDRAILS_ENABLED", "description": "Enable Fiddler AI guardrails for LLM safety", @@ -633,6 +650,24 @@ "sensitive": false, "category": "guardrails" }, + { + "name": "FIDDLER_FAILURE_MODE", + "description": "Default failure posture when guardrails cannot be evaluated. 'open' allows the request and surfaces a degraded-state banner; 'closed' returns HTTP 503. Org/chatflow config can override.", + "required": false, + "default": "open", + "example": "open", + "sensitive": false, + "category": "guardrails" + }, + { + "name": "FIDDLER_OBSERVABILITY_ONLY", + "description": "Shadow/pilot mode. When 'true', real violations are recorded in metadata but never block (use to calibrate thresholds before flipping enforcement on in production).", + "required": false, + "default": "false", + "example": "false", + "sensitive": false, + "category": "guardrails" + }, { "name": "FIDDLER_SAFETY_ENABLED", "description": "Enable safety checks (harmful, violent, etc.)", diff --git a/.claude/plans/fiddler-guardrails-spec.md b/.claude/plans/fiddler-guardrails-spec.md index 576d7b3057b..43bb65c7a64 100644 --- a/.claude/plans/fiddler-guardrails-spec.md +++ b/.claude/plans/fiddler-guardrails-spec.md @@ -2,11 +2,14 @@ **Linear**: AGENT-139 **Last Updated**: 2025-11-11 +**Phase 7 Amendment**: 2026-04-24 — see "Phase 7 Amendment" section below for current failure semantics > **📖 Document Purpose:** Technical requirements, API formats, schemas, and acceptance criteria (source of truth for WHAT to build). > **📊 For Implementation Status:** See [fiddler-guardrails-status.md](fiddler-guardrails-status.md) for progress, gaps, and completion time. > **🔨 For Build Instructions:** See [fiddler-guardrails-implementation.md](fiddler-guardrails-implementation.md) for step-by-step implementation guide. +> **⚠️ Phase 7 Amendment (2026-04-24, issue [#1059](https://github.com/the-answerai/theanswer/issues/1059)):** The original "Fail-open by default" design principle was correct as a default, but the implementation made every failure indistinguishable from "content was safe" with no admin signal — silently degrading enforcement to telemetry under any dependency failure. The current implementation makes failure posture configurable as `failureMode: 'open' | 'closed'` (env → org → chatflow hierarchy, default `'open'`), surfaces degraded states in chat UI + structured logs + metadata, exposes a diagnostic `/api/v1/guardrails/selftest` endpoint, adds AgentFlow V2 output parity, and adds an `observabilityOnly` shadow mode for safe pilots. Full details in [fiddler-guardrails-status.md → Phase 7](fiddler-guardrails-status.md#phase-7-failure-semantics--diagnostics). + --- ## Executive Summary @@ -21,7 +24,7 @@ Integrate Fiddler AI Guardrails into AnswerAgent to automatically validate LLM i **Key Design Principles**: - **3-tier configuration**: environment → organization → chatflow (deep merge with per-dimension/per-type overrides) - **Performance**: <150ms p95 latency with caching (<10ms on cache hit) -- **Reliability**: Fail-open by default, circuit breaker protection +- **Reliability**: Configurable failure posture (default fail-open, opt-in fail-closed), circuit breaker protection — see Phase 7 Amendment above - **Multi-tenancy**: Organization-scoped credentials and configurations - **Flexibility**: Per-dimension safety thresholds, per-type PII confidence and actions diff --git a/.claude/plans/fiddler-guardrails-status.md b/.claude/plans/fiddler-guardrails-status.md index bde7cf17abf..51b4401b48f 100644 --- a/.claude/plans/fiddler-guardrails-status.md +++ b/.claude/plans/fiddler-guardrails-status.md @@ -1,9 +1,9 @@ # Fiddler Guardrails - Implementation Status **Linear:** AGENT-139 -**Last Updated:** 2025-11-11 -**Branch:** `feature/AGENT-139-phase-3-api-simple-ui` -**PR:** #686 (staging) +**Last Updated:** 2026-04-24 +**Branch:** `1059-fiddler-guardrails-fail-semantics` (Phase 7) +**Issue:** [the-answerai/theanswer#1059](https://github.com/the-answerai/theanswer/issues/1059) > **📋 Document Purpose:** Track implementation progress, critical gaps, and time to completion. > **📖 For Technical Details:** See [fiddler-guardrails-spec.md](fiddler-guardrails-spec.md) for requirements and schemas. @@ -19,26 +19,21 @@ | **Phase 2: Input Validation** | ✅ COMPLETE | 8/8 (100%) | _(in Phase 1)_ | 5/5 | | **Phase 3: API & Simple UI** | ✅ COMPLETE | 8/8 (100%) | 171 | 4/5 | | **Phase 4: Advanced Config** | ✅ COMPLETE | 6/6 (100%) | 1,104 | 3/5 | -| **Phase 5: Output Validation** | ⏸️ DEPRIORITIZED | 0/7 (0%) | 0 | 3/5 | +| **Phase 5: Output Validation** | ✅ COMPLETE | 7/7 (100%) | _(folded into Phase 7)_ | 3/5 | | **Phase 6: Chatflow Overrides** | ✅ COMPLETE | 2/2 (100%) | 312 | 2/5 | +| **Phase 7: Failure Semantics & Diagnostics** | ✅ COMPLETE | 13/13 (100%) | ~1,500 | 4/5 | **MVP Progress:** 30/30 tasks (100%) - Phases 1-4, 6 complete -**Total Progress (with Phase 5):** 32/37 tasks (86.5%) -**Code Verified:** 3,028 lines core implementation + ~1,800 supporting code +**Total Progress:** 52/52 tasks (100%) +**Code Verified:** 3,028 lines core implementation + ~1,800 supporting code + Phase 7 additions --- ## 🔴 Remaining Issues (Non-Blocking) -### 1. **Credential Fallback Missing** - Priority: HIGH +### 1. **Credential Fallback Missing** — ✅ RESOLVED in Phase 7 -**Location:** `packages/server/src/utils/buildChatflow.ts:286-294` - -**Problem:** No fallback to `FIDDLER_API_KEY` env var when org credential not configured - -**Fix:** Add env var check before skipping validation - -**Estimated Effort:** 30 minutes +**Resolution (2026-04-24):** `FiddlerGuardrailsService.loadCredentials` now resolves credentials in a 4-tier priority order ending with the `FIDDLER_API_KEY`/`FIDDLER_API_URL` env vars. Also fixed a separate bug where `Organization`/`Platform`-visible credentials were not resolved across workspaces in the same org. --- @@ -54,26 +49,29 @@ --- -### 3. **Logging Not Comprehensive** - Priority: MEDIUM -- ✅ Has: Basic console.warn -- ❌ Missing: Structured logging with dimension scores, PII types, actions taken +### 3. **Logging Not Comprehensive** — ✅ RESOLVED in Phase 7 -**Estimated Effort:** 1 hour +**Resolution (2026-04-24):** All `console.warn`/`console.error` calls in `FiddlerGuardrailsService` and the new shared `runStage` helper replaced with structured `logger.*`. Per-stage `[Guardrails] Stage degraded`, `[Guardrails] Stage unavailable`, and `[Guardrails] Input validation passed`/`Output validation completed` log lines include dimension scores, PII types, `reason`, `httpStatus`, `latencyMs`, and `failureMode` for downstream alerting. --- ### 4. **No Unit Tests** - Priority: LOW - ❌ 0% test coverage -- ❌ Missing tests for: FiddlerGuardrailsService, config hierarchy, CircuitBreaker +- ❌ Missing tests for: FiddlerGuardrailsService, config hierarchy, CircuitBreaker, runStage, selftest endpoint **Estimated Effort:** 8-10 hours (post-launch acceptable) --- -### 5. **Environment Variables Incomplete** - Priority: LOW -- ❌ Missing in `.env.template`: Circuit breaker config, cache config +### 5. **Environment Variables Incomplete** — ✅ RESOLVED in Phase 7 -**Estimated Effort:** 15 minutes +**Resolution (2026-04-24):** `.alphaAgent/spec/env-vars.json` now registers `FIDDLER_API_KEY`, `FIDDLER_API_URL`, `FIDDLER_FAILURE_MODE`, and `FIDDLER_OBSERVABILITY_ONLY`. Pre-existing entries (circuit breaker, cache, thresholds, actions) were already present and verified. + +--- + +### 6. **Langfuse Span Attachment** — DEFERRED + +The Phase 7 implementation emits structured `logger.*` calls and persists health metadata on `chat_message.guardrails_metadata`, but does not yet attach `guardrails.input` / `guardrails.output` spans to the parent Langfuse trace. Tracked as a follow-up to Phase 7; low effort once existing analytic-handlers integration is touched again. --- @@ -291,5 +289,53 @@ try { --- -**Status:** MVP Ready (100% of required features complete, Phase 5 output validation deprioritized) -**Next Review:** Post-launch feedback +**Status:** Phase 7 complete — failure semantics, observability surfaces, and AgentFlow V2 output parity all shipped on branch `1059-fiddler-guardrails-fail-semantics`. MVP plus enterprise-grade enforcement posture. + +**Next Review:** Post-merge soak in staging; Langfuse span attachment as follow-up. + +--- + +## Phase 7: Failure Semantics & Diagnostics + +**Trigger:** Audit finding (issue [#1059](https://github.com/the-answerai/theanswer/issues/1059)) — every failure path in the existing implementation produced behavior indistinguishable from "content was safe", silently degrading enforcement to telemetry whenever Fiddler was unreachable, the API key was rotated, the circuit was open, or an org was multi-workspace. + +**Scope:** + +1. Make failure posture configurable per environment / org / chatflow (`failureMode: 'open' | 'closed'`). +2. Replace silent fallbacks in `FiddlerGuardrailsService` with typed errors so callers can distinguish "API said safe" from "API could not be evaluated". +3. Add operator-visible degraded-state surfaces: amber chat banner, structured logs, in-accordion Health detail, Langfuse-ready metadata shape. +4. Close the AgentFlow V2 output-validation gap (was previously zero coverage). +5. Add a diagnostic `/api/v1/guardrails/selftest` admin endpoint. +6. Add observability-only shadow mode for safe pilot rollouts. +7. Fix multi-workspace credential resolution (Organization/Platform visibility). + +**Files added:** + +- `packages/server/src/services/guardrails/errors.ts` — typed error classes + `toFiddlerError` classifier +- `packages/server/src/services/guardrails/runStage.ts` — shared input/output stage helper owning fail-open/closed branching +- `packages/server/src/services/guardrails/extractContext.ts` — RAG source-doc extraction (lifted out of buildChatflow.ts so AgentFlow V2 can share) +- `packages/server/src/services/guardrails/selftest.ts` — diagnostic report builder +- `packages/server/src/controllers/guardrails/index.ts` — selftest controller +- `packages/server/src/routes/guardrails/index.ts` — `/api/v1/guardrails/selftest` mount + +**Files modified:** + +- `packages/server/src/types/guardrails.ts` — `failureMode`, `observabilityOnly`, `GuardrailStageStatus`, `GuardrailHealthReason`, `status` field on validation results +- `packages/server/src/Interface.ts` — `health` block on `GuardrailsMetadata`; `guardrailsMetadata` propagated through `IExecuteFlowParams` +- `packages/components/src/Interface.ts` — `streamErrorEvent` added to `IServerSideEventStreamer` interface +- `packages/server/src/services/guardrails/config.ts` — `FIDDLER_FAILURE_MODE`, `FIDDLER_OBSERVABILITY_ONLY` env support; merged in `deepMergeConfigs` +- `packages/server/src/services/guardrails/FiddlerGuardrailsService.ts` — full refactor (typed errors, structured logger, honest fallbacks, status threading, visibility-aware credential resolution) +- `packages/server/src/services/guardrails/index.ts` — barrel re-exports +- `packages/server/src/utils/buildChatflow.ts` — three call sites converted to `runStage`; fail-closed throws 503 via `buildFailClosedError` +- `packages/server/src/utils/buildAgentflow.ts` — output validation added to `executeAgentFlow` (V2 parity) +- `packages-answers/ui/src/Message/Message.tsx` — amber degraded banner + Health section in accordion + shield-icon warning state +- `packages-answers/ui/src/GuardrailsSettings/MasterConfig.tsx` — Failure Mode `ToggleButtonGroup` + Observability-Only switch +- `packages/ui/src/ui-component/extended/ChatflowGuardrails.jsx` — per-chatflow Failure Mode override (empty inherits org) + +**Backward compatibility:** + +- `failureMode` defaults to `'open'`; existing deployments see no behavior change +- No DB migration; reuses existing `chat_message.guardrails_metadata` TEXT column +- Public service methods (`createFromContext`, `validateInput`, `validateOutput`) keep their existing signatures; new `status` field is additive + +**Code verified:** ~1,500 lines across 6 new files + 12 modified files. Server `tsc --noEmit` clean. All ReadLints clean. diff --git a/.claude/skills/error-handling.md b/.claude/skills/error-handling.md index 3ebed5d0726..46cf7f17ca6 100644 --- a/.claude/skills/error-handling.md +++ b/.claude/skills/error-handling.md @@ -48,7 +48,9 @@ Error: chatflowService.getChatflowById - Chatflow abc123 not found // Service errors 'Error: chatflowService.getChatflowById - Chatflow abc123 not found' 'Error: resourceService.createResource - Failed to save resource' -'Error: FiddlerGuardrailsService.post - Connection failed' +// Note: FiddlerGuardrailsService now throws typed FiddlerError subclasses +// (FiddlerAuthError, FiddlerUpstreamError, etc.) rather than InternalFlowiseError — +// see Section 9 for the rationale. ``` ## 3. Complete StatusCodes Reference @@ -201,74 +203,90 @@ try { } ``` -## 8. Fail-Open Pattern +## 8. Configurable Fail-Open / Fail-Closed Pattern -Returns null/default on error, allowing main request to complete. Used for optional features: +> **Updated 2026-04-24 (Phase 7, issue #1059):** The original "fail-open at all costs" pattern was a defect, not a feature — it converted enforcement layers into telemetry under any dependency failure. The correct pattern below classifies failures with typed errors, threads a `status` field through every evaluation, and lets each call site (or the resolved org/chatflow `failureMode`) decide whether to allow or block. + +**Service-level resolution returns a structured report, not just null:** ```typescript // Location: FiddlerGuardrailsService -public static async initialize(user: IUser): Promise { +public static async resolveFromContext( + chatflowId: string, + workspaceId: string, + organizationId?: string +): Promise<{ + service: FiddlerGuardrailsService | null + config: GuardrailsConfig + credentialSource: FiddlerCredentialSource + reason: 'ok' | 'disabled' | 'no_credentials' | 'unexpected' + error?: Error +}> { + let config: GuardrailsConfig try { - const config = await this.loadConfig(user.organizationId!) - if (!config.enabled) { - return null - } + config = await getGuardrailsConfig(chatflowId, organizationId) + } catch (error) { + logger.error('[Guardrails] Failed to load config', { error, chatflowId, workspaceId }) + return { service: null, config: { enabled: false } as GuardrailsConfig, credentialSource: 'none', reason: 'unexpected', error: error as Error } + } - const credentials = await this.loadCredentials(user.organizationId!, config) - if (!credentials) { - console.warn(`Guardrails enabled but no credentials found`) - return null - } + if (!config.enabled) { + return { service: null, config, credentialSource: 'none', reason: 'disabled' } + } - return new FiddlerGuardrailsService(credentials, config) - } catch (error) { - // Fail-open: log error but don't throw - console.error('Error initializing Fiddler Guardrails (fail-open):', error) - return null + const { credentials, source } = await this.loadCredentials(workspaceId, organizationId, config) + if (!credentials) { + logger.warn('[Guardrails] Enabled but no credentials could be resolved', { chatflowId, workspaceId, organizationId }) + return { service: null, config, credentialSource: 'none', reason: 'no_credentials' } } + + return { service: new FiddlerGuardrailsService(credentials, config), config, credentialSource: source, reason: 'ok' } } ``` -**Input validation fail-open:** +**Validate methods return `{result, status}` instead of swallowing errors:** + ```typescript public async validateInput(text: string): Promise { - try { - const [safetyResult, piiResult] = await Promise.all([ - this.evaluateSafety(text), - this.detectPII(text) - ]) - // ... process results - } catch (error) { - // Fail-open: on error, allow the input to pass through - return { - safetyResult: { dimensions: [], violations: [], isUnsafe: false }, - piiResult: { detections: [], hasPII: false }, - blocked: false, - redacted: false, - violations: {} - } + const [safety, pii] = await Promise.all([this.evaluateSafety(text), this.detectPII(text)]) + + return { + safetyResult: safety.result, + piiResult: pii.result, + blocked: /* per-dimension/per-type block actions */, + redacted: pii.result.redactedText !== undefined, + violations: { safety: safety.result.violations, pii: pii.result.detections }, + // status.degraded === true when ANY sub-stage could not reach Fiddler. + // Caller (runStage) decides fail-open vs fail-closed based on resolved config. + status: mergeStageStatus([safety.status, pii.status]) } } ``` -## 9. Circuit Breaker Pattern +**Caller uses a shared `runStage` helper to apply the policy:** -Prevents cascading failures with external services: +```typescript +// Location: services/guardrails/runStage.ts +const stage = await runInputStage(question, { chatflowId, workspaceId, organizationId, chatId }) + +if (stage.kind === 'blocked_degraded') { + // failureMode === 'closed' AND something degraded + throw buildFailClosedError(stage.reason) // HTTP 503 +} +if (stage.kind === 'blocked_violation') { + throw new InternalFlowiseError(StatusCodes.BAD_REQUEST, stage.message) // real 400 +} +// kind === 'ok' — request continues. stage.health.degraded surfaces in chat banner if true. +``` + +## 9. Circuit Breaker Pattern (with typed errors) + +> **Updated 2026-04-24 (Phase 7, issue #1059):** The original implementation accepted a `fallback` parameter that returned fake-safe results (zero scores, empty PII detections). This made "API down" indistinguishable from "API said safe". The current pattern throws typed `FiddlerError` subclasses; callers catch them and record a `degraded` status with a stable machine-readable `reason` enum. ```typescript -private async executeWithCircuitBreaker( - operation: () => Promise, - fallback?: () => T -): Promise { - // Check if circuit allows execution +private async executeWithCircuitBreaker(operation: () => Promise): Promise { if (!this.circuitBreaker.canExecute()) { - if (fallback) { - return fallback() - } - throw new InternalFlowiseError( - StatusCodes.SERVICE_UNAVAILABLE, - 'Error: FiddlerGuardrailsService - Circuit breaker is open' - ) + throw new FiddlerCircuitOpenError() // reason: 'circuit_open' } try { @@ -277,27 +295,31 @@ private async executeWithCircuitBreaker( return result } catch (error) { this.circuitBreaker.recordFailure() - - // Use fallback if available - if (fallback) { - return fallback() - } - throw error + throw toFiddlerError(error) // classifies into auth_error | api_error | timeout | network_error } } ``` -**Usage with fallback:** +**Caller catches the typed error and produces a `degraded` status:** + ```typescript -const apiResponse = await this.executeWithCircuitBreaker( - async () => { - return await this.post('/v3/guardrails/ftl-safety', data) - }, - (): SafetyAPIResponse => { - // Fallback returns safe defaults - return { fdl_harmful: 0, fdl_violent: 0 /* ... */ } +public async evaluateSafety(text: string): Promise<{ result: ...; status: GuardrailStageStatus }> { + if (!this.config.safety.enabled) { + return { result: { ... }, status: DISABLED_STATUS } } -) + const startedAt = Date.now() + try { + const apiResponse = await this.executeWithCircuitBreaker(async () => { + return await this.post('/v3/guardrails/ftl-safety', { data: { input: text } }) + }) + return { result: { ...apiResponse, violations, isUnsafe: violations.length > 0 }, status: { ...HEALTHY_STATUS, latencyMs: Date.now() - startedAt } } + } catch (error) { + const fiddlerErr = error instanceof FiddlerError ? error : toFiddlerError(error) + // Honest fallback: no violations *because we could not evaluate*, NOT because content was safe. + // Caller must consult status.degraded before trusting result. + return { result: { dimensions: [], violations: [], isUnsafe: false }, status: toStatus(fiddlerErr, startedAt) } + } +} ``` ## 10. Promise.all Error Handling diff --git a/CHANGELOG.md b/CHANGELOG.md index ef1343248c3..c1538b4df79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [Unreleased] + +### Features + +* **guardrails:** add configurable fail-open / fail-closed failure semantics with full env → org → chatflow hierarchy, defaulting to `open` for backward compatibility ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** add observability-only (shadow) mode that records real Fiddler violations without blocking, for safe pilot rollouts ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** add `GET /api/v1/guardrails/selftest` diagnostic endpoint that reports resolved config, credential source, live `healthCheck` latency/status, and circuit-breaker state ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** add Failure Mode toggle and Observability-Only switch to admin Guardrails Settings; add per-chatflow Failure Mode override in chatflow guardrails dialog ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** simple presets (Strict / Balanced / Lenient) now set `failureMode` and `observabilityOnly` so degraded-outage behavior matches preset intent; per-chatflow **Observability-only (override)** switch mirrors org admin controls ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** add amber degraded-state banner above chat messages when safety checks could not be evaluated, with plain-English reason text and an in-accordion Health detail block (mode, per-stage reason, HTTP status, latency) ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** add output validation parity for AgentFlow V2 chatflows (was missing entirely; previously only chain and legacy multi-agent flows ran output validation) ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) + +### Bug Fixes + +* **guardrails:** correctly resolve credentials with `Organization` or `Platform` visibility; previously the runtime filtered strictly by `workspaceId` so a single org-visible credential was invisible to chatflows running in a sibling workspace, silently disabling guardrails ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** stop returning fake-safe results on circuit-open / API errors; previously every failure was indistinguishable from "content was safe", which silently degraded enforcement to telemetry under any dependency failure. Failures now surface as a `degraded` health status that callers act on per `failureMode`. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** replace all `console.warn` / `console.error` calls in the guardrails service with structured `logger.*` so failures appear in Datadog/log aggregation streams ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** admin and chatflow guardrails UIs no longer drop `failureMode` / `observabilityOnly` in React state on partial saves: org page applies the merged config returned by the API; Simple/Advanced saves merge onto existing config; chatflow saves merge partial updates and re-read `chatbotConfig` from the save response ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) + +### Internal + +* **guardrails:** typed errors (`FiddlerAuthError`, `FiddlerUpstreamError`, `FiddlerTimeoutError`, `FiddlerNetworkError`, `FiddlerCircuitOpenError`) replace silent fallbacks; new shared `runStage` helper centralizes fail-open/fail-closed branching across all 5 input/output call sites +* **guardrails:** new env vars `FIDDLER_FAILURE_MODE`, `FIDDLER_OBSERVABILITY_ONLY` (registered in `.alphaAgent/spec/env-vars.json`) +* **guardrails:** new `health` field on `chat_message.guardrails_metadata` JSON column (no DB migration; reuses existing TEXT column) + +### Plan-Tier Awareness (Freemium vs Paid Fiddler) + +* **guardrails:** new `FiddlerUnsupportedError` distinguishes "Fiddler API said this guardrail is not in your plan" (HTTP 404 + `not supported by the freemium guardrails API`) from a real outage. Plan-tier 404s are now classified `reason: "unsupported"` and treated as `ok: true, degraded: false` — they do not block requests, do not show a degraded banner to end users, and do not trip the circuit breaker. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** unsupported endpoints are cached process-wide per `(sha256(apiKey), endpoint)` so a freemium account does not pay a network round-trip per chat for an endpoint Fiddler will never serve. Cache is keyed by hashed API key so flipping a credential to a paid plan invalidates the prior decision automatically. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** `FiddlerGuardrailsService.capabilityMatrix()` probes safety, PII, and faithfulness independently and returns per-endpoint `GuardrailStageStatus`. The `/api/v1/guardrails/selftest` response now exposes a `capabilities` block plus human-readable `notes` when PII or faithfulness are unavailable on the connected plan. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** admin Guardrails Settings page renders a "Plan capabilities" chip row under the connected credential card (Safety / PII / Faithfulness — green available, amber not-included, red degraded) with a Recheck button. Operators see plan limitations once, up front, instead of via a recurring per-chat banner. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** validation accordion on chat messages now describes `reason: "unsupported"` as "Not included in Fiddler plan (skipped silently)" instead of generic `api_error`. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) + +### Bug Fixes (continued) + +* **guardrails:** fix Fiddler faithfulness 400 Bad Request — payload was being sent as `{data: {input, context}}`. Fiddler's `/v3/guardrails/ftl-response-faithfulness` requires `{data: {prompt, response, context}}`. Plumbed user prompt through `runOutputStage` from both `buildChatflow.ts` and `buildAgentflow.ts`. Faithfulness checks are now skipped (rather than thrown) when no prompt is available. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) +* **guardrails:** stop showing an amber "Safety checks were unavailable (HTTP 404)" banner to end users on freemium Fiddler plans. The 404 was a plan-tier capability gap, not a runtime outage; runtime now treats it as `unsupported` and keeps the chat banner-free. ([#1059](https://github.com/the-answerai/theanswer/issues/1059)) + +--- + ## [2.2.9](https://github.com/the-answerai/theanswer/compare/v2.2.8...v2.2.9) (2026-02-04) ### Bug Fixes diff --git a/apps/web/e2e/tests/GUARDRAILS_TESTS.md b/apps/web/e2e/tests/GUARDRAILS_TESTS.md index a5e513d2497..5e4b1290414 100644 --- a/apps/web/e2e/tests/GUARDRAILS_TESTS.md +++ b/apps/web/e2e/tests/GUARDRAILS_TESTS.md @@ -331,12 +331,54 @@ jobs: --- +## Pending Test Coverage (Phase 7) + +The Phase 7 work (issue [#1059](https://github.com/the-answerai/theanswer/issues/1059)) added new behavior that is **not yet covered** by E2E tests. When extending this spec file, add coverage for: + +### Failure Mode (fail-open vs fail-closed) +- ✅ Admin can toggle Failure Mode in Master Config (Fail Open / Fail Closed) +- ✅ Toggle round-trips through `PUT /api/v1/organizations/:id/config/guardrails` and persists `failureMode` +- ✅ Per-chatflow Failure Mode override appears in Chatflow Configuration > Guardrails tab when override is enabled +- ✅ Per-chatflow override correctly inherits org default when left untouched + +### Observability-Only Mode +- ✅ Admin can toggle Observability-Only switch +- ✅ When enabled, real Fiddler block-action violations show up in `guardrailsMetadata` but the chat still completes (no 400) + +### Diagnostic Selftest Endpoint +- ✅ `GET /api/v1/guardrails/selftest` returns 200 with `healthCheck.ok=true` when guardrails healthy +- ✅ Returns 200 with `healthCheck.degraded=true, reason='no_credentials'` when no credential resolves +- ✅ Returns 200 with `healthCheck.reason='auth_error'` when credential is rejected by Fiddler +- ✅ Returns 200 with `healthCheck.reason='disabled'` when guardrails are disabled by config +- ✅ `credentialSource` reflects actual provenance (`workspace`, `organization`, `env`, etc.) + +### Degraded-State UI +- ✅ Amber "Safety checks were unavailable for this message" banner renders above chat response when `health.anyDegraded=true` +- ✅ Banner does NOT render for healthy validations +- ✅ Shield icon turns amber (warning state) when degraded +- ✅ Accordion shows new "Health" section with `mode`, per-stage `reason`, `httpStatus`, `latencyMs` +- ✅ `[BUG] Jira-style` integration: simulated upstream 401 produces both the chat banner AND a `logger.error('[Guardrails] Stage degraded ...')` line in server logs + +### Fail-Closed Blocking +- ✅ With `failureMode='closed'` and a known-bad API key, prediction returns HTTP 503 with body `Safety checks are temporarily unavailable (reason: auth_error). Please try again in a moment.` +- ✅ Streaming chatflow on fail-closed + degraded emits a trailing SSE `error` event so client UI can surface it + +### AgentFlow V2 Parity +- ✅ AgentFlow V2 chatflow (`type === 'AGENTFLOW'`) emits the same `guardrailsMetadata` shape as chain chatflows (input + output + health) +- ✅ AgentFlow V2 input violation produces the same 400 block as chain path + +### Credential Visibility +- ✅ `Organization`-visible `fiddlerApi` credential created in workspace A is correctly resolved by chatflows running in workspace B of the same org (was a bug, now fixed) + +--- + ## Related Documentation - **Implementation Plan:** `.claude/plans/fiddler-guardrails-implementation.md` -- **Progress Tracking:** `.claude/plans/fiddler-guardrails-progress.md` +- **Progress Tracking:** `.claude/plans/fiddler-guardrails-status.md` - **Backend API:** `packages/server/CLAUDE.md` - **Frontend Components:** `packages-answers/ui/src/GuardrailsSettings.tsx` +- **Issue:** [the-answerai/theanswer#1059](https://github.com/the-answerai/theanswer/issues/1059) — Phase 7 failure semantics + diagnostics - **Playwright Docs:** https://playwright.dev/ --- diff --git a/packages-answers/ui/src/GuardrailsSettings.tsx b/packages-answers/ui/src/GuardrailsSettings.tsx index 5e7c58fdbfc..11437c2be24 100644 --- a/packages-answers/ui/src/GuardrailsSettings.tsx +++ b/packages-answers/ui/src/GuardrailsSettings.tsx @@ -67,8 +67,9 @@ export default function GuardrailsSettings({ organizationId }: { organizationId: try { setSaving(true) setError(null) - await guardrailsApi.updateGuardrailsConfig(organizationId, newConfig) - setConfig(newConfig) + const response = await guardrailsApi.updateGuardrailsConfig(organizationId, newConfig) + // API returns the deep-merged guardrails payload; use it so failureMode / observabilityOnly are not lost in UI after partial saves + setConfig(response?.data != null ? response.data : newConfig) setSuccess(true) setTimeout(() => setSuccess(false), 3000) router.refresh() diff --git a/packages-answers/ui/src/GuardrailsSettings/AdvancedMode.tsx b/packages-answers/ui/src/GuardrailsSettings/AdvancedMode.tsx index d247c304d0c..511376a2312 100644 --- a/packages-answers/ui/src/GuardrailsSettings/AdvancedMode.tsx +++ b/packages-answers/ui/src/GuardrailsSettings/AdvancedMode.tsx @@ -71,6 +71,95 @@ const PII_TYPES = [ { key: 'AGE', label: 'Age Information', confidenceRec: '0.85-0.95', actionRec: 'warn' } ] as const +// Shared sx for the inner Simple/Per-Dimension and Simple/Per-Type tabs. +// The AnswerAI dark theme has `primary.main` defined as a translucent white, +// so MUI's default Tab indicator (`bgcolor: primary.main`) renders invisible. +// We override the indicator + selected text to use `text.primary` for clear +// visibility in both modes. +const innerTabsSx = { + mb: 2, + borderBottom: 1, + borderColor: 'divider', + minHeight: 36, + '& .MuiTabs-indicator': { + backgroundColor: 'text.primary', + height: 2 + }, + '& .MuiTab-root': { + textTransform: 'none' as const, + fontWeight: 500, + minHeight: 36, + color: 'text.secondary', + '&.Mui-selected': { + color: 'text.primary', + fontWeight: 600 + }, + '&:hover': { + color: 'text.primary' + } + } +} + +// Shared sx for sliders. Default MUI Slider uses `primary.main` for the rail, +// track and thumb — all invisible in the AnswerAI dark theme. Anchor on +// `text.primary` so the slider track is visible regardless of mode. +const sliderSx = { + color: 'text.primary', + '& .MuiSlider-rail': { + opacity: 0.32 + }, + '& .MuiSlider-track': { + border: 'none' + }, + '& .MuiSlider-thumb': { + boxShadow: 'none', + '&:hover, &.Mui-focusVisible': { + boxShadow: '0 0 0 6px rgba(127, 127, 127, 0.16)' + } + }, + '& .MuiSlider-mark': { + backgroundColor: 'text.secondary', + opacity: 0.6 + }, + '& .MuiSlider-markLabel': { + color: 'text.secondary', + fontSize: 12 + } +} + +// Shared sx for the per-section Enable switches (Safety, PII, Faithfulness). +// The default MUI Switch reads as grey-on-grey in the AnswerAI dark theme +// because the active state resolves to translucent `primary.main`. Anchor +// checked thumb + track on `info.main` (Material Blue, shared across modes) +// so "on" is unmistakable and consistent with the page-level switches. +const sectionSwitchSx = { + '& .MuiSwitch-switchBase.Mui-checked': { + color: 'info.main', + '& + .MuiSwitch-track': { + backgroundColor: 'info.main', + opacity: 0.5 + } + }, + '& .MuiSwitch-switchBase.Mui-checked:hover': { + backgroundColor: 'rgba(33, 150, 243, 0.08)' + } +} + +// Shared sx for the AccordionSummary section heads ("Input Validation", +// "Output Validation", "Advanced Settings"). Aligns with the subtitle2 + +// fontWeight 600 rhythm used elsewhere on the page and gives the summary a +// subtle hover affordance that works in both themes. +const sectionSummarySx = { + px: 1, + borderRadius: 1, + '&:hover': { + bgcolor: 'action.hover' + }, + '& .MuiAccordionSummary-content': { + my: 1.5 + } +} + export default function AdvancedMode({ config, onSave, saving, error, success, onClearError }: AdvancedModeProps) { // State for all configuration const [enabled, setEnabled] = useState(config?.enabled ?? false) @@ -150,7 +239,8 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o } } - await onSave(newConfig) + // Merge onto existing config so failureMode, observabilityOnly, and other untouched fields are preserved (esp. chatflow overrides) + await onSave({ ...config, ...newConfig }) setHasChanges(false) } @@ -263,8 +353,10 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o <> {/* Input Validation */} - } sx={{ p: 0 }}> - Input Validation + } sx={sectionSummarySx}> + + Input Validation + {/* Safety Check */} @@ -282,6 +374,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o setSafetyEnabled(e.target.checked) markChanged() }} + sx={sectionSwitchSx} /> } label='Enable' @@ -290,11 +383,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o {safetyEnabled && ( <> - setSafetyView(value)} - sx={{ mb: 2, borderBottom: 1, borderColor: 'divider' }} - > + setSafetyView(value)} sx={innerTabsSx}> @@ -324,6 +413,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o { value: 1, label: '1.0' } ]} valueLabelDisplay='auto' + sx={sliderSx} /> @@ -415,6 +505,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o step={0.01} valueLabelDisplay='auto' size='small' + sx={sliderSx} /> @@ -466,6 +557,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o setPiiEnabled(e.target.checked) markChanged() }} + sx={sectionSwitchSx} /> } label='Enable' @@ -474,11 +566,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o {piiEnabled && ( <> - setPiiView(value)} - sx={{ mb: 2, borderBottom: 1, borderColor: 'divider' }} - > + setPiiView(value)} sx={innerTabsSx}> @@ -507,6 +595,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o { value: 1, label: '1.0' } ]} valueLabelDisplay='auto' + sx={sliderSx} /> @@ -602,6 +691,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o step={0.01} valueLabelDisplay='auto' size='small' + sx={sliderSx} /> @@ -653,8 +743,10 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o {/* Output Validation */} - } sx={{ p: 0 }}> - Output Validation + } sx={sectionSummarySx}> + + Output Validation + @@ -677,6 +769,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o setFaithfulnessEnabled(e.target.checked) markChanged() }} + sx={sectionSwitchSx} /> } label='Enable' @@ -707,6 +800,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o { value: 0.1, label: '0.1' } ]} valueLabelDisplay='auto' + sx={sliderSx} /> @@ -738,8 +832,10 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o {/* Advanced Settings */} - } sx={{ p: 0 }}> - Advanced Settings + } sx={sectionSummarySx}> + + Advanced Settings + @@ -765,6 +861,7 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o step={1} marks valueLabelDisplay='auto' + sx={sliderSx} /> @@ -775,7 +872,28 @@ export default function AdvancedMode({ config, onSave, saving, error, success, o {/* Actions */} - {hasChanges && ( diff --git a/packages-answers/ui/src/GuardrailsSettings/MasterConfig.tsx b/packages-answers/ui/src/GuardrailsSettings/MasterConfig.tsx index 01f77e28c3b..c7590d38be9 100644 --- a/packages-answers/ui/src/GuardrailsSettings/MasterConfig.tsx +++ b/packages-answers/ui/src/GuardrailsSettings/MasterConfig.tsx @@ -10,10 +10,26 @@ import FormControlLabel from '@mui/material/FormControlLabel' import Button from '@mui/material/Button' import Alert from '@mui/material/Alert' import CircularProgress from '@mui/material/CircularProgress' +import ToggleButton from '@mui/material/ToggleButton' +import ToggleButtonGroup from '@mui/material/ToggleButtonGroup' +import Chip from '@mui/material/Chip' +import Tooltip from '@mui/material/Tooltip' import credentialsApi from 'flowise-ui/src/api/credentials' +import guardrailsApi from 'flowise-ui/src/api/guardrails' import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackbarAction } from 'flowise-ui/src/store/actions' import useConfirm from 'flowise-ui/src/hooks/useConfirm' -import { IconX, IconUnlink, IconEdit, IconShieldCheck } from '@tabler/icons-react' +import { + IconX, + IconUnlink, + IconEdit, + IconShieldCheck, + IconRefresh, + IconCheck, + IconMinus, + IconAlertTriangle, + IconCircleDot, + IconLock +} from '@tabler/icons-react' // Use core Flowise dialog with defaultVisibility prop for org-wide credentials const AddEditCredentialDialog = dynamic(() => import('flowise-ui/src/views/credentials/AddEditCredentialDialog'), { ssr: false }) @@ -21,9 +37,45 @@ const ConfirmDialog = dynamic(() => import('flowise-ui/src/ui-component/dialog/C const FIDDLER_CREDENTIAL_NAME = 'fiddlerApi' +// Shared sx for "neutral" outlined action buttons (Edit, Change, Recheck). +// MUI's default outlined Button anchors text + border on `primary.main`, +// which in the AnswerAI dark theme resolves to rgba(255,255,255,0.12) — +// translucent white. The buttons end up reading as if they were disabled +// even when fully enabled. Anchor on `text.primary` + `divider` so they +// have clear contrast in both modes; the actual `disabled` state is left +// to MUI's default treatment so it still reads as inert when applicable. +const outlinedActionSx = { + color: 'text.primary', + borderColor: 'divider', + '&:hover': { + borderColor: 'text.primary', + bgcolor: 'action.hover' + } +} + +// Shared sx for the page switches ("Enable Guardrails", "Observability-only"). +// The default MUI `color='primary'` resolves to translucent white in the +// AnswerAI dark theme, making the on-state nearly invisible. Anchor the +// checked state on `info.main` (Material Blue, shared across light/dark) so +// the on-state reads unambiguously in both modes. +const pageSwitchSx = { + '& .MuiSwitch-switchBase.Mui-checked': { + color: 'info.main', + '& + .MuiSwitch-track': { + backgroundColor: 'info.main', + opacity: 0.5 + } + }, + '& .MuiSwitch-switchBase.Mui-checked:hover': { + backgroundColor: 'rgba(33, 150, 243, 0.08)' + } +} + interface GuardrailConfig { enabled?: boolean credentialId?: string + failureMode?: 'open' | 'closed' + observabilityOnly?: boolean } interface CredentialDialogProps { @@ -38,7 +90,11 @@ interface CredentialDialogProps { interface MasterConfigProps { config: GuardrailConfig onConfigChange: (updates: Partial) => void - onSave: (config: GuardrailConfig) => void + // Returns a promise so callers can await persistence before triggering + // server-side reads (e.g. the capability selftest, which reads the org- + // stored credentialId — racing it against the in-flight PATCH returned + // stale plan capabilities until the user manually clicked Recheck). + onSave: (config: GuardrailConfig) => Promise | void } interface Credential { @@ -47,12 +103,39 @@ interface Credential { credentialName: string } +interface CapabilityStatus { + ok: boolean + degraded: boolean + reason: string + httpStatus?: number + message?: string + latencyMs?: number +} + +interface SelftestReport { + capabilities?: { + safety: CapabilityStatus + pii: CapabilityStatus + faithfulness: CapabilityStatus + } + notes?: string[] +} + export default function MasterConfig({ config, onConfigChange, onSave }: MasterConfigProps) { const [enabled, setEnabled] = useState(config?.enabled ?? false) const [selectedCredential, setSelectedCredential] = useState(config?.credentialId ?? '') + const [failureMode, setFailureMode] = useState<'open' | 'closed'>(config?.failureMode ?? 'open') + const [observabilityOnly, setObservabilityOnly] = useState(config?.observabilityOnly ?? false) const [credentials, setCredentials] = useState([]) const [loadingCredentials, setLoadingCredentials] = useState(true) + // Capability matrix from /api/v1/guardrails/selftest. Tells the admin which + // guardrail endpoints their Fiddler plan tier actually includes so they + // don't enable e.g. PII and then see "(HTTP 404)" banners on every chat. + const [capabilities, setCapabilities] = useState(null) + const [loadingCapabilities, setLoadingCapabilities] = useState(false) + const [capabilityError, setCapabilityError] = useState(null) + // Credential modal state const [showCredentialDialog, setShowCredentialDialog] = useState(false) const [credentialDialogProps, setCredentialDialogProps] = useState>({}) @@ -89,6 +172,8 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC useEffect(() => { setEnabled(config?.enabled ?? false) setSelectedCredential(config?.credentialId ?? '') + setFailureMode(config?.failureMode ?? 'open') + setObservabilityOnly(config?.observabilityOnly ?? false) }, [config]) const loadCredentials = async () => { @@ -104,12 +189,63 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC } } + /** + * Fetch the per-endpoint capability matrix. Only meaningful when guardrails + * are enabled and a credential is connected — otherwise the server returns + * `disabled` / `no_credentials` and the UI hides the section. + */ + const loadCapabilities = async () => { + try { + setLoadingCapabilities(true) + setCapabilityError(null) + const response = await guardrailsApi.getSelftest() + setCapabilities(response?.data?.capabilities || null) + } catch (error: any) { + setCapabilityError(error?.response?.data?.message || error?.message || 'Could not run capability check') + setCapabilities(null) + } finally { + setLoadingCapabilities(false) + } + } + + // Run the capability probe whenever the connected credential or the + // enabled flag changes. A new credential could be on a different Fiddler + // plan tier; a disable/enable transition resets the status. + useEffect(() => { + if (!enabled) { + setCapabilities(null) + setCapabilityError(null) + return + } + if (!config?.credentialId) { + setCapabilities(null) + setCapabilityError(null) + return + } + loadCapabilities() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [enabled, config?.credentialId]) + const handleEnabledChange = (event: React.ChangeEvent) => { const newEnabled = event.target.checked setEnabled(newEnabled) onConfigChange({ enabled: newEnabled }) } + const handleFailureModeChange = (_: unknown, value: 'open' | 'closed' | null) => { + if (!value) return + setFailureMode(value) + onConfigChange({ failureMode: value }) + onSave({ ...config, failureMode: value }) + } + + const handleObservabilityOnlyChange = (event: React.ChangeEvent) => { + const next = event.target.checked + setObservabilityOnly(next) + onConfigChange({ observabilityOnly: next }) + onSave({ ...config, observabilityOnly: next }) + } + const handleCreateCredential = async () => { try { // Load the Fiddler credential component schema @@ -135,18 +271,28 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC } } - const handleCredentialDialogConfirm = (credentialId: string) => { + const handleCredentialDialogConfirm = async (credentialId: string) => { setShowCredentialDialog(false) // Refresh credentials list loadCredentials() - // For new credentials, auto-select and auto-enable guardrails + // For new credentials, auto-select and auto-enable guardrails. + // Same async-save discipline as the "Change credential" handler: + // skip onConfigChange and await onSave so the capability selftest + // probes the persisted credential, not the still-old org config. if (credentialId && credentialDialogProps.type === 'ADD') { + const previousCredentialId = selectedCredential + const previousEnabled = enabled setSelectedCredential(credentialId) setEnabled(true) - onConfigChange({ credentialId: credentialId, enabled: true }) - onSave({ ...config, credentialId, enabled: true }) + try { + await Promise.resolve(onSave({ ...config, credentialId, enabled: true })) + } catch (err) { + setSelectedCredential(previousCredentialId) + setEnabled(previousEnabled) + console.error('Failed to attach new Fiddler credential:', err) + } } } @@ -210,7 +356,7 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC {/* Enable/Disable Toggle */} } + control={} label='Enable Guardrails' sx={{ mb: 2 }} /> @@ -265,10 +411,10 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC {/* Show Change when user owns any credentials they could switch to */} {credentials.length > 0 && ( @@ -334,6 +480,7 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC disabled={!enabled || editLoading} onClick={handleEditCredential} startIcon={editLoading ? : } + sx={outlinedActionSx} > Edit @@ -349,11 +496,11 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC {/* > 1 because the currently selected credential doesn't count as an alternative */} {credentials.length > 1 && ( @@ -384,6 +531,109 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC )} + {/* Plan capability matrix. + Surfaces which guardrails the connected Fiddler key actually + supports so admins don't enable e.g. PII detection on a + freemium plan and then see (HTTP 404) banners on every chat. */} + {enabled && config?.credentialId && ( + + + + Plan capabilities + + + + {capabilityError ? ( + + {capabilityError} + + ) : capabilities ? ( + + {(['safety', 'pii', 'faithfulness'] as const).map((key) => { + const status = capabilities[key] + const label = key === 'pii' ? 'PII' : key.charAt(0).toUpperCase() + key.slice(1) + let color: 'success' | 'warning' | 'error' | 'default' = 'default' + let Icon = IconCircleDot + let tooltip: string = `Reason: ${status?.reason || 'unknown'}` + if (status?.reason === 'ok') { + color = 'success' + Icon = IconCheck + tooltip = `Available · ${status.latencyMs ?? 0}ms` + } else if (status?.reason === 'unsupported') { + color = 'warning' + Icon = IconMinus + tooltip = 'Not included in your Fiddler plan; runtime will skip silently.' + } else if (status?.degraded) { + color = 'error' + Icon = IconAlertTriangle + tooltip = `${status.reason}${status.httpStatus ? ` · HTTP ${status.httpStatus}` : ''}${ + status.message ? ` · ${status.message}` : '' + }` + } + return ( + + } + label={label} + sx={(theme) => { + if (color === 'default') return {} + const palette = theme.palette[color] + return { + fontWeight: 500, + borderColor: palette.main, + color: palette.main, + bgcolor: theme.palette.mode === 'dark' ? `${palette.main}1A` : `${palette.main}14`, + '& .MuiChip-icon': { color: palette.main } + } + }} + /> + + ) + })} + + ) : loadingCapabilities ? ( + + + + Probing Fiddler endpoints… + + + ) : ( + + Capability check has not run yet. + + )} + {capabilities && + (capabilities.pii.reason === 'unsupported' || capabilities.faithfulness.reason === 'unsupported') && ( + + Amber endpoints are not included in your Fiddler plan and will be skipped silently at runtime — no + per-message warning banners. + + )} + + )} + {/* Dropdown for selecting existing credentials (collapsed by default) */} {showCredentialDropdown && credentials.length > 0 && ( @@ -395,22 +645,57 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC { + // Same translucent-primary trap as the preset cards: anchor + // the selected state on `text.primary` border + `action.selected` + // bg so the chosen credential reads clearly in both modes. + const isDark = theme.palette.mode === 'dark' + const isSelected = selectedCredential === cred.id + return { + p: 1.5, + cursor: 'pointer', + border: isSelected ? '2px solid' : '1px solid', + borderColor: isSelected + ? isDark + ? 'rgba(255, 255, 255, 0.45)' + : 'rgba(15, 23, 42, 0.5)' + : 'divider', + ...(isSelected && { bgcolor: 'action.selected' }), + '&:hover': { + borderColor: isSelected + ? isDark + ? 'rgba(255, 255, 255, 0.6)' + : 'rgba(15, 23, 42, 0.7)' + : 'text.secondary', + bgcolor: isSelected ? 'action.selected' : 'action.hover' + } } }} - onClick={() => { + onClick={async () => { + setShowCredentialDropdown(false) + // Optimistically reflect the new credential in the local + // "Connected as" header so the UI feels instant. + const previousCredentialId = selectedCredential + const previousEnabled = enabled setSelectedCredential(cred.id) setEnabled(true) - onConfigChange({ credentialId: cred.id, enabled: true }) - onSave({ ...config, credentialId: cred.id, enabled: true }) - setShowCredentialDropdown(false) + // CRITICAL: do NOT call onConfigChange here. That would update + // the parent's config.credentialId synchronously, fire the + // capabilities useEffect immediately, and probe the selftest + // endpoint while the credential PATCH is still in flight — + // returning stale plan-capability data until the user manually + // clicked Recheck. Instead, await the save so the parent's + // post-PATCH setConfig() drives the recheck against the now- + // persisted credentialId. + try { + await Promise.resolve(onSave({ ...config, credentialId: cred.id, enabled: true })) + } catch (err) { + // handleSave already surfaces the error via parent state. + // Roll back optimistic local state so the UI matches reality. + setSelectedCredential(previousCredentialId) + setEnabled(previousEnabled) + console.error('Failed to switch Fiddler credential:', err) + } }} > {cred.name} @@ -421,6 +706,113 @@ export default function MasterConfig({ config, onConfigChange, onSave }: MasterC )} + {/* Failure behavior — runtime semantics when Fiddler is unavailable + or when an admin wants pilot-mode visibility before enforcing. + Wrapped in a single card so the failure-mode + observability + controls feel like one logical group, matching the credential + card and capability card above for a consistent visual rhythm. */} + + + Failure behavior + + + What the runtime does when Fiddler can't be reached (bad key, outage, timeout) and how violations are surfaced. + Separate from threshold strictness in presets below. + + + {/* Fail-open vs fail-closed switch */} + + When Fiddler is unavailable + + { + // The AnswerAI theme defines `primary.{main,light,dark}` as + // translucent whites/slates rather than a vivid color, so anchoring + // the selected state on `primary.*` produced ghosted text in dark + // mode. We use MUI's mode-balanced `action.selected` token + a + // visible border + `text.primary` for guaranteed contrast in both + // modes — same pattern used for menu/table selection across the app. + const isDark = theme.palette.mode === 'dark' + return { + '& .MuiToggleButton-root': { + color: 'text.primary', + borderColor: 'divider', + textTransform: 'none', + fontWeight: 500, + px: 1.5, + transition: 'background-color 120ms ease, border-color 120ms ease' + }, + '& .MuiToggleButton-root:hover': { + bgcolor: 'action.hover' + }, + '& .MuiToggleButton-root.Mui-selected': { + bgcolor: 'action.selected', + color: 'text.primary', + fontWeight: 600, + borderColor: isDark ? 'rgba(255, 255, 255, 0.45)' : 'rgba(15, 23, 42, 0.5)', + '&:hover': { + bgcolor: isDark ? 'rgba(255, 255, 255, 0.22)' : 'rgba(15, 23, 42, 0.12)' + } + } + } + }} + > + + + Fail open (allow, warn) + + + + Fail closed (block, 503) + + + + {failureMode === 'closed' + ? 'Block the request with HTTP 503 when checks can\u2019t be run. Higher safety, lower availability.' + : 'Deliver the message with a warning banner when checks can\u2019t be run. Higher availability, lower safety.'} + + + {/* Shadow pilot toggle. Violations are recorded but never block. */} + + + } + label={ + + Observability-only (shadow mode) + + } + /> + + Real violations are recorded in message metadata but never block the request. Use before flipping enforcement on in + production. + + + + {/* Credential Modal - uses core dialog with defaultVisibility enhancement */} { const presetConfig = getPresetConfig(selectedPreset) if (presetConfig) { + // Merge preset onto current config so failureMode, observabilityOnly, and chatflow-only fields are not dropped await onSave({ + ...config, ...presetConfig, enabled: config?.enabled ?? true, - credentialId: config?.credentialId // Preserve credential from master config + credentialId: config?.credentialId }) setHasChanges(false) } @@ -75,11 +77,12 @@ export default function SimpleMode({ config, onSave, saving, error, success, onC return ( - {/* Preset Selection (only shown when enabled) */} - + {/* Preset Selection (only shown when enabled). + Uses subtitle2 to match the visual rhythm of the master-config + section headings ("Plan capabilities", "Failure behavior") above. */} <> - - Choose a Preset Configuration + + Choose a preset configuration Select a preset that matches your use case. You can customize settings further in Advanced mode. @@ -89,40 +92,76 @@ export default function SimpleMode({ config, onSave, saving, error, success, onC handlePresetChange(e.target.value)}> - {GUARDRAILS_PRESETS.map((preset) => ( - - handlePresetChange(preset.id)} - > - - - } - label={ - - {preset.name} - - {preset.description} - - + {GUARDRAILS_PRESETS.map((preset) => { + const isSelected = selectedPreset === preset.id + return ( + + { + // The AnswerAI theme defines `primary.main` as a translucent + // white/slate, so opacity-suffix bg tints on `primary.main` + // produce malformed CSS (rgba(...)1A is invalid). Anchor the + // selected state on `action.selected` (mode-balanced) + + // `text.primary` border for guaranteed visibility in both modes. + const isDark = theme.palette.mode === 'dark' + return { + cursor: 'pointer', + transition: 'background-color 120ms ease, border-color 120ms ease', + borderColor: isSelected + ? isDark + ? 'rgba(255, 255, 255, 0.45)' + : 'rgba(15, 23, 42, 0.5)' + : 'divider', + borderWidth: isSelected ? 2 : 1, + ...(isSelected && { + bgcolor: 'action.selected' + }), + '&:hover': { + borderColor: isSelected + ? isDark + ? 'rgba(255, 255, 255, 0.6)' + : 'rgba(15, 23, 42, 0.7)' + : 'text.secondary', + bgcolor: isSelected + ? isDark + ? 'rgba(255, 255, 255, 0.22)' + : 'rgba(15, 23, 42, 0.12)' + : 'action.hover' } - sx={{ flex: 1 }} - /> - - - - - ))} + } + }} + onClick={() => handlePresetChange(preset.id)} + > + + + } + label={ + + + {preset.name} + + + {preset.description} + + + } + sx={{ flex: 1 }} + /> + + + + + ) + })} {selectedPreset === 'custom' && ( @@ -136,7 +175,28 @@ export default function SimpleMode({ config, onSave, saving, error, success, onC - {hasChanges && ( diff --git a/packages-answers/ui/src/Message/Message.tsx b/packages-answers/ui/src/Message/Message.tsx index a51f109eb2a..c0ea6fa7a14 100644 --- a/packages-answers/ui/src/Message/Message.tsx +++ b/packages-answers/ui/src/Message/Message.tsx @@ -177,9 +177,10 @@ export const MessageCard = ({ const hasInputValidation = guardrailsMetadata.inputValidation const hasOutputValidation = guardrailsMetadata.outputValidation + const health = (guardrailsMetadata as any).health - // If no validations were run, don't show anything - if (!hasInputValidation && !hasOutputValidation) return null + // If no validations were run AND no health signal, don't show anything + if (!hasInputValidation && !hasOutputValidation && !health) return null let status: 'good' | 'warning' | 'error' = 'good' @@ -208,6 +209,11 @@ export const MessageCard = ({ } } + // Degraded guardrail stage (couldn't reach Fiddler / auth error / circuit open) + // in fail-open mode still wants the shield icon to turn amber so admins can + // see at a glance that this message bypassed protection. + if (health?.anyDegraded && status !== 'error') status = 'warning' + const colors = { good: '#4caf50', // Green warning: '#ff9800', // Orange @@ -219,6 +225,73 @@ export const MessageCard = ({ const guardrailsStatus = getGuardrailsStatus() + // Human-friendly explanations for each degraded reason so the banner doesn't + // dump raw enum strings at the user. Admin-mode (debugVisible) reveals the + // full reason/http/message detail via the accordion. + const describeDegradedReason = (reason: string): string => { + switch (reason) { + case 'auth_error': + return 'The guardrails API key was rejected. The response was generated without safety checks.' + case 'no_credentials': + return 'No guardrail credentials are configured. The response was generated without safety checks.' + case 'timeout': + return 'The guardrails service timed out. The response was generated without safety checks.' + case 'circuit_open': + return 'The guardrails service is temporarily unavailable (too many recent failures). The response was generated without safety checks.' + case 'network_error': + return 'Could not reach the guardrails service. The response was generated without safety checks.' + case 'api_error': + return 'The guardrails service returned an error. The response was generated without safety checks.' + default: + return 'Safety checks could not run for this message. The response was generated without safety checks.' + } + } + + // Friendlier labels for the (admin-visible) accordion Health row. `unsupported` + // is a permanent plan-tier capability gap rather than an outage, so it's + // explicitly labeled differently — no banner, no shield change, just info. + const describeHealthReason = (reason: string): string => { + switch (reason) { + case 'ok': + return 'OK' + case 'disabled': + return 'Disabled' + case 'disabled_stage': + return 'Stage disabled' + case 'no_credentials': + return 'No credentials' + case 'auth_error': + return 'Auth error' + case 'api_error': + return 'API error' + case 'timeout': + return 'Timeout' + case 'circuit_open': + return 'Circuit open' + case 'network_error': + return 'Network error' + case 'unsupported': + return 'Not included in Fiddler plan (skipped silently)' + case 'unexpected': + return 'Unexpected error' + default: + return reason + } + } + + // Pick the most informative degraded reason across input/output stages. + const degradedReason: { stage: 'input' | 'output'; reason: string; httpStatus?: number } | null = React.useMemo(() => { + const health = (guardrailsMetadata as any)?.health + if (!health?.anyDegraded) return null + if (health.input?.degraded) { + return { stage: 'input', reason: health.input.reason, httpStatus: health.input.httpStatus } + } + if (health.output?.degraded) { + return { stage: 'output', reason: health.output.reason, httpStatus: health.output.httpStatus } + } + return null + }, [guardrailsMetadata]) + // Helper functions for score display const formatFaithfulnessScore = (score: number) => { // Fiddler faithfulness uses inverted scale: lower = less faithful @@ -1172,108 +1245,218 @@ export const MessageCard = ({ )} ) : null} + {!isUserMessage && degradedReason ? ( + + ⚠️ + + + Safety checks {degradedReason.stage === 'input' ? 'on your message' : 'on this response'} were unavailable + + + {describeDegradedReason(degradedReason.reason)} + {degradedReason.httpStatus ? ` (HTTP ${degradedReason.httpStatus})` : ''} + + + + ) : null} {guardrailsVisible && guardrailsMetadata ? ( } aria-controls='panel1a-content' id='panel1a-header'> 🛡️ Guardrails Validation + {(guardrailsMetadata as any).health?.anyDegraded ? ' — Degraded' : ''} {guardrailsMetadata.outputValidation?.faithfulnessScore !== undefined && ` - Faithfulness: ${(guardrailsMetadata.outputValidation.faithfulnessScore * 1000).toFixed(2)}`} - {guardrailsMetadata.inputValidation && ( + {(guardrailsMetadata as any).health ? ( - Input Validation + Health - {guardrailsMetadata.inputValidation.blocked && ( - ⚠️ Input was blocked - )} - {guardrailsMetadata.inputValidation.redacted && ( - 🔒 PII was redacted - )} - {guardrailsMetadata.inputValidation.violations?.safety?.length > 0 && ( - - Safety:{' '} - {guardrailsMetadata.inputValidation.violations.safety - .map((v) => { - const severity = formatSafetyScore(v.score) - return `${v.dimension} (${severity.text} Risk - score: ${v.score.toFixed(3)})` - }) - .join(', ')} - - )} - {guardrailsMetadata.inputValidation.violations?.pii?.length > 0 && ( - - PII:{' '} - {guardrailsMetadata.inputValidation.violations.pii - .map( - (p) => - `${p.label} (${formatPIIConfidence(p.score)} confidence - score: ${p.score.toFixed( - 3 - )})` - ) - .join(', ')} - - )} - - )} - {guardrailsMetadata.outputValidation && ( - - - Output Validation + + Failure mode: {(guardrailsMetadata as any).health.mode} - {guardrailsMetadata.outputValidation.faithfulnessScore !== undefined && ( - - {(() => { - const faithful = formatFaithfulnessScore( - guardrailsMetadata.outputValidation.faithfulnessScore - ) - return ( - <> - - - - Faithfulness: {faithful.text} {faithful.icon} - - - (Raw Score:{' '} - {guardrailsMetadata.outputValidation.faithfulnessScore.toFixed(4)}) - - - - ) - })()} - - )} - {guardrailsMetadata.outputValidation.violations?.safety?.length > 0 && ( - - Safety:{' '} - {guardrailsMetadata.outputValidation.violations.safety - .map((v) => { - const severity = formatSafetyScore(v.score) - return `${v.dimension} (${severity.text} Risk - score: ${v.score.toFixed(3)})` - }) - .join(', ')} + {(guardrailsMetadata as any).health.input && ( + + Input: {describeHealthReason((guardrailsMetadata as any).health.input.reason)} + {(guardrailsMetadata as any).health.input.httpStatus + ? ` (HTTP ${(guardrailsMetadata as any).health.input.httpStatus})` + : ''} + {(guardrailsMetadata as any).health.input.latencyMs !== undefined + ? ` · ${(guardrailsMetadata as any).health.input.latencyMs}ms` + : ''} )} - {guardrailsMetadata.outputValidation.violations?.pii?.length > 0 && ( - - PII:{' '} - {guardrailsMetadata.outputValidation.violations.pii - .map( - (p) => - `${p.label} (${formatPIIConfidence(p.score)} confidence - score: ${p.score.toFixed( - 3 - )})` - ) - .join(', ')} + {(guardrailsMetadata as any).health.output && ( + + Output: {describeHealthReason((guardrailsMetadata as any).health.output.reason)} + {(guardrailsMetadata as any).health.output.httpStatus + ? ` (HTTP ${(guardrailsMetadata as any).health.output.httpStatus})` + : ''} + {(guardrailsMetadata as any).health.output.latencyMs !== undefined + ? ` · ${(guardrailsMetadata as any).health.output.latencyMs}ms` + : ''} )} - )} + ) : null} + {guardrailsMetadata.inputValidation && + (() => { + const iv = guardrailsMetadata.inputValidation + const safetyCount = iv.violations?.safety?.length ?? 0 + const piiCount = iv.violations?.pii?.length ?? 0 + const hasDetail = !!iv.blocked || !!iv.redacted || safetyCount > 0 || piiCount > 0 + + if (!hasDetail) { + return ( + + + Input Validation + + + ✓ Clean + + + no safety violations, no PII detected + + + ) + } + + return ( + + + Input Validation + + {iv.blocked && ⚠️ Input was blocked} + {iv.redacted && 🔒 PII was redacted} + {safetyCount > 0 && ( + + Safety:{' '} + {iv.violations.safety + .map((v) => { + const severity = formatSafetyScore(v.score) + return `${v.dimension} (${severity.text} Risk - score: ${v.score.toFixed(3)})` + }) + .join(', ')} + + )} + {piiCount > 0 && ( + + PII:{' '} + {iv.violations.pii + .map( + (p) => + `${p.label} (${formatPIIConfidence( + p.score + )} confidence - score: ${p.score.toFixed(3)})` + ) + .join(', ')} + + )} + + ) + })()} + {guardrailsMetadata.outputValidation && + (() => { + const ov = guardrailsMetadata.outputValidation + const safetyCount = ov.violations?.safety?.length ?? 0 + const piiCount = ov.violations?.pii?.length ?? 0 + const hasFaithfulness = ov.faithfulnessScore !== undefined + const hasViolation = safetyCount > 0 || piiCount > 0 + const hasDetail = hasFaithfulness || hasViolation + + if (!hasDetail) { + return ( + + + Output Validation + + + ✓ Clean + + + no safety violations, no PII detected + + + ) + } + + return ( + + + Output Validation + + {hasFaithfulness && ( + + {(() => { + const faithful = formatFaithfulnessScore(ov.faithfulnessScore as number) + return ( + <> + + + + Faithfulness: {faithful.text} {faithful.icon} + + + (Raw Score: {(ov.faithfulnessScore as number).toFixed(4)}) + + + + ) + })()} + + )} + {!hasViolation && hasFaithfulness && ( + + No safety violations, no PII detected. + + )} + {safetyCount > 0 && ( + + Safety:{' '} + {ov.violations.safety + .map((v) => { + const severity = formatSafetyScore(v.score) + return `${v.dimension} (${severity.text} Risk - score: ${v.score.toFixed(3)})` + }) + .join(', ')} + + )} + {piiCount > 0 && ( + + PII:{' '} + {ov.violations.pii + .map( + (p) => + `${p.label} (${formatPIIConfidence( + p.score + )} confidence - score: ${p.score.toFixed(3)})` + ) + .join(', ')} + + )} + + ) + })()} diff --git a/packages-answers/ui/src/theme/components/muiOverrides.ts b/packages-answers/ui/src/theme/components/muiOverrides.ts index 696fa1cf62e..99889a1249e 100644 --- a/packages-answers/ui/src/theme/components/muiOverrides.ts +++ b/packages-answers/ui/src/theme/components/muiOverrides.ts @@ -459,11 +459,20 @@ export const muiComponentOverrides = (mode: 'light' | 'dark'): Components + + + +Guardrails that silently bypass enforcement when something goes wrong are not guardrails — they are telemetry. AnswerAgentAI exposes the failure posture as a first-class configuration so an admin can choose how the platform reacts when Fiddler cannot be evaluated (bad API key, network outage, upstream timeout, circuit open, no credential resolved). + +### Fail-Open vs Fail-Closed + +| Failure scenario | Fail-Open (default) | Fail-Closed | +|---|---|---| +| Real safety violation (200 + score) | Block with HTTP 400 | Block with HTTP 400 | +| Bad API key (401 from Fiddler) | Request continues, amber banner in chat, structured error log | Request returns HTTP 503, no LLM call made | +| Missing credential | Request continues, amber banner, structured warn log | Request returns HTTP 503 | +| Fiddler timeout / 5xx | Request continues, amber banner | Request returns HTTP 503 | +| Circuit breaker open | Request continues, amber banner | Request returns HTTP 503 | +| Guardrails disabled by config | Request continues, no banner | Request continues, no banner | + +**Default is `open`** so existing deployments see no behavior change until an operator explicitly opts in to strict mode. + +### Configuration Hierarchy + +`failureMode` follows the standard env → organization → chatflow hierarchy: + +1. **Environment variable** — `FIDDLER_FAILURE_MODE=open|closed` (default `open`) +2. **Organization config** — set in the admin Guardrails Settings page (`organizationConfig.guardrails.failureMode`) +3. **Chatflow override** — set in the per-chatflow Guardrails tab (`chatbotConfig.guardrails.failureMode`) + +Each layer overrides the previous. A single sensitive chatflow can be `closed` while the rest of the organization is `open`. + +### What End Users See + +- **Fail-open + degraded:** an amber banner above the chat response: _"Safety checks were unavailable for this message. The response was generated without safety checks."_ Health detail (reason, HTTP status, latency) is shown in the existing 🛡️ Guardrails Validation accordion. +- **Fail-closed + degraded:** the prediction returns HTTP 503 with the body `Safety checks are temporarily unavailable (reason: …). Please try again in a moment.` No partial LLM output is delivered. +- **Healthy or skipped (disabled):** unchanged from before — the green 🛡️ Guardrails Validation accordion appears for healthy validations and nothing renders when guardrails are disabled. + +### Observability-Only (Shadow) Mode + +For a safe pilot before flipping enforcement on in production: + +- Toggle **Observability-Only** in the admin Guardrails Settings page, or set `FIDDLER_OBSERVABILITY_ONLY=true`. +- When enabled, even **real** Fiddler 200-OK violations are downgraded from `block` to `warn`. Violations are still recorded in `chat_message.guardrails_metadata` for analytics, dashboards, and Langfuse traces, but no request is blocked. +- Use this to measure violation volume, calibrate thresholds, and confirm zero false-positives before going hot. + +### Diagnostic Selftest Endpoint + +Operators can verify the entire pipeline (config resolution, credential lookup, live Fiddler reachability, per-endpoint plan capabilities, circuit state) without sending a real chat: + +```bash +curl -H "Authorization: Bearer $YOUR_API_KEY" \ + "https://your-deployment/api/v1/guardrails/selftest?chatflowId=" +``` + +**Response shape:** + +```json +{ + "resolved": { + "chatflowId": "abc-123", + "workspaceId": "ws-456", + "organizationId": "org-789" + }, + "config": { + "enabled": true, + "failureMode": "open", + "observabilityOnly": false, + "safety": { "enabled": true, "threshold": 0.1, "action": "block" }, + "pii": { "enabled": true, "confidenceThreshold": 0.8, "action": "redact" }, + "faithfulness": { "enabled": false, "threshold": 0.005, "action": "warn" } + }, + "credentialSource": "organization", + "healthCheck": { + "ok": true, + "degraded": false, + "reason": "ok", + "latencyMs": 287 + }, + "capabilities": { + "safety": { "ok": true, "degraded": false, "reason": "ok", "latencyMs": 287 }, + "pii": { "ok": true, "degraded": false, "reason": "unsupported", "httpStatus": 404 }, + "faithfulness": { "ok": true, "degraded": false, "reason": "ok", "latencyMs": 312 } + }, + "circuitState": "CLOSED", + "notes": [ + "PII detection (sensitive-information) is not included in this Fiddler plan tier; PII checks will be skipped silently at runtime." + ] +} +``` + +`credentialSource` will be one of `config_credential_id`, `workspace`, `organization`, `env`, or `none`. `healthCheck.reason` and `capabilities..reason` are a stable enum (`ok`, `disabled`, `no_credentials`, `auth_error`, `api_error`, `timeout`, `circuit_open`, `network_error`, `unsupported`, `unexpected`) suitable for alerting. + +## Fiddler Plan Tiers (Freemium vs Paid) + +Not every Fiddler endpoint is included in every plan tier. The freemium guardrails API ships with safety enabled, but PII detection (`/v3/guardrails/sensitive-information`) and faithfulness (`/v3/guardrails/ftl-response-faithfulness`) require a paid Fiddler subscription. When the runtime calls an endpoint that is not included in the caller's plan, Fiddler responds with HTTP 404 and a body indicating the guardrail is not supported. + +AnswerAgentAI distinguishes "Fiddler said no" (plan-tier capability gap) from "Fiddler can't be reached" (degradation): + +- **Capability gap (`reason: "unsupported"`)** — The API answered, the plan just doesn't include the endpoint. The runtime caches the unsupported state per `(api_key, endpoint)` so subsequent calls short-circuit without hitting the network or tripping the circuit breaker. **No per-message banner is shown** because the situation is permanent and configuration-level — not a runtime outage. Admins see it once on the Guardrails Settings page and via `selftest`. +- **Outage (`reason: "auth_error" | "timeout" | "api_error" | "circuit_open" | "network_error"`)** — Fiddler is unhealthy. Behavior follows `failureMode` (open: per-message banner, closed: HTTP 503). + +**Plan capability matrix in the admin UI.** When a Fiddler credential is connected, the Guardrails Settings page renders a row of chips showing the live capability for each endpoint — green ✓ for available, amber − for not included in your plan, red ! for actually degraded. A "Recheck" button re-runs the probe. + +**What the user sees with a freemium key.** Safety still runs; PII and faithfulness silently skip. There is no per-message warning banner — chats look identical to a fully-supported plan. Violations metadata records `reason: "unsupported"` for the skipped sub-stage so audit/Langfuse traces remain accurate. + +### Credential Visibility + +Guardrail credentials are saved with `Organization` visibility by default. The runtime now correctly resolves credentials in this priority order: + +1. Explicit `credentialId` referenced in the chatflow or organization config (workspace-owned OR Organization/Platform-visible) +2. Any `fiddlerApi` credential owned by the chatflow's workspace +3. Any `fiddlerApi` credential with `Organization` or `Platform` visibility in the chatflow's organization +4. `FIDDLER_API_KEY` + `FIDDLER_API_URL` environment variables + +Multi-workspace organizations are now correctly served by a single org-visible credential. + ## Use Cases
@@ -851,13 +979,22 @@ A: Configure `chatflowConfig` on the ChatFlow entity: **Q: What happens if my Fiddler API key is invalid?** -A: The integration uses **fail-open** design: +A: The integration's behavior is governed by the configured **failureMode** (`open` by default): + +- **Fail-open:** Invalid credentials, timeouts, 5xx, or network errors → log structured warning, continue the chat, surface an amber degraded banner above the message. +- **Fail-closed:** Same conditions → return HTTP 503 to the caller, no LLM call made. + +**Plan-tier 404s are handled separately** — they are not treated as degradation. If Fiddler responds with `404 NotFound` and "not supported by the freemium guardrails API", the runtime classifies the endpoint as `unsupported`, caches that decision per (api_key, endpoint), and silently skips that guardrail without a banner. See **Fiddler Plan Tiers** above. + +**Q: I'm seeing an `HTTP 404` banner in chat. What's wrong?** + +A: That banner is shown when Fiddler returns 404 for a reason _other_ than a plan-tier capability gap (which is detected via the response body). The most common causes: -- Invalid credentials → Log warning, continue without guardrails -- API errors → Return safe defaults (no violations detected) -- Circuit breaker open → Skip guardrails, allow requests through +1. Wrong `apiUrl` in the credential — verify it points to your Fiddler guardrails host (e.g. `https://guardrails.cloud.fiddler.ai`). +2. The credential is being resolved from an unexpected source — call `/api/v1/guardrails/selftest?chatflowId=` and inspect `credentialSource`. +3. A Fiddler-side incident — check Fiddler status. The circuit breaker will open after 5 failures and self-heal after 30s. -This ensures your application never goes down due to guardrails service issues. +If you're on a freemium plan and the banner specifically mentions PII / sensitive-information / faithfulness, ensure you're on the latest version — those endpoints are now classified as `unsupported` and skipped silently rather than banner'd. ### Usage & Best Practices diff --git a/packages/server/src/Interface.ts b/packages/server/src/Interface.ts index 7dce6621597..d611110b4a5 100644 --- a/packages/server/src/Interface.ts +++ b/packages/server/src/Interface.ts @@ -70,6 +70,26 @@ export enum UserPlan { * Guardrails Metadata * Captures validation results from Fiddler Guardrails for audit and client display */ +export interface GuardrailsHealthEntry { + ok: boolean + degraded: boolean + reason: + | 'ok' + | 'disabled' + | 'disabled_stage' + | 'no_credentials' + | 'auth_error' + | 'api_error' + | 'timeout' + | 'circuit_open' + | 'network_error' + | 'unsupported' + | 'unexpected' + httpStatus?: number + message?: string + latencyMs?: number +} + export interface GuardrailsMetadata { inputValidation?: InputValidationResult & { blocked: boolean @@ -106,6 +126,17 @@ export interface GuardrailsMetadata { }> } } + /** + * Health/failure posture for this request. Present on every prediction that + * reaches the guardrail layer (including short-circuits like disabled/no-creds). + * Callers that pre-date this field will see it as undefined. + */ + health?: { + mode: 'open' | 'closed' + anyDegraded: boolean + input?: GuardrailsHealthEntry + output?: GuardrailsHealthEntry + } } /** @@ -561,6 +592,12 @@ export interface IExecuteFlowParams extends IPredictionQueueAppServer { isUpsert?: boolean user?: IUser isRecursive?: boolean + /** + * Guardrails metadata carried in from an upstream input-validation stage. + * Agentflow V2 merges its output-stage result into this object so the + * persisted ChatMessage has a single coherent `health` + validation record. + */ + guardrailsMetadata?: Partial parentExecutionId?: string iterationContext?: ICommonObject isTool?: boolean diff --git a/packages/server/src/controllers/guardrails/index.ts b/packages/server/src/controllers/guardrails/index.ts new file mode 100644 index 00000000000..6ac060223ba --- /dev/null +++ b/packages/server/src/controllers/guardrails/index.ts @@ -0,0 +1,37 @@ +import type { NextFunction, Request, Response } from 'express' +import { StatusCodes } from 'http-status-codes' +import { InternalFlowiseError } from '../../errors/internalFlowiseError' +import { runGuardrailsSelftest } from '../../services/guardrails/selftest' + +/** + * GET /api/v1/guardrails/selftest + * + * Returns a diagnostic report describing whether guardrails can be reached + * from the current runtime for the caller's organization (optionally scoped + * to a specific chatflow via ?chatflowId=...). + * + * Admin-gated: any authenticated user in an org can trigger it today; we + * intentionally avoid restricting further until we see how ops wants to wire + * this into existing dashboards. + */ +const getSelftest = async (req: Request, res: Response, next: NextFunction) => { + try { + if (!req.user) { + throw new InternalFlowiseError(StatusCodes.UNAUTHORIZED, 'Unauthorized - No user') + } + const organizationId = req.user.organizationId + if (!organizationId) { + throw new InternalFlowiseError(StatusCodes.PRECONDITION_FAILED, 'User organization ID not found') + } + + const chatflowId = typeof req.query.chatflowId === 'string' ? req.query.chatflowId : undefined + const workspaceIdOverride = typeof req.query.workspaceId === 'string' ? req.query.workspaceId : undefined + + const report = await runGuardrailsSelftest({ organizationId, chatflowId, workspaceIdOverride }) + return res.json(report) + } catch (error) { + next(error) + } +} + +export default { getSelftest } diff --git a/packages/server/src/routes/guardrails/index.ts b/packages/server/src/routes/guardrails/index.ts new file mode 100644 index 00000000000..f8adb0be97f --- /dev/null +++ b/packages/server/src/routes/guardrails/index.ts @@ -0,0 +1,13 @@ +import express from 'express' +import guardrailsController from '../../controllers/guardrails' + +const router = express.Router() + +/** + * Diagnostic selftest: reports resolved config, credential source, live + * healthCheck() latency/status, and circuit-breaker state. Operators hit this + * to verify guardrail wiring in any environment. + */ +router.get('/selftest', guardrailsController.getSelftest) + +export default router diff --git a/packages/server/src/routes/index.ts b/packages/server/src/routes/index.ts index 3e8dbe7aa5a..c0ed00daca1 100644 --- a/packages/server/src/routes/index.ts +++ b/packages/server/src/routes/index.ts @@ -90,6 +90,7 @@ import { createAuthMeRouter } from '../aai/routes/auth-me' import { createCredentialsRefreshRouter } from '../aai/routes/credentials-refresh' import organizationsRouter from './organizations' +import guardrailsRouter from './guardrails' const router = express.Router() @@ -184,6 +185,7 @@ router.use('/admin', adminRouter) router.use('/settings', settingsRouter) router.use('/pricing', pricingRouter) router.use('/organizations', organizationsRouter) +router.use('/guardrails', guardrailsRouter) export default router diff --git a/packages/server/src/services/guardrails/FiddlerGuardrailsService.ts b/packages/server/src/services/guardrails/FiddlerGuardrailsService.ts index 3b74f8942f2..ba759ff4d2a 100644 --- a/packages/server/src/services/guardrails/FiddlerGuardrailsService.ts +++ b/packages/server/src/services/guardrails/FiddlerGuardrailsService.ts @@ -9,18 +9,20 @@ * - Fail-open by default */ -import axios, { AxiosInstance, AxiosError } from 'axios' -import { StatusCodes } from 'http-status-codes' -import { InternalFlowiseError } from '../../errors/internalFlowiseError' -import { getErrorMessage } from '../../errors/utils' +import axios, { AxiosInstance } from 'axios' +import { createHash } from 'crypto' +import { In } from 'typeorm' import { CircuitBreaker } from './CircuitBreaker' import { getGuardrailsConfig } from './config' import { getRunningExpressApp } from '../../utils/getRunningExpressApp' -import { Credential } from '../../database/entities/Credential' +import { Credential, CredentialVisibility } from '../../database/entities/Credential' import { decryptCredentialData } from '../../utils' +import logger from '../../utils/logger' +import { FiddlerCircuitOpenError, FiddlerError, FiddlerUnsupportedError, toFiddlerError } from './errors' import { GuardrailsConfig, GuardrailAction, + GuardrailStageStatus, SafetyAPIResponse, SafetyEvaluationResult, SafetyViolation, @@ -33,18 +35,89 @@ import { OutputValidationResult } from '../../types/guardrails' +const HEALTHY_STATUS: GuardrailStageStatus = { ok: true, degraded: false, reason: 'ok' } +const DISABLED_STATUS: GuardrailStageStatus = { ok: true, degraded: false, reason: 'disabled_stage' } + +/** + * Build a status from a FiddlerError. Keeps a stable, machine-readable reason + * so alerting rules and UI banners can branch on it. + * + * Note: `unsupported` (plan-tier capability gap) is NOT degraded — Fiddler + * answered, just told us the endpoint isn't available on this plan. Treating + * it as degraded would surface a per-message banner for a permanent config + * issue. Surfaced once to admins via the selftest capability matrix instead. + */ +const toStatus = (err: FiddlerError, startedAt: number): GuardrailStageStatus => { + const isUnsupported = err instanceof FiddlerUnsupportedError + return { + ok: isUnsupported, + degraded: !isUnsupported, + reason: err.reason, + httpStatus: err.httpStatus, + message: err.message, + latencyMs: Date.now() - startedAt + } +} + +/** + * Provenance of resolved credentials. Surfaced by the selftest endpoint so + * operators can tell where the key is actually coming from. + */ +export type FiddlerCredentialSource = 'config_credential_id' | 'workspace' | 'organization' | 'env' | 'none' + export interface FiddlerCredentials { apiKey: string apiUrl: string } +/** + * Process-wide cache of (apiKeyHash + endpoint) → unsupported. Once Fiddler + * tells us a guardrail isn't included in the caller's plan tier, we record + * it here so subsequent calls short-circuit without hammering the API or + * tripping the circuit breaker on a 404 that will never resolve. + * + * Keyed on a hash of the API key (not the key itself) so that: + * - different orgs with different keys/plan tiers don't pollute each other + * - we never log or persist raw credentials + * - rotating an API key on an existing credential row produces a new + * fingerprint, which auto-invalidates prior plan-tier decisions for the + * rotated key (a property a credential-ID-keyed cache would lose without + * extra invalidation plumbing — important when admins upgrade from a + * freemium key to a paid key on the same credential) + * + * Lives for the process lifetime; restart clears it. + */ +const unsupportedEndpoints: Set = new Set() + +/** + * 64-bit fingerprint used solely as a namespace key for `unsupportedEndpoints` + * above. NOT a password hash and NOT a security boundary: + * + * - never persisted (no DB, log, file, or wire) + * - never compared against another value to grant access + * - not derived for storage or offline verification + * + * CWE-916 (use of password hash with insufficient computational effort) does + * not apply: there is no offline attack surface for this value. KDFs like + * bcrypt / scrypt / argon2 / PBKDF2 would add per-request latency without + * improving the threat model. The CodeQL rule `js/insufficient-password-hash` + * fires here on data-flow alone (the variable is named `apiKey` and reaches + * `crypto.createHash`) and is a false positive in this context. Documented + * and dismissed in GHAS — see PR #1060 / alerts #318, #319. + */ +const apiKeyHash = (apiKey: string): string => createHash('sha256').update(apiKey).digest('hex').slice(0, 16) + +const unsupportedCacheKey = (keyHash: string, endpoint: string): string => `${keyHash}|${endpoint}` + export class FiddlerGuardrailsService { private client: AxiosInstance private circuitBreaker: CircuitBreaker private config: GuardrailsConfig + private apiKeyHash: string constructor(credentials: FiddlerCredentials, config: GuardrailsConfig) { this.config = config + this.apiKeyHash = apiKeyHash(credentials.apiKey) // Initialize HTTP client with connection pooling this.client = axios.create({ @@ -96,112 +169,193 @@ export class FiddlerGuardrailsService { workspaceId: string, organizationId?: string ): Promise { - try { - // 1. Load configuration (env → org → chatflow) - const config = await getGuardrailsConfig(chatflowId, organizationId) + const resolution = await this.resolveFromContext(chatflowId, workspaceId, organizationId) + return resolution.service + } - // 2. Check if guardrails are enabled - if (!config.enabled) { - return null + /** + * Resolve context into a diagnostic report describing whether a service + * could be built and, if not, why. Callers that need to react to degraded + * states (render a banner, fail-closed, etc.) use this richer return. + * + * `service` is non-null ONLY when config is enabled and credentials were + * resolved. All other outcomes set `service` to null and populate `reason`. + */ + public static async resolveFromContext( + chatflowId: string, + workspaceId: string, + organizationId?: string + ): Promise<{ + service: FiddlerGuardrailsService | null + config: GuardrailsConfig + credentialSource: FiddlerCredentialSource + reason: 'ok' | 'disabled' | 'no_credentials' | 'unexpected' + error?: Error + }> { + let config: GuardrailsConfig + try { + config = await getGuardrailsConfig(chatflowId, organizationId) + } catch (error) { + logger.error('[Guardrails] Failed to load config (fail-safe)', { + error: (error as Error).message, + chatflowId, + workspaceId, + organizationId + }) + // Return a minimal-disabled config so the caller's health/fail-mode logic still works + return { + service: null, + config: { enabled: false } as GuardrailsConfig, + credentialSource: 'none', + reason: 'unexpected', + error: error as Error } + } + + if (!config.enabled) { + return { service: null, config, credentialSource: 'none', reason: 'disabled' } + } - // 3. Load credentials with fallback chain (using workspaceId since credentials are workspace-scoped) - const credentials = await this.loadCredentials(workspaceId, config) + try { + const { credentials, source } = await this.loadCredentials(workspaceId, organizationId, config) if (!credentials) { - console.warn(`Guardrails enabled but no credentials found for workspace ${workspaceId}`) - return null + logger.warn('[Guardrails] Enabled but no credentials could be resolved', { + chatflowId, + workspaceId, + organizationId, + credentialId: config.credentialId + }) + return { service: null, config, credentialSource: 'none', reason: 'no_credentials' } + } + return { + service: new FiddlerGuardrailsService(credentials, config), + config, + credentialSource: source, + reason: 'ok' } - - // 4. Initialize and return service - return new FiddlerGuardrailsService(credentials, config) } catch (error) { - // Fail-open: log error but don't throw - console.error('Error initializing Fiddler Guardrails service (fail-open):', error) - return null + logger.error('[Guardrails] Unexpected error resolving service', { + error: (error as Error).message, + chatflowId, + workspaceId, + organizationId + }) + return { service: null, config, credentialSource: 'none', reason: 'unexpected', error: error as Error } } } /** - * Load Fiddler credentials with multi-tier fallback - * Priority: Config credentialId → Workspace credential by name → Environment variables + * Load Fiddler credentials with visibility-aware multi-tier lookup. + * + * Priority: + * 1. Config `credentialId` (chatflow/org override) — owned by workspace OR visible to org + * 2. Any `fiddlerApi` credential owned by workspace OR visible to org (Organization/Platform visibility) + * 3. Environment variables (`FIDDLER_API_KEY` + `FIDDLER_API_URL`) * - * @param workspaceId - Workspace ID for credential scoping (credentials are workspace-scoped) - * @param config - Guardrails configuration - * @returns Credentials or null if not found + * Previously this filtered by `workspaceId` only. Because the admin UI saves + * guardrail credentials with `visibility: ['Organization']`, a credential + * created under workspace A was invisible to a chatflow executing under + * workspace B of the same org, silently disabling guardrails. */ - private static async loadCredentials(workspaceId: string, config: GuardrailsConfig): Promise { + public static async loadCredentials( + workspaceId: string, + organizationId: string | undefined, + config: GuardrailsConfig + ): Promise<{ credentials: FiddlerCredentials | null; source: FiddlerCredentialSource }> { try { const appServer = getRunningExpressApp() const credentialRepository = appServer.AppDataSource.getRepository(Credential) - // Priority 1: Use credentialId from config (chatflow/org override) + // Build visibility-aware where-clause: match by workspace OR (org AND visible to org). + const orgVisibilityClause = organizationId + ? [ + { + organizationId, + visibility: In([CredentialVisibility.ORGANIZATION, CredentialVisibility.PLATFORM]) + } + ] + : [] + + // Priority 1: Explicit credentialId from config if (config.credentialId) { const credential = await credentialRepository.findOne({ - where: { - id: config.credentialId, - workspaceId - } + where: [ + { id: config.credentialId, workspaceId }, + ...orgVisibilityClause.map((c) => ({ id: config.credentialId, ...c })) + ] }) if (credential) { - const credentialData = await decryptCredentialData(credential.encryptedData) + const data = await decryptCredentialData(credential.encryptedData) return { - apiKey: credentialData.fiddlerApiKey, - apiUrl: credentialData.fiddlerApiUrl + credentials: { apiKey: data.fiddlerApiKey, apiUrl: data.fiddlerApiUrl }, + source: 'config_credential_id' } } } - // Priority 2: Find workspace's Fiddler credential by name - const credentials = await credentialRepository.find({ - where: { - credentialName: 'fiddlerApi', - workspaceId - } + // Priority 2: fiddlerApi credential owned by workspace + const workspaceCred = await credentialRepository.findOne({ + where: { credentialName: 'fiddlerApi', workspaceId }, + order: { updatedDate: 'DESC' } }) - - if (credentials && credentials.length > 0) { - const credentialData = await decryptCredentialData(credentials[0].encryptedData) + if (workspaceCred) { + const data = await decryptCredentialData(workspaceCred.encryptedData) return { - apiKey: credentialData.fiddlerApiKey, - apiUrl: credentialData.fiddlerApiUrl + credentials: { apiKey: data.fiddlerApiKey, apiUrl: data.fiddlerApiUrl }, + source: 'workspace' } } - // Priority 3: Fallback to environment variables + // Priority 2b: fiddlerApi credential visible across the organization + if (organizationId) { + const orgCred = await credentialRepository.findOne({ + where: orgVisibilityClause.map((c) => ({ credentialName: 'fiddlerApi', ...c })), + order: { updatedDate: 'DESC' } + }) + if (orgCred) { + const data = await decryptCredentialData(orgCred.encryptedData) + return { + credentials: { apiKey: data.fiddlerApiKey, apiUrl: data.fiddlerApiUrl }, + source: 'organization' + } + } + } + + // Priority 3: Environment variables const envApiKey = process.env.FIDDLER_API_KEY const envApiUrl = process.env.FIDDLER_API_URL - if (envApiKey && envApiUrl) { - // eslint-disable-next-line no-console - console.log(`Using Fiddler credentials from environment variables for workspace ${workspaceId}`) - return { - apiKey: envApiKey, - apiUrl: envApiUrl - } + logger.info('[Guardrails] Using credentials from environment variables', { workspaceId, organizationId }) + return { credentials: { apiKey: envApiKey, apiUrl: envApiUrl }, source: 'env' } } - // No credentials found - return null + return { credentials: null, source: 'none' } } catch (error) { - console.error('Error loading Fiddler credentials:', error) - return null + logger.error('[Guardrails] Error loading Fiddler credentials', { + error: (error as Error).message, + workspaceId, + organizationId + }) + return { credentials: null, source: 'none' } } } /** - * Execute request with circuit breaker protection + * Execute a Fiddler API call with circuit breaker protection. + * + * Throws typed `FiddlerError` subclasses on any failure (including open + * circuit) so callers can classify and record a degraded status. No silent + * fallbacks — "API down" must be distinguishable from "API said safe". + * + * `FiddlerUnsupportedError` is treated as a circuit-breaker SUCCESS: the + * API responded normally, the caller's plan just doesn't include the + * endpoint. Penalizing the breaker on these would trip it and break the + * other (working) endpoints on the same Fiddler account. */ - private async executeWithCircuitBreaker(operation: () => Promise, fallback?: () => T): Promise { - // Check if circuit allows execution + private async executeWithCircuitBreaker(operation: () => Promise): Promise { if (!this.circuitBreaker.canExecute()) { - if (fallback) { - return fallback() - } - throw new InternalFlowiseError( - StatusCodes.SERVICE_UNAVAILABLE, - 'Error: FiddlerGuardrailsService - Circuit breaker is open (service unavailable)' - ) + throw new FiddlerCircuitOpenError() } try { @@ -209,150 +363,142 @@ export class FiddlerGuardrailsService { this.circuitBreaker.recordSuccess() return result } catch (error) { - this.circuitBreaker.recordFailure() - - // If we have a fallback, use it instead of throwing - if (fallback) { - return fallback() + const fiddlerErr = toFiddlerError(error) + if (fiddlerErr instanceof FiddlerUnsupportedError) { + this.circuitBreaker.recordSuccess() + } else { + this.circuitBreaker.recordFailure() } - - throw error + throw fiddlerErr } } /** - * Make POST request to Fiddler API + * Make POST request to Fiddler API. Throws typed `FiddlerError`. + * + * Short-circuits with `FiddlerUnsupportedError` if a previous call to this + * (apiKey, endpoint) pair was rejected with a plan-tier 404 — avoids + * repeatedly calling an endpoint Fiddler has already told us is unavailable. */ private async post(endpoint: string, data: any): Promise { + const cacheKey = unsupportedCacheKey(this.apiKeyHash, endpoint) + if (unsupportedEndpoints.has(cacheKey)) { + throw new FiddlerUnsupportedError(`Endpoint ${endpoint} is not included in your Fiddler plan.`, endpoint) + } try { const response = await this.client.post(endpoint, data) return response.data } catch (error) { - if (axios.isAxiosError(error)) { - const axiosError = error as AxiosError - throw new InternalFlowiseError( - axiosError.response?.status || StatusCodes.INTERNAL_SERVER_ERROR, - `Error: FiddlerGuardrailsService.post - ${getErrorMessage(error)}` - ) + const fiddlerErr = toFiddlerError(error) + if (fiddlerErr instanceof FiddlerUnsupportedError) { + unsupportedEndpoints.add(cacheKey) + logger.warn('[Guardrails] Fiddler endpoint not included in plan; caching as unsupported', { + endpoint, + httpStatus: fiddlerErr.httpStatus, + message: fiddlerErr.message + }) } - throw new InternalFlowiseError( - StatusCodes.INTERNAL_SERVER_ERROR, - `Error: FiddlerGuardrailsService.post - ${getErrorMessage(error)}` - ) + throw fiddlerErr } } /** - * Evaluate safety (11 dimensions) - * Returns violations for dimensions exceeding threshold + * Public capability probe. True if a previous call to this endpoint on + * this API key was rejected as plan-tier-unsupported. Used by the admin + * selftest endpoint to render a capability matrix without making a live + * call (which would itself be cached as unsupported). */ - public async evaluateSafety(text: string): Promise> { + public isEndpointUnsupported(endpoint: string): boolean { + return unsupportedEndpoints.has(unsupportedCacheKey(this.apiKeyHash, endpoint)) + } + + /** + * Test-only: clear the process-wide unsupported cache. Not part of the + * public contract. + * @internal + */ + public static __resetUnsupportedCacheForTests(): void { + unsupportedEndpoints.clear() + } + + /** + * Evaluate safety (11 dimensions). Returns both the evaluation result and a + * stage health status. Throws nothing — errors are reflected in status. + */ + public async evaluateSafety(text: string): Promise<{ + result: Partial + status: GuardrailStageStatus + }> { if (!this.config.safety.enabled) { - return { dimensions: [], violations: [], isUnsafe: false } + return { + result: { dimensions: [], violations: [], isUnsafe: false }, + status: DISABLED_STATUS + } } + const startedAt = Date.now() try { - const apiResponse = await this.executeWithCircuitBreaker( - async () => { - const response = await this.post('/v3/guardrails/ftl-safety', { - data: { input: text } - }) - return response - }, - (): SafetyAPIResponse => { - // Return empty scores for all dimensions - return { - fdl_harmful: 0, - fdl_violent: 0, - fdl_unethical: 0, - fdl_illegal: 0, - fdl_sexual: 0, - fdl_racist: 0, - fdl_jailbreaking: 0, - fdl_harassing: 0, - fdl_hateful: 0, - fdl_sexist: 0, - fdl_roleplaying: 0 - } - } - ) + const apiResponse = await this.executeWithCircuitBreaker(async () => { + return await this.post('/v3/guardrails/ftl-safety', { + data: { input: text } + }) + }) - // Parse API response and evaluate per-dimension thresholds const violations: SafetyViolation[] = [] const dimensions = Object.keys(apiResponse) as SafetyDimension[] for (const dimension of dimensions) { const score = apiResponse[dimension] - - // Get threshold: per-dimension override or global threshold const threshold = this.config.safety.dimensionThresholds?.[dimension] ?? this.config.safety.threshold if (score > threshold) { - // Get action: per-dimension override or global action const action = this.config.safety.dimensionActions?.[dimension] ?? this.config.safety.action - - violations.push({ - dimension, - score, - threshold, - action - }) + violations.push({ dimension, score, threshold, action }) } } return { - ...apiResponse, - violations, - isUnsafe: violations.length > 0 - } as SafetyEvaluationResult + result: { ...apiResponse, violations, isUnsafe: violations.length > 0 } as SafetyEvaluationResult, + status: { ...HEALTHY_STATUS, latencyMs: Date.now() - startedAt } + } } catch (error) { - // Fail-open: return no violations on error - return { dimensions: [], violations: [], isUnsafe: false } + const fiddlerErr = error instanceof FiddlerError ? error : toFiddlerError(error) + return { + // Honest fallback: no violations detected because we could not evaluate, + // NOT because content was safe. Caller must consult `status.degraded`. + result: { dimensions: [], violations: [], isUnsafe: false }, + status: toStatus(fiddlerErr, startedAt) + } } } /** - * Detect PII (15+ types) with per-type filtering and redaction - * Returns filtered detections and optionally redacted text + * Detect PII (15+ types) with per-type filtering and redaction. Returns + * both the evaluation result and a stage health status. Throws nothing. */ - public async detectPII(text: string): Promise { + public async detectPII(text: string): Promise<{ result: PIIDetectionResult; status: GuardrailStageStatus }> { if (!this.config.pii.enabled) { - return { detections: [], hasPII: false } + return { result: { detections: [], hasPII: false }, status: DISABLED_STATUS } } + const startedAt = Date.now() try { - const apiResponse = await this.executeWithCircuitBreaker( - async () => { - const response = await this.post('/v3/guardrails/sensitive-information', { - data: { input: text } - }) - return response - }, - (): PIIAPIResponse => { - return { fdl_sensitive_information_scores: [] } - } - ) + const apiResponse = await this.executeWithCircuitBreaker(async () => { + return await this.post('/v3/guardrails/sensitive-information', { + data: { input: text } + }) + }) - // Filter detections by per-type confidence thresholds and enabled types const rawDetections = apiResponse.fdl_sensitive_information_scores || [] const filteredDetections: PIIDetection[] = [] for (const entity of rawDetections) { const entityLabel = entity.label as PIIType + if (this.config.pii.enabledTypes && !this.config.pii.enabledTypes.includes(entityLabel)) continue - // Check if type is enabled (if enabledTypes is specified) - if (this.config.pii.enabledTypes && !this.config.pii.enabledTypes.includes(entityLabel)) { - continue - } - - // Get confidence threshold: per-type override or global threshold const threshold = this.config.pii.typeConfidenceThresholds?.[entityLabel] ?? this.config.pii.confidenceThreshold - - // Filter by confidence if (entity.score >= threshold) { - // Get action: per-type override or global action const action = this.config.pii.typeActions?.[entityLabel] ?? this.config.pii.action - filteredDetections.push({ label: entityLabel, score: entity.score, @@ -364,10 +510,8 @@ export class FiddlerGuardrailsService { } } - // Generate redacted text if any redact actions let redactedText: string | undefined const hasRedactActions = filteredDetections.some((d) => d.action === 'redact' || d.action === 'replace') - if (hasRedactActions) { redactedText = this.redactPII( text, @@ -376,14 +520,20 @@ export class FiddlerGuardrailsService { } return { - ...apiResponse, - detections: filteredDetections, - hasPII: filteredDetections.length > 0, - redactedText + result: { + ...apiResponse, + detections: filteredDetections, + hasPII: filteredDetections.length > 0, + redactedText + }, + status: { ...HEALTHY_STATUS, latencyMs: Date.now() - startedAt } } } catch (error) { - // Fail-open: return no detections on error - return { detections: [], hasPII: false } + const fiddlerErr = error instanceof FiddlerError ? error : toFiddlerError(error) + return { + result: { detections: [], hasPII: false }, + status: toStatus(fiddlerErr, startedAt) + } } } @@ -405,184 +555,187 @@ export class FiddlerGuardrailsService { } /** - * Validate input text (safety + PII checks in parallel) - * Returns combined validation result with blocking/redaction logic + * Validate input text (safety + PII checks in parallel). Always resolves; + * per-stage failures are reflected in `result.status.degraded`/`reason`. + * + * Observability-only mode: if `config.observabilityOnly === true`, even + * real 'block' actions are downgraded to warnings so the request continues + * to the LLM while violations are still recorded for analytics. */ public async validateInput(text: string): Promise { - try { - // Run safety and PII checks in parallel for best performance - const [safetyResult, piiResult] = await Promise.all([this.evaluateSafety(text), this.detectPII(text)]) - - // Determine if input should be blocked - const shouldBlock = - safetyResult.violations?.some((v) => v.action === 'block') || piiResult.detections.some((d) => d.action === 'block') - - // Determine if input should be redacted - const shouldRedact = piiResult.redactedText !== undefined - - // Build result - const result: InputValidationResult = { - safetyResult, - piiResult, - blocked: shouldBlock, - redacted: shouldRedact, - redactedText: piiResult.redactedText, - violations: { - safety: safetyResult.violations, - pii: piiResult.detections - } - } + const [safety, pii] = await Promise.all([this.evaluateSafety(text), this.detectPII(text)]) + + const safetyResult = safety.result + const piiResult = pii.result + + const obsOnly = this.config.observabilityOnly === true + const shouldBlock = + !obsOnly && + ((safetyResult.violations || []).some((v) => v.action === 'block') || + (piiResult.detections || []).some((d) => d.action === 'block')) + const shouldRedact = piiResult.redactedText !== undefined + + const result: InputValidationResult = { + safetyResult, + piiResult, + blocked: shouldBlock, + redacted: shouldRedact, + redactedText: piiResult.redactedText, + violations: { + safety: safetyResult.violations, + pii: piiResult.detections + }, + status: mergeStageStatus([safety.status, pii.status]) + } - // Add block message if blocked (combine both safety and PII) - if (shouldBlock) { - const safetyBlocks = safetyResult.violations?.filter((v) => v.action === 'block') - const piiBlocks = piiResult.detections.filter((d) => d.action === 'block') + if (shouldBlock) { + const safetyBlocks = safetyResult.violations?.filter((v) => v.action === 'block') + const piiBlocks = (piiResult.detections || []).filter((d) => d.action === 'block') - const messages: string[] = [] - if (safetyBlocks && safetyBlocks.length > 0) { - messages.push(`Safety: ${safetyBlocks?.map((v) => v.dimension).join(', ')}`) - } - if (piiBlocks && piiBlocks.length > 0) { - messages.push(`PII: ${piiBlocks.map((d) => d.label).join(', ')}`) - } - - result.message = `Content blocked - ${messages.join('; ')}` + const messages: string[] = [] + if (safetyBlocks && safetyBlocks.length > 0) { + messages.push(`Safety: ${safetyBlocks.map((v) => v.dimension).join(', ')}`) } - - return result - } catch (error) { - // Fail-open: on error, allow the input to pass through - return { - safetyResult: { dimensions: [], violations: [], isUnsafe: false }, - piiResult: { detections: [], hasPII: false }, - blocked: false, - redacted: false, - violations: {} + if (piiBlocks.length > 0) { + messages.push(`PII: ${piiBlocks.map((d) => d.label).join(', ')}`) } + result.message = `Content blocked - ${messages.join('; ')}` } + + return result } /** * Evaluate faithfulness (RAG hallucination detection) - * Uses Fiddler's Fast Faithfulness model to detect hallucinations + * Uses Fiddler's Fast Faithfulness model to detect hallucinations. * * Note: Fiddler faithfulness score uses inverted scale: * - Score < 0.005 = unfaithful (hallucination/inaccuracy) * - Score ≥ 0.005 = faithful (accurate response) * - * @param text - AI-generated response to evaluate - * @param context - Source context/documents to compare against + * Wire format (verified against /v3/guardrails/ftl-response-faithfulness): + * `{ data: { prompt, response, context } }` + * Previously sent `{ data: { input, output } }`, which Fiddler rejected + * with HTTP 400 "Input is invalid". The model needs all three fields to + * decide whether the response is grounded in the context for the prompt. + * + * @param response - AI-generated response to evaluate + * @param context - Source context/documents to compare against + * @param prompt - Original user prompt that produced `response` * @returns Faithfulness evaluation with score and action */ public async evaluateFaithfulness( - text: string, - context: string + response: string, + context: string, + prompt: string ): Promise<{ - score: number - threshold: number - action: GuardrailAction + result: { + score: number + threshold: number + action: GuardrailAction + } + status: GuardrailStageStatus }> { if (!this.config.faithfulness.enabled) { return { - score: 1.0, - threshold: this.config.faithfulness.threshold, - action: 'warn' + result: { + score: 1.0, + threshold: this.config.faithfulness.threshold, + action: 'warn' + }, + status: DISABLED_STATUS } } + const startedAt = Date.now() try { - const result = await this.executeWithCircuitBreaker( - async () => { - const response = await this.post<{ fdl_faithful_score: number }>('/v3/guardrails/ftl-response-faithfulness', { - data: { - input: context, - output: text - } - }) - return response - }, - () => ({ - fdl_faithful_score: 1.0 // Fail-open: assume faithful + const api = await this.executeWithCircuitBreaker(async () => { + return await this.post<{ fdl_faithful_score: number }>('/v3/guardrails/ftl-response-faithfulness', { + data: { prompt, response, context } }) - ) - - const score = result.fdl_faithful_score - const threshold = this.config.faithfulness.threshold + }) return { - score, - threshold, - action: this.config.faithfulness.action + result: { + score: api.fdl_faithful_score, + threshold: this.config.faithfulness.threshold, + action: this.config.faithfulness.action + }, + status: { ...HEALTHY_STATUS, latencyMs: Date.now() - startedAt } } } catch (error) { - // Fail-open: return high faithfulness score on error + const fiddlerErr = error instanceof FiddlerError ? error : toFiddlerError(error) return { - score: 1.0, - threshold: this.config.faithfulness.threshold, - action: 'warn' + // Honest fallback: no score — caller must consult status.degraded + // before interpreting. We still supply 1.0 so legacy consumers don't NaN. + result: { + score: 1.0, + threshold: this.config.faithfulness.threshold, + action: 'warn' + }, + status: toStatus(fiddlerErr, startedAt) } } } /** * Validate output text (safety + PII + faithfulness checks in parallel) - * Returns combined validation result + * Returns combined validation result. * * Key Differences from Input Validation: * - Never blocks (always fail-open) - * - Includes faithfulness check (if context provided) + * - Includes faithfulness check (if context AND prompt provided) * - Default action is 'warn' instead of 'block' * - * @param text - AI-generated output to validate + * @param text - AI-generated output to validate * @param context - Optional RAG context for faithfulness checking + * @param prompt - Optional originating user prompt (required by Fiddler's + * faithfulness endpoint; if omitted faithfulness is skipped) * @returns Output validation result with all violations */ - public async validateOutput(text: string, context?: string): Promise { - try { - // Build array of check promises - const checks: Promise[] = [this.evaluateSafety(text), this.detectPII(text)] + public async validateOutput(text: string, context?: string, prompt?: string): Promise { + const promises: Promise[] = [this.evaluateSafety(text), this.detectPII(text)] + if (context && prompt && this.config.faithfulness.enabled) { + promises.push(this.evaluateFaithfulness(text, context, prompt)) + } - // Only add faithfulness check if context is provided - if (context && this.config.faithfulness.enabled) { - checks.push(this.evaluateFaithfulness(text, context)) - } + const results = await Promise.all(promises) + const [safety, pii, faith] = results as [ + { result: Partial; status: GuardrailStageStatus }, + { result: PIIDetectionResult; status: GuardrailStageStatus }, + ( + | { + result: { score: number; threshold: number; action: GuardrailAction } + status: GuardrailStageStatus + } + | undefined + ) + ] - // Run all checks in parallel for performance - const results = await Promise.all(checks) - const [safetyResult, piiResult, faithfulnessResult] = results - - // Output validation never replaces (Phase 5: warn only) - const shouldReplace = false - const shouldRedact = piiResult.redactedText !== undefined - - // Build result - const result: OutputValidationResult = { - replaced: shouldReplace, - redacted: shouldRedact, - redactedText: piiResult.redactedText, - violations: { - safety: safetyResult.violations, - pii: piiResult.detections - } - } + const safetyResult = safety.result + const piiResult = pii.result - // Add faithfulness violations if checked - if (faithfulnessResult) { - result.violations.faithfulness = { - score: faithfulnessResult.score, - threshold: faithfulnessResult.threshold - } - } + const shouldRedact = piiResult.redactedText !== undefined - return result - } catch (error) { - // Fail-open: on error, allow the output unchanged - return { - replaced: false, - redacted: false, - violations: {} + const result: OutputValidationResult = { + replaced: false, // Output validation never replaces (warn-only posture) + redacted: shouldRedact, + redactedText: piiResult.redactedText, + violations: { + safety: safetyResult.violations, + pii: piiResult.detections + }, + status: mergeStageStatus([safety.status, pii.status, faith?.status].filter(Boolean) as GuardrailStageStatus[]) + } + + if (faith) { + result.violations.faithfulness = { + score: faith.result.score, + threshold: faith.result.threshold } } + + return result } /** @@ -600,17 +753,79 @@ export class FiddlerGuardrailsService { } /** - * Health check + * Structured health check. Returns a normalized status suitable for display + * in the admin selftest endpoint. Probes only the safety endpoint — + * call `capabilityMatrix()` for a per-endpoint plan-tier report. */ - public async healthCheck(): Promise { + public async healthCheck(): Promise { + const startedAt = Date.now() try { - // Simple test call to verify API connectivity - await this.post('/v3/guardrails/ftl-safety', { - prompt: 'test' - }) - return true + await this.post('/v3/guardrails/ftl-safety', { data: { input: 'ping' } }) + return { ...HEALTHY_STATUS, latencyMs: Date.now() - startedAt } } catch (error) { - return false + const err = error instanceof FiddlerError ? error : toFiddlerError(error) + return toStatus(err, startedAt) } } + + /** + * Live probe of all three guardrail endpoints, classifying each as `ok`, + * `unsupported` (plan-tier 404), or degraded (with reason). Used by the + * selftest endpoint and admin UI to render a capability matrix so admins + * can see at a glance which guardrails their Fiddler plan actually + * supports — instead of toggling them on and finding out later via a + * per-message banner. + * + * Faithfulness uses minimal valid inputs Fiddler accepts (200 OK or 504 + * "service busy"); we treat 504 here as inconclusive but capable. + */ + public async capabilityMatrix(): Promise<{ + safety: GuardrailStageStatus + pii: GuardrailStageStatus + faithfulness: GuardrailStageStatus + }> { + const probe = async (endpoint: string, payload: any): Promise => { + const startedAt = Date.now() + try { + await this.post(endpoint, payload) + return { ...HEALTHY_STATUS, latencyMs: Date.now() - startedAt } + } catch (error) { + const err = error instanceof FiddlerError ? error : toFiddlerError(error) + return toStatus(err, startedAt) + } + } + + // Run sequentially so a 401/network failure short-circuits the breaker + // before we hammer Fiddler with three failing requests in parallel. + const safety = await probe('/v3/guardrails/ftl-safety', { data: { input: 'ping' } }) + const pii = await probe('/v3/guardrails/sensitive-information', { data: { input: 'ping' } }) + const faithfulness = await probe('/v3/guardrails/ftl-response-faithfulness', { + data: { prompt: 'ping', response: 'ping', context: 'ping' } + }) + return { safety, pii, faithfulness } + } + + /** + * Expose resolved config for diagnostic endpoints. + */ + public getConfig(): GuardrailsConfig { + return this.config + } +} + +/** + * Merge multiple stage statuses into a single summary status. + * Any degraded sub-stage marks the whole stage as degraded, and the first + * non-ok reason wins (preserves specificity over generic 'ok'). + */ +function mergeStageStatus(statuses: GuardrailStageStatus[]): GuardrailStageStatus { + if (statuses.length === 0) return HEALTHY_STATUS + + const degraded = statuses.find((s) => s.degraded) + if (degraded) return degraded + + // Prefer the first non-disabled_stage status so UI shows a concrete reason + const concrete = statuses.find((s) => s.reason === 'ok') || statuses[0] + const totalLatency = statuses.reduce((sum, s) => sum + (s.latencyMs || 0), 0) + return { ...concrete, latencyMs: totalLatency || concrete.latencyMs } } diff --git a/packages/server/src/services/guardrails/config.ts b/packages/server/src/services/guardrails/config.ts index 7f8e6ecbb2d..a8421dd5821 100644 --- a/packages/server/src/services/guardrails/config.ts +++ b/packages/server/src/services/guardrails/config.ts @@ -13,7 +13,22 @@ import { getRunningExpressApp } from '../../utils/getRunningExpressApp' import { Organization } from '../../database/entities/Organization' import { ChatFlow } from '../../database/entities/ChatFlow' -import { GuardrailsConfig, DEFAULT_GUARDRAILS_CONFIG, OrganizationConfig, ChatflowConfig, GuardrailAction } from '../../types/guardrails' +import { + GuardrailsConfig, + DEFAULT_GUARDRAILS_CONFIG, + OrganizationConfig, + ChatflowConfig, + GuardrailAction, + GuardrailFailureMode +} from '../../types/guardrails' + +const parseFailureMode = (raw: string | undefined): GuardrailFailureMode | undefined => { + if (!raw) return undefined + const v = raw.trim().toLowerCase() + if (v === 'closed' || v === 'fail-closed' || v === 'fail_closed') return 'closed' + if (v === 'open' || v === 'fail-open' || v === 'fail_open') return 'open' + return undefined +} /** * Load configuration from environment variables @@ -24,9 +39,13 @@ export function getEnvironmentConfig(): Partial { const piiThreshold = parseFloat(process.env.FIDDLER_PII_THRESHOLD || '0.8') // Fiddler faithfulness uses inverted scale: < 0.005 = unfaithful const faithfulnessThreshold = parseFloat(process.env.FIDDLER_FAITHFULNESS_THRESHOLD || '0.005') + const failureMode = parseFailureMode(process.env.FIDDLER_FAILURE_MODE) || 'open' + const observabilityOnly = process.env.FIDDLER_OBSERVABILITY_ONLY === 'true' return { enabled, + failureMode, + observabilityOnly, safety: { enabled: process.env.FIDDLER_SAFETY_ENABLED !== 'false', threshold: safetyThreshold, @@ -123,6 +142,16 @@ export function deepMergeConfigs(base: Partial, override: Part merged.credentialId = override.credentialId } + // Merge failure policy + if (override.failureMode !== undefined) { + merged.failureMode = override.failureMode + } + + // Merge observability-only shadow mode + if (override.observabilityOnly !== undefined) { + merged.observabilityOnly = override.observabilityOnly + } + // Merge safety config if (override.safety) { merged.safety = { diff --git a/packages/server/src/services/guardrails/errors.ts b/packages/server/src/services/guardrails/errors.ts new file mode 100644 index 00000000000..e0eb62d10e6 --- /dev/null +++ b/packages/server/src/services/guardrails/errors.ts @@ -0,0 +1,120 @@ +/** + * Typed errors for Fiddler guardrail evaluation failures. + * + * These exist so the runtime can distinguish "guardrail said content is unsafe" + * (a real block) from "guardrail could not be evaluated" (a degraded state whose + * handling depends on failureMode). Previously the service swallowed every + * failure and returned fake-safe results, which is how enforcement silently + * degraded in production when a key rotated or the upstream returned 401. + */ + +import { GuardrailHealthReason } from '../../types/guardrails' + +export class FiddlerError extends Error { + public readonly reason: GuardrailHealthReason + public readonly httpStatus?: number + + constructor(reason: GuardrailHealthReason, message: string, httpStatus?: number) { + super(message) + this.name = 'FiddlerError' + this.reason = reason + this.httpStatus = httpStatus + } +} + +export class FiddlerAuthError extends FiddlerError { + constructor(message: string = 'Fiddler rejected the API key (401)', httpStatus: number = 401) { + super('auth_error', message, httpStatus) + this.name = 'FiddlerAuthError' + } +} + +export class FiddlerUpstreamError extends FiddlerError { + constructor(message: string, httpStatus?: number) { + super('api_error', message, httpStatus) + this.name = 'FiddlerUpstreamError' + } +} + +/** + * Thrown when Fiddler responds with HTTP 404 + a body indicating that the + * requested guardrail endpoint is not included in the caller's plan tier + * (e.g. PII detection on the freemium API). This is a stable capability + * gap — the API responded normally, it just told us we don't have access. + * + * Treated as `ok=true, degraded=false` upstream (per-message) so end users + * are not spammed with a banner on every chat for a config-level constraint. + * The admin UI surfaces this once via the selftest capability matrix. + */ +export class FiddlerUnsupportedError extends FiddlerError { + public readonly endpoint: string + + constructor(message: string, endpoint: string, httpStatus: number = 404) { + super('unsupported', message, httpStatus) + this.name = 'FiddlerUnsupportedError' + this.endpoint = endpoint + } +} + +export class FiddlerTimeoutError extends FiddlerError { + constructor(message: string = 'Fiddler request timed out') { + super('timeout', message) + this.name = 'FiddlerTimeoutError' + } +} + +export class FiddlerNetworkError extends FiddlerError { + constructor(message: string) { + super('network_error', message) + this.name = 'FiddlerNetworkError' + } +} + +export class FiddlerCircuitOpenError extends FiddlerError { + constructor(message: string = 'Fiddler circuit breaker is open') { + super('circuit_open', message) + this.name = 'FiddlerCircuitOpenError' + } +} + +/** + * Classify an arbitrary thrown value into a FiddlerError. Idempotent: if the + * input is already a FiddlerError it is returned as-is. + */ +export const toFiddlerError = (err: unknown): FiddlerError => { + if (err instanceof FiddlerError) return err + + // axios errors + const axiosLike = err as any + if (axiosLike?.isAxiosError) { + const status: number | undefined = axiosLike.response?.status + const body = axiosLike.response?.data + const message: string = body?.error?.message || axiosLike.message || 'Fiddler upstream error' + const errorReason: string | undefined = body?.error?.errors?.[0]?.reason + const requestPath: string = axiosLike.config?.url || '' + + if (axiosLike.code === 'ECONNABORTED' || axiosLike.code === 'ETIMEDOUT') { + return new FiddlerTimeoutError(message) + } + if (axiosLike.code === 'ECONNREFUSED' || axiosLike.code === 'ENOTFOUND' || axiosLike.code === 'EAI_AGAIN') { + return new FiddlerNetworkError(message) + } + if (status === 401 || status === 403) { + return new FiddlerAuthError(message, status) + } + // Plan-tier capability gap: Fiddler returns 404 with a "not supported" + // body for endpoints the caller's plan doesn't include (e.g. freemium + // PII detection). Distinguish from generic 4xx so we can short-circuit + // future calls and avoid spamming users with degraded banners. + if (status === 404 && (errorReason === 'NotFound' || /not supported/i.test(message))) { + return new FiddlerUnsupportedError(message, requestPath, status) + } + if (status !== undefined) { + return new FiddlerUpstreamError(message, status) + } + return new FiddlerNetworkError(message) + } + + const msg = err instanceof Error ? err.message : String(err) + return new FiddlerError('unexpected', msg) +} diff --git a/packages/server/src/services/guardrails/extractContext.ts b/packages/server/src/services/guardrails/extractContext.ts new file mode 100644 index 00000000000..3a0b7b49423 --- /dev/null +++ b/packages/server/src/services/guardrails/extractContext.ts @@ -0,0 +1,22 @@ +/** + * Extract RAG source-document text for faithfulness checking. + * + * Moved out of buildChatflow.ts so both the chain path and the AgentFlow V2 + * path can share a single implementation. Handles both JSON-string and + * already-parsed-array inputs. + */ + +import logger from '../../utils/logger' + +export const extractTextFromSourceDocuments = (sourceDocuments: any): string | undefined => { + try { + const docs = typeof sourceDocuments === 'string' ? JSON.parse(sourceDocuments) : sourceDocuments + if (!Array.isArray(docs) || docs.length === 0) return undefined + const textChunks = docs.map((doc) => doc?.pageContent || '').filter((text: string) => text.length > 0) + if (textChunks.length === 0) return undefined + return textChunks.join('\n\n') + } catch (error) { + logger.warn('[Guardrails] Failed to extract text from sourceDocuments', { error: (error as Error).message }) + return undefined + } +} diff --git a/packages/server/src/services/guardrails/index.ts b/packages/server/src/services/guardrails/index.ts index 09cef8cc33f..e67db3cc5d6 100644 --- a/packages/server/src/services/guardrails/index.ts +++ b/packages/server/src/services/guardrails/index.ts @@ -8,7 +8,7 @@ export { CircuitBreaker, CircuitState } from './CircuitBreaker' export type { CircuitBreakerConfig } from './CircuitBreaker' export { FiddlerGuardrailsService } from './FiddlerGuardrailsService' -export type { FiddlerCredentials } from './FiddlerGuardrailsService' +export type { FiddlerCredentials, FiddlerCredentialSource } from './FiddlerGuardrailsService' export { GuardrailsCache, getGuardrailsCache } from './cache' @@ -20,3 +20,7 @@ export { getGuardrailsConfig, validateConfig } from './config' + +export * from './errors' +export { runInputStage, runOutputStage } from './runStage' +export type { RunStageResult } from './runStage' diff --git a/packages/server/src/services/guardrails/runStage.ts b/packages/server/src/services/guardrails/runStage.ts new file mode 100644 index 00000000000..79bd56ee73f --- /dev/null +++ b/packages/server/src/services/guardrails/runStage.ts @@ -0,0 +1,329 @@ +/** + * Shared guardrail stage runner. + * + * Centralizes the decision logic for a single input- or output-validation + * stage so all call sites (chain input/output, agent input/output, agentflow + * v2 input/output) behave identically: + * + * 1. Resolve the service + config (env -> org -> chatflow). + * 2. If disabled or no credentials, emit a health entry describing why and + * either (open) allow the request to continue or (closed) signal the + * caller to block with a 503. + * 3. If the service is healthy, call validateInput/validateOutput and + * classify the returned status. Real `blocked=true` violations remain 400. + * Degraded calls produce either a warning (open) or a block (closed). + * + * The helper returns a typed `RunStageResult` rather than throwing so the + * caller can merge into its existing `guardrailsMetadata` object and decide + * how to propagate the 503 (streaming vs non-streaming paths need different + * handling). + */ + +import { StatusCodes } from 'http-status-codes' +import { InternalFlowiseError } from '../../errors/internalFlowiseError' +import logger from '../../utils/logger' +import { FiddlerGuardrailsService } from './FiddlerGuardrailsService' +import { + GuardrailFailureMode, + GuardrailsConfig, + GuardrailStageStatus, + InputValidationResult, + OutputValidationResult +} from '../../types/guardrails' +import type { GuardrailsHealthEntry } from '../../Interface' + +export interface RunStageContext { + chatflowId: string + workspaceId?: string + organizationId?: string + chatId?: string +} + +export type RunStageResult = + | { + kind: 'ok' + failureMode: GuardrailFailureMode + health: GuardrailsHealthEntry + // Present only if guardrails ran (not when disabled/no-creds) + inputResult?: InputValidationResult + outputResult?: OutputValidationResult + } + | { + kind: 'blocked_violation' // real 400 (content actually unsafe) + failureMode: GuardrailFailureMode + health: GuardrailsHealthEntry + inputResult: InputValidationResult + message: string + } + | { + kind: 'blocked_degraded' // 503 fail-closed on degraded + failureMode: 'closed' + health: GuardrailsHealthEntry + reason: string + } + +const toHealthEntry = (status: GuardrailStageStatus): GuardrailsHealthEntry => ({ + ok: status.ok, + degraded: status.degraded, + reason: status.reason, + httpStatus: status.httpStatus, + message: status.message, + latencyMs: status.latencyMs +}) + +const healthOk = (reason: GuardrailStageStatus['reason'] = 'ok'): GuardrailsHealthEntry => ({ + ok: true, + degraded: false, + reason +}) + +const degradedFrom = (reason: GuardrailStageStatus['reason'], message?: string, httpStatus?: number): GuardrailsHealthEntry => ({ + ok: false, + degraded: true, + reason, + message, + httpStatus +}) + +/** + * Runs the input stage. `text` is the user question. + * + * - Returns `kind: 'ok'` if the request may proceed. `inputResult` carries + * violations/redactions for the caller to act on (e.g. swap in redacted text). + * - Returns `kind: 'blocked_violation'` when Fiddler reports a real block. + * Caller should throw 400. + * - Returns `kind: 'blocked_degraded'` only in fail-closed mode when the stage + * could not be evaluated. Caller should throw 503. + */ +export async function runInputStage(text: string, ctx: RunStageContext): Promise { + const { chatflowId, workspaceId, organizationId, chatId } = ctx + + if (!workspaceId) { + // No workspace context means we literally cannot resolve credentials. + // Treat as disabled-for-this-request; never fail-closed on a missing + // workspace id because that's a platform-level bug, not a guardrail + // outage — we don't want to mask plumbing failures as safety blocks. + return { + kind: 'ok', + failureMode: 'open', + health: healthOk('disabled') + } + } + + const resolution = await FiddlerGuardrailsService.resolveFromContext(chatflowId, workspaceId, organizationId) + const { service, config } = resolution + const failureMode: GuardrailFailureMode = config.failureMode || 'open' + + // Not enabled: legitimate skip, no banner. + if (resolution.reason === 'disabled') { + return { kind: 'ok', failureMode, health: healthOk('disabled') } + } + + // Enabled but could not resolve a credential OR unexpected error loading config. + if (!service) { + const reason: GuardrailsHealthEntry['reason'] = resolution.reason === 'no_credentials' ? 'no_credentials' : 'unexpected' + const health = degradedFrom(reason, resolution.error?.message) + + logger.error('[Guardrails] Stage unavailable (input)', { + chatflowId, + chatId, + workspaceId, + organizationId, + reason, + failureMode, + error: resolution.error?.message + }) + + if (failureMode === 'closed') { + return { kind: 'blocked_degraded', failureMode: 'closed', health, reason } + } + return { kind: 'ok', failureMode, health } + } + + try { + const result = await service.validateInput(text) + const health = toHealthEntry(result.status) + + if (result.status.degraded) { + logger.error('[Guardrails] Stage degraded (input)', { + chatflowId, + chatId, + workspaceId, + organizationId, + reason: result.status.reason, + httpStatus: result.status.httpStatus, + latencyMs: result.status.latencyMs, + failureMode + }) + + if (failureMode === 'closed') { + return { + kind: 'blocked_degraded', + failureMode: 'closed', + health, + reason: result.status.reason + } + } + // Fail-open: allow request to proceed but surface the degraded state + return { kind: 'ok', failureMode, health, inputResult: result } + } + + if (result.blocked) { + logger.info('[Guardrails] Blocking content violation (input)', { + chatflowId, + chatId, + workspaceId, + safetyViolations: result.violations.safety?.map((v) => `${v.dimension}(${v.score.toFixed(2)})`), + piiDetections: result.violations.pii?.map((p) => `${p.label}(${p.score.toFixed(2)})`) + }) + return { + kind: 'blocked_violation', + failureMode, + health, + inputResult: result, + message: result.message || 'Content blocked by guardrails' + } + } + + logger.info('[Guardrails] Input validation passed', { + chatflowId, + chatId, + workspaceId, + redacted: result.redacted, + safetyViolations: result.violations.safety?.map((v) => `${v.dimension}(${v.score.toFixed(2)})`), + piiDetections: result.violations.pii?.map((p) => `${p.label}(${p.score.toFixed(2)})`) + }) + + return { kind: 'ok', failureMode, health, inputResult: result } + } catch (error) { + // Defensive: validateInput is designed not to throw, but if any future + // refactor regresses that contract, fall back to the failureMode path. + const msg = (error as Error).message + logger.error('[Guardrails] Unexpected throw in validateInput (fail-mode engaged)', { + chatflowId, + chatId, + error: msg, + failureMode + }) + const health = degradedFrom('unexpected', msg) + if (failureMode === 'closed') { + return { kind: 'blocked_degraded', failureMode: 'closed', health, reason: 'unexpected' } + } + return { kind: 'ok', failureMode, health } + } +} + +/** + * Runs the output stage. `text` is the LLM response, `context` is optional RAG + * source material for faithfulness checking, and `prompt` is the originating + * user question (required by Fiddler's faithfulness endpoint). + * + * Output validation does not block real violations (warn-only posture for + * content already generated), but fail-closed on a degraded upstream still + * returns `blocked_degraded` so callers can emit a trailing 503/error event. + */ +export async function runOutputStage( + text: string, + context: string | undefined, + ctx: RunStageContext & { prompt?: string } +): Promise { + const { chatflowId, workspaceId, organizationId, chatId } = ctx + + if (!workspaceId) { + return { kind: 'ok', failureMode: 'open', health: healthOk('disabled') } + } + + const resolution = await FiddlerGuardrailsService.resolveFromContext(chatflowId, workspaceId, organizationId) + const { service, config } = resolution + const failureMode: GuardrailFailureMode = config.failureMode || 'open' + + if (resolution.reason === 'disabled') { + return { kind: 'ok', failureMode, health: healthOk('disabled') } + } + + if (!service) { + const reason: GuardrailsHealthEntry['reason'] = resolution.reason === 'no_credentials' ? 'no_credentials' : 'unexpected' + const health = degradedFrom(reason, resolution.error?.message) + + logger.error('[Guardrails] Stage unavailable (output)', { + chatflowId, + chatId, + workspaceId, + organizationId, + reason, + failureMode + }) + + if (failureMode === 'closed') { + return { kind: 'blocked_degraded', failureMode: 'closed', health, reason } + } + return { kind: 'ok', failureMode, health } + } + + try { + const result = await service.validateOutput(text, context, ctx.prompt) + const health = toHealthEntry(result.status) + + if (result.status.degraded) { + logger.error('[Guardrails] Stage degraded (output)', { + chatflowId, + chatId, + workspaceId, + organizationId, + reason: result.status.reason, + httpStatus: result.status.httpStatus, + latencyMs: result.status.latencyMs, + failureMode + }) + + if (failureMode === 'closed') { + return { + kind: 'blocked_degraded', + failureMode: 'closed', + health, + reason: result.status.reason + } + } + } else { + logger.info('[Guardrails] Output validation completed', { + chatflowId, + chatId, + workspaceId, + redacted: result.redacted, + faithfulnessScore: result.violations.faithfulness?.score, + safetyViolations: result.violations.safety?.map((v) => `${v.dimension}(${v.score.toFixed(2)})`), + piiDetections: result.violations.pii?.map((p) => `${p.label}(${p.score.toFixed(2)})`) + }) + } + + return { kind: 'ok', failureMode, health, outputResult: result } + } catch (error) { + const msg = (error as Error).message + logger.error('[Guardrails] Unexpected throw in validateOutput', { + chatflowId, + chatId, + error: msg, + failureMode + }) + const health = degradedFrom('unexpected', msg) + if (failureMode === 'closed') { + return { kind: 'blocked_degraded', failureMode: 'closed', health, reason: 'unexpected' } + } + return { kind: 'ok', failureMode, health } + } +} + +/** + * Stable, copy-paste-friendly 503 error for fail-closed outcomes. Use as the + * thrown error in any caller that wants uniform wire-format. + */ +export const buildFailClosedError = (reason: string): InternalFlowiseError => + new InternalFlowiseError( + StatusCodes.SERVICE_UNAVAILABLE, + `Safety checks are temporarily unavailable (reason: ${reason}). Please try again in a moment.` + ) + +/** + * Legacy type re-export to avoid `unused` lint on the import above. + */ +export type { GuardrailsConfig } diff --git a/packages/server/src/services/guardrails/selftest.ts b/packages/server/src/services/guardrails/selftest.ts new file mode 100644 index 00000000000..e2e60f6ac0c --- /dev/null +++ b/packages/server/src/services/guardrails/selftest.ts @@ -0,0 +1,140 @@ +/** + * Guardrails selftest service. + * + * Produces a structured report describing: (a) resolved config for the given + * chatflow/organization (env -> org -> chatflow merged), (b) provenance of + * any credentials that could be loaded, (c) a live healthCheck() call with + * HTTP status + latency, and (d) the current circuit-breaker state. + * + * Purpose: let operators curl a single endpoint to answer "is guardrails + * actually wired up in this environment?" without grepping server logs. + */ + +import { getRunningExpressApp } from '../../utils/getRunningExpressApp' +import { ChatFlow } from '../../database/entities/ChatFlow' +import { Workspace } from '../../enterprise/database/entities/workspace.entity' +import { FiddlerGuardrailsService } from './FiddlerGuardrailsService' +import { getGuardrailsConfig } from './config' +import type { GuardrailsConfig, GuardrailStageStatus } from '../../types/guardrails' +import type { FiddlerCredentialSource } from './FiddlerGuardrailsService' + +export interface GuardrailsSelftestReport { + resolved: { + chatflowId?: string + workspaceId?: string + organizationId?: string + } + config: GuardrailsConfig + credentialSource: FiddlerCredentialSource + healthCheck: GuardrailStageStatus | { ok: false; degraded: true; reason: 'disabled' | 'no_credentials' | 'unexpected' } + /** + * Per-endpoint capability matrix. Lets admins see which guardrails their + * Fiddler plan tier actually supports without trial-and-error from the + * chat UI. Populated only when a live service could be constructed. + */ + capabilities?: { + safety: GuardrailStageStatus + pii: GuardrailStageStatus + faithfulness: GuardrailStageStatus + } + circuitState?: 'CLOSED' | 'OPEN' | 'HALF_OPEN' + notes: string[] +} + +export async function runGuardrailsSelftest(options: { + organizationId: string + chatflowId?: string + workspaceIdOverride?: string +}): Promise { + const notes: string[] = [] + const appServer = getRunningExpressApp() + + // Resolve a workspaceId — either explicit, or from the chatflow, or fall back to first workspace in org. + let workspaceId = options.workspaceIdOverride + if (!workspaceId && options.chatflowId) { + const cf = await appServer.AppDataSource.getRepository(ChatFlow).findOne({ + where: { id: options.chatflowId } + }) + workspaceId = cf?.workspaceId + if (!workspaceId) notes.push(`Chatflow ${options.chatflowId} has no workspaceId; falling back to organization workspace lookup.`) + } + if (!workspaceId) { + const workspace = await appServer.AppDataSource.getRepository(Workspace).findOne({ + where: { organizationId: options.organizationId } + }) + workspaceId = workspace?.id + if (!workspaceId) { + notes.push(`No workspace found for organization ${options.organizationId}. Cannot resolve workspace-scoped credentials.`) + } + } + + const config = await getGuardrailsConfig(options.chatflowId || '', options.organizationId) + + if (!config.enabled) { + return { + resolved: { chatflowId: options.chatflowId, workspaceId, organizationId: options.organizationId }, + config, + credentialSource: 'none', + healthCheck: { ok: false, degraded: true, reason: 'disabled' }, + notes: ['Guardrails are disabled at the resolved config level.', ...notes] + } + } + + if (!workspaceId) { + return { + resolved: { chatflowId: options.chatflowId, workspaceId, organizationId: options.organizationId }, + config, + credentialSource: 'none', + healthCheck: { ok: false, degraded: true, reason: 'unexpected' }, + notes: ['Could not determine a workspaceId; credential lookup skipped.', ...notes] + } + } + + const { credentials, source } = await FiddlerGuardrailsService.loadCredentials(workspaceId, options.organizationId, config) + + if (!credentials) { + return { + resolved: { chatflowId: options.chatflowId, workspaceId, organizationId: options.organizationId }, + config, + credentialSource: 'none', + healthCheck: { ok: false, degraded: true, reason: 'no_credentials' }, + notes: [ + 'Guardrails are enabled but no credentials could be resolved. Set FIDDLER_API_KEY/FIDDLER_API_URL, or create a workspace/organization credential named "fiddlerApi".', + ...notes + ] + } + } + + // Instantiate a throwaway service + run a live capability matrix. We + // intentionally reconstruct here rather than using createFromContext so we + // can capture the credential source for the report. Capability matrix + // probes all three guardrail endpoints so admins can see which guardrails + // their Fiddler plan tier supports. + const service = new FiddlerGuardrailsService(credentials, config) + const capabilities = await service.capabilityMatrix() + // Backwards-compat: top-level healthCheck mirrors safety status (the + // canonical "is Fiddler reachable at all?" signal). + const healthCheck = capabilities.safety + const circuitState = service.getCircuitStatus().state + + if (capabilities.pii.reason === 'unsupported') { + notes.push( + 'PII detection (sensitive-information) is not included in this Fiddler plan tier; PII checks will be skipped silently at runtime.' + ) + } + if (capabilities.faithfulness.reason === 'unsupported') { + notes.push( + 'Faithfulness (ftl-response-faithfulness) is not included in this Fiddler plan tier; RAG hallucination checks will be skipped silently at runtime.' + ) + } + + return { + resolved: { chatflowId: options.chatflowId, workspaceId, organizationId: options.organizationId }, + config, + credentialSource: source, + healthCheck, + capabilities, + circuitState, + notes + } +} diff --git a/packages/server/src/types/guardrails.ts b/packages/server/src/types/guardrails.ts index a138d0d863d..e2326637d5c 100644 --- a/packages/server/src/types/guardrails.ts +++ b/packages/server/src/types/guardrails.ts @@ -49,6 +49,46 @@ export type PIIType = export type GuardrailAction = 'block' | 'redact' | 'replace' | 'warn' | 'continue' +// ============================================================================ +// Failure Semantics +// ============================================================================ + +/** + * How the runtime should react when a guardrail stage cannot be evaluated + * (missing credential, upstream outage, auth error, timeout, circuit open). + * + * - 'open' = continue to the LLM, annotate response metadata with degraded=true. + * Admins/users see an amber banner; ops sees structured logs + Langfuse spans. + * - 'closed' = stop the request cold (503) so the chat cannot proceed unprotected. + */ +export type GuardrailFailureMode = 'open' | 'closed' + +/** + * Machine-readable reason why a guardrail stage was (or wasn't) evaluated. + * Stable enum so alerting rules and UI banners can switch on it. + */ +export type GuardrailHealthReason = + | 'ok' // evaluated successfully + | 'disabled' // config.enabled === false + | 'disabled_stage' // stage-level disabled (safety/pii/faithfulness) + | 'no_credentials' // no API key resolvable + | 'auth_error' // upstream 401/403 + | 'api_error' // upstream non-auth error (4xx/5xx) + | 'timeout' // request exceeded timeout + | 'circuit_open' // circuit breaker rejected call + | 'network_error' // connection failure + | 'unsupported' // Fiddler plan tier does not include this guardrail (404 NotFound) + | 'unexpected' // unhandled throw; bug-class + +export interface GuardrailStageStatus { + ok: boolean + degraded: boolean + reason: GuardrailHealthReason + httpStatus?: number + message?: string + latencyMs?: number +} + // ============================================================================ // Safety Configuration // ============================================================================ @@ -216,6 +256,20 @@ export interface GuardrailsConfig { */ credentialId?: string + /** + * Failure policy when guardrails cannot be evaluated. + * @default 'open' + */ + failureMode: GuardrailFailureMode + + /** + * Observability-only mode: when true, violations are recorded in metadata + * but never block the request (even 200-OK violations from Fiddler). + * Intended for shadow/canary rollouts before flipping enforcement on. + * @default false + */ + observabilityOnly?: boolean + /** * Safety checks configuration */ @@ -347,6 +401,11 @@ export interface InputValidationResult { safety?: SafetyViolation[] pii?: PIIDetection[] } + /** + * Per-stage health for this evaluation. + * If any sub-stage (safety/pii) degraded, status.degraded=true. + */ + status: GuardrailStageStatus } export interface OutputValidationResult { @@ -363,6 +422,7 @@ export interface OutputValidationResult { threshold: number } } + status: GuardrailStageStatus } // ============================================================================ @@ -371,6 +431,8 @@ export interface OutputValidationResult { export const DEFAULT_GUARDRAILS_CONFIG: GuardrailsConfig = { enabled: true, + failureMode: 'open', + observabilityOnly: false, safety: { enabled: true, threshold: 0.1, diff --git a/packages/server/src/utils/buildAgentflow.ts b/packages/server/src/utils/buildAgentflow.ts index bd4a1acf4ab..487df5c19d1 100644 --- a/packages/server/src/utils/buildAgentflow.ts +++ b/packages/server/src/utils/buildAgentflow.ts @@ -32,7 +32,8 @@ import { INodeOverrides, IVariableOverride, INodeDirectedGraph, - IUser + IUser, + GuardrailsMetadata } from '../Interface' import { RUNTIME_MESSAGES_LENGTH_VAR_PREFIX, @@ -62,6 +63,8 @@ import { CachePool } from '../CachePool' import { ChatMessage } from '../database/entities/ChatMessage' import { Telemetry } from './telemetry' import { DEFAULT_CUSTOMER_ID, OVERRIDE_CUSTOMER_ID } from '../aai-utils/billing/config' +import { runOutputStage } from '../services/guardrails/runStage' +import { extractTextFromSourceDocuments } from '../services/guardrails/extractContext' import { getWorkspaceSearchOptions } from '../enterprise/utils/ControllerServiceUtils' import { UsageCacheManager } from '../UsageCacheManager' import { generateTTSForResponseStream, shouldAutoPlayTTS } from './buildChatflow' @@ -1603,7 +1606,8 @@ export const executeAgentFlow = async ({ orgId, workspaceId, subscriptionId, - productId + productId, + guardrailsMetadata: incomingGuardrailsMetadata }: IExecuteAgentFlowParams) => { logger.debug('\n🚀 Starting flow execution') @@ -2309,6 +2313,59 @@ export const executeAgentFlow = async ({ } } + /* Output validation with Fiddler Guardrails (AgentFlow V2). + * Agentflow V2 previously had zero output-side guardrail coverage. We run + * the output stage here and merge into any inputValidation metadata that + * upstream executeFlow carried in. Fail-closed on degraded upstream still + * emits a trailing SSE error; content already streamed cannot be un-streamed. + */ + let guardrailsMetadata: Partial | undefined = incomingGuardrailsMetadata + { + const outputContext = lastNodeOutput?.sourceDocuments ? extractTextFromSourceDocuments(lastNodeOutput.sourceDocuments) : undefined + const outStage = await runOutputStage(content, outputContext, { + chatflowId: chatflowid, + workspaceId, + organizationId: orgId, + chatId, + prompt: finalUserInput + }) + + const existingHealth = guardrailsMetadata?.health + guardrailsMetadata = { + ...guardrailsMetadata, + health: { + mode: outStage.failureMode, + anyDegraded: (existingHealth?.anyDegraded ?? false) || outStage.health.degraded, + input: existingHealth?.input, + output: outStage.health + } + } + + if (outStage.kind === 'ok' && outStage.outputResult) { + guardrailsMetadata.outputValidation = { + blocked: false, + redacted: outStage.outputResult.redacted, + faithfulnessScore: outStage.outputResult.violations.faithfulness?.score, + violations: { + safety: outStage.outputResult.violations.safety, + pii: outStage.outputResult.violations.pii + } + } + } + + if (outStage.kind === 'blocked_degraded') { + sseStreamer?.streamErrorEvent( + chatId, + `Safety checks were unavailable (reason: ${outStage.reason}). This response bypassed guardrail enforcement.` + ) + logger.error('[Guardrails] Agentflow v2 output degraded fail-closed', { + chatflowId: chatflowid, + chatId, + reason: outStage.reason + }) + } + } + const userMessage: Omit = { role: 'userMessage', content: finalUserInput, @@ -2319,7 +2376,8 @@ export const executeAgentFlow = async ({ createdDate: userMessageDateTime, fileUploads: uploads ? JSON.stringify(fileUploads) : undefined, leadEmail: incomingInput.leadEmail, - executionId: newExecution.id + executionId: newExecution.id, + guardrailsMetadata: guardrailsMetadata ? JSON.stringify(guardrailsMetadata) : undefined } await utilAddChatMessage(userMessage, appDataSource) @@ -2331,7 +2389,8 @@ export const executeAgentFlow = async ({ chatType: evaluationRunId ? ChatType.EVALUATION : isInternal ? ChatType.INTERNAL : ChatType.EXTERNAL, chatId, sessionId, - executionId: newExecution.id + executionId: newExecution.id, + guardrailsMetadata: guardrailsMetadata ? JSON.stringify(guardrailsMetadata) : undefined } if (lastNodeOutput?.sourceDocuments) apiMessage.sourceDocuments = JSON.stringify(lastNodeOutput.sourceDocuments) if (lastNodeOutput?.usedTools) apiMessage.usedTools = JSON.stringify(lastNodeOutput.usedTools) @@ -2398,6 +2457,8 @@ export const executeAgentFlow = async ({ result.executionId = newExecution.id result.agentFlowExecutedData = agentFlowExecutedData + if (guardrailsMetadata) result.guardrailsMetadata = guardrailsMetadata + if (sessionId) result.sessionId = sessionId if (shouldAutoPlayTTS(chatflow.textToSpeech) && result.text) { diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts index db54fae51a3..2e130fa12f0 100644 --- a/packages/server/src/utils/buildChatflow.ts +++ b/packages/server/src/utils/buildChatflow.ts @@ -46,7 +46,6 @@ import { ChatFlow } from '../database/entities/ChatFlow' import { ChatMessage } from '../database/entities/ChatMessage' import { Variable } from '../database/entities/Variable' import { getRunningExpressApp } from '../utils/getRunningExpressApp' -import { FiddlerGuardrailsService } from '../services/guardrails/FiddlerGuardrailsService' import { isFlowValidForStream, buildFlow, @@ -81,6 +80,8 @@ import { BillingService } from '../aai-utils/billing' import { executeAgentFlow } from './buildAgentflow' import { Workspace } from '../enterprise/database/entities/workspace.entity' import { Organization } from '../enterprise/database/entities/organization.entity' +import { runInputStage, runOutputStage, buildFailClosedError } from '../services/guardrails/runStage' +import { extractTextFromSourceDocuments } from '../services/guardrails/extractContext' const shouldAutoPlayTTS = (textToSpeechConfig: string | undefined | null): boolean => { if (!textToSpeechConfig) return false @@ -348,56 +349,39 @@ export const executeFlow = async ({ const userMessageDateTime = new Date() const chatflowid = chatflow.id - /* Input validation with Fiddler Guardrails + /* Input validation with Fiddler Guardrails (see runInputStage for full policy). * - Safety checks (11 dimensions) * - PII detection and redaction * - Actions: block, redact, warn, continue + * - Failure semantics: fail-open emits a banner; fail-closed returns 503. */ let guardrailsMetadata: Partial | undefined - if (workspaceId) { - try { - const guardrailsService = await FiddlerGuardrailsService.createFromContext(chatflowid, workspaceId, orgId) - - if (guardrailsService) { - const validationResult = await guardrailsService.validateInput(question) + { + const stage = await runInputStage(question, { + chatflowId: chatflowid, + workspaceId, + organizationId: orgId, + chatId + }) - // Build metadata for client and database - if (validationResult) { - guardrailsMetadata = { - inputValidation: validationResult - } + guardrailsMetadata = { + health: { mode: stage.failureMode, anyDegraded: stage.health.degraded, input: stage.health } + } - // Enhanced structured logging - logger.info({ - message: '[Guardrails] Input validation triggered', - chatflowId: chatflowid, - chatId, - userId: user?.id, - organizationId: user?.organizationId, - blocked: validationResult.blocked, - redacted: validationResult.redacted, - safetyViolations: validationResult.violations.safety?.map((v) => `${v.dimension}(${v.score.toFixed(2)})`), - piiDetections: validationResult.violations.pii?.map((p) => `${p.label}(${p.score.toFixed(2)})`) - }) - } + if (stage.kind === 'blocked_degraded') { + throw buildFailClosedError(stage.reason) + } - // Handle blocking - if (validationResult.blocked) { - throw new InternalFlowiseError(StatusCodes.BAD_REQUEST, validationResult.message || 'Content blocked by guardrails') - } + if (stage.kind === 'blocked_violation') { + guardrailsMetadata.inputValidation = stage.inputResult + throw new InternalFlowiseError(StatusCodes.BAD_REQUEST, stage.message) + } - // Handle redaction - if (validationResult.redacted && validationResult.redactedText) { - question = validationResult.redactedText - } + if (stage.inputResult) { + guardrailsMetadata.inputValidation = stage.inputResult + if (stage.inputResult.redacted && stage.inputResult.redactedText) { + question = stage.inputResult.redactedText } - } catch (error) { - // Fail-open by default: log error but continue processing - if (error instanceof InternalFlowiseError && error.statusCode === StatusCodes.BAD_REQUEST) { - // Re-throw blocking errors - throw error - } - logger.error('[Guardrails] Validation error (fail-open)', { error, chatflowId: chatflowid, chatId }) } } @@ -563,7 +547,8 @@ export const executeFlow = async ({ orgId, workspaceId, subscriptionId, - productId + productId, + guardrailsMetadata }) } @@ -712,62 +697,56 @@ export const executeFlow = async ({ } await utilAddChatMessage(userMessage, appDataSource) - /* Output validation with Fiddler Guardrails (Agent Flow) - * - Safety checks - * - PII detection - * - Faithfulness checks (RAG hallucination detection) - * - Always fails open (never blocks output) + /* Output validation with Fiddler Guardrails (Agent Flow). + * Never blocks content already streamed; on fail-closed + degraded, + * emits a trailing SSE error event so the client UI can surface it. */ - if (workspaceId && guardrailsMetadata) { - try { - const guardrailsService = await FiddlerGuardrailsService.createFromContext(chatflowid, workspaceId, orgId) - - if (guardrailsService) { - // Extract context from sourceDocuments for faithfulness checking - const context = sourceDocuments?.length ? extractTextFromSourceDocuments(sourceDocuments) : undefined - - // Validate output (faithfulness check only runs if context exists) - const outputValidation = await guardrailsService.validateOutput(finalResult, context) - - // Merge with existing guardrailsMetadata - guardrailsMetadata = { - ...guardrailsMetadata, - outputValidation: { - blocked: false, // Output validation never blocks - redacted: outputValidation.redacted, - faithfulnessScore: outputValidation.violations.faithfulness?.score, - violations: { - safety: outputValidation.violations.safety, - pii: outputValidation.violations.pii - } - } - } + { + const context = sourceDocuments?.length ? extractTextFromSourceDocuments(sourceDocuments) : undefined + const outStage = await runOutputStage(finalResult, context, { + chatflowId: chatflowid, + workspaceId, + organizationId: orgId, + chatId, + prompt: incomingInput.question + }) - // Enhanced structured logging - logger.info({ - message: '[Guardrails] Output validation triggered (agent flow)', - chatflowId: chatflowid, - chatId, - userId: user?.id, - organizationId: user?.organizationId, - replaced: outputValidation.replaced, - redacted: outputValidation.redacted, - faithfulnessScore: outputValidation.violations.faithfulness?.score, - faithfulnessThreshold: outputValidation.violations.faithfulness?.threshold, - faithful: outputValidation.violations.faithfulness - ? outputValidation.violations.faithfulness.score >= outputValidation.violations.faithfulness.threshold - : undefined, - safetyViolations: outputValidation.violations.safety?.map((v) => `${v.dimension}(${v.score.toFixed(2)})`), - piiDetections: outputValidation.violations.pii?.map((p) => `${p.label}(${p.score.toFixed(2)})`) - }) - - // Note: We do NOT replace output text in Phase 5 - // Streaming clients have already received the original text - // Output validation is for monitoring and display only + const existingHealth = guardrailsMetadata?.health + guardrailsMetadata = { + ...guardrailsMetadata, + health: { + mode: outStage.failureMode, + anyDegraded: (existingHealth?.anyDegraded ?? false) || outStage.health.degraded, + input: existingHealth?.input, + output: outStage.health } - } catch (error) { - // Fail-open: Log error but continue with original output - logger.error('[Guardrails] Output validation error (fail-open) - agent flow', { error, chatflowId: chatflowid, chatId }) + } + + if (outStage.kind === 'ok' && outStage.outputResult) { + guardrailsMetadata.outputValidation = { + blocked: false, + redacted: outStage.outputResult.redacted, + faithfulnessScore: outStage.outputResult.violations.faithfulness?.score, + violations: { + safety: outStage.outputResult.violations.safety, + pii: outStage.outputResult.violations.pii + } + } + } + + if (outStage.kind === 'blocked_degraded') { + // Streaming clients already received the LLM output. We cannot un-ring + // that bell. Emit a trailing error so the client renders the fail-closed + // banner, but persist the apiMessage below so the audit trail is complete. + sseStreamer?.streamErrorEvent( + chatId, + `Safety checks were unavailable (reason: ${outStage.reason}). This response bypassed guardrail enforcement.` + ) + logger.error('[Guardrails] Agent output degraded fail-closed (streaming)', { + chatflowId: chatflowid, + chatId, + reason: outStage.reason + }) } } @@ -1020,62 +999,57 @@ export const executeFlow = async ({ } else if (result.json) resultText = '```json\n' + JSON.stringify(result.json, null, 2) else resultText = JSON.stringify(result, null, 2) - /* Output validation with Fiddler Guardrails - * - Safety checks - * - PII detection - * - Faithfulness checks (RAG hallucination detection) - * - Always fails open (never blocks output) + /* Output validation with Fiddler Guardrails (chain path). + * - Safety checks, PII detection, Faithfulness checks (RAG) + * - Never blocks content already generated; fail-closed + degraded emits + * a trailing error event and records health metadata for audit. */ - if (workspaceId && guardrailsMetadata) { - try { - const guardrailsService = await FiddlerGuardrailsService.createFromContext(chatflowid, workspaceId, orgId) - - if (guardrailsService) { - // Extract context from sourceDocuments for faithfulness checking - const context = result.sourceDocuments ? extractTextFromSourceDocuments(result.sourceDocuments) : undefined - - // Validate output (faithfulness check only runs if context exists) - const outputValidation = await guardrailsService.validateOutput(resultText, context) - - // Merge with existing guardrailsMetadata - guardrailsMetadata = { - ...guardrailsMetadata, - outputValidation: { - blocked: false, // Output validation never blocks - redacted: outputValidation.redacted, - faithfulnessScore: outputValidation.violations.faithfulness?.score, - violations: { - safety: outputValidation.violations.safety, - pii: outputValidation.violations.pii - } - } + { + const context = result.sourceDocuments ? extractTextFromSourceDocuments(result.sourceDocuments) : undefined + const outStage = await runOutputStage(resultText, context, { + chatflowId: chatflowid, + workspaceId, + organizationId: orgId, + chatId, + prompt: incomingInput.question + }) + + const existingHealth = guardrailsMetadata?.health + guardrailsMetadata = { + ...guardrailsMetadata, + health: { + mode: outStage.failureMode, + anyDegraded: (existingHealth?.anyDegraded ?? false) || outStage.health.degraded, + input: existingHealth?.input, + output: outStage.health + } + } + + if (outStage.kind === 'ok' && outStage.outputResult) { + guardrailsMetadata.outputValidation = { + blocked: false, + redacted: outStage.outputResult.redacted, + faithfulnessScore: outStage.outputResult.violations.faithfulness?.score, + violations: { + safety: outStage.outputResult.violations.safety, + pii: outStage.outputResult.violations.pii } + } + } - // Enhanced structured logging - logger.info({ - message: '[Guardrails] Output validation triggered', - chatflowId: chatflowid, + if (outStage.kind === 'blocked_degraded') { + if (isStreamValid && sseStreamer) { + sseStreamer.streamErrorEvent( chatId, - userId: user?.id, - organizationId: user?.organizationId, - replaced: outputValidation.replaced, - redacted: outputValidation.redacted, - faithfulnessScore: outputValidation.violations.faithfulness?.score, - faithfulnessThreshold: outputValidation.violations.faithfulness?.threshold, - faithful: outputValidation.violations.faithfulness - ? outputValidation.violations.faithfulness.score >= outputValidation.violations.faithfulness.threshold - : undefined, - safetyViolations: outputValidation.violations.safety?.map((v) => `${v.dimension}(${v.score.toFixed(2)})`), - piiDetections: outputValidation.violations.pii?.map((p) => `${p.label}(${p.score.toFixed(2)})`) - }) - - // Note: We do NOT replace output text in Phase 5 - // Streaming clients have already received the original text - // Output validation is for monitoring and display only + `Safety checks were unavailable (reason: ${outStage.reason}). This response bypassed guardrail enforcement.` + ) } - } catch (error) { - // Fail-open: Log error but continue with original output - logger.error('[Guardrails] Output validation error (fail-open)', { error, chatflowId: chatflowid, chatId }) + logger.error('[Guardrails] Output degraded fail-closed', { + chatflowId: chatflowid, + chatId, + reason: outStage.reason, + streaming: isStreamValid + }) } } @@ -1536,38 +1510,4 @@ const incrementFailedMetricCounter = (metricsProvider: IMetricsProvider, isInter } } -/** - * Extract text from source documents for faithfulness checking - * Handles both JSON string and parsed array formats - * - * @param {any} sourceDocuments - Source documents from RAG retrieval - * @returns {string | undefined} - Concatenated text from all documents or undefined if none found - */ -const extractTextFromSourceDocuments = (sourceDocuments: any): string | undefined => { - try { - // Handle both JSON string and already-parsed array - const docs = typeof sourceDocuments === 'string' ? JSON.parse(sourceDocuments) : sourceDocuments - - // Ensure it's an array with content - if (!Array.isArray(docs) || docs.length === 0) { - return undefined - } - - // Extract pageContent from each document and filter out empty strings - const textChunks = docs.map((doc) => doc.pageContent || '').filter((text) => text.length > 0) - - // Return undefined if no valid text found - if (textChunks.length === 0) { - return undefined - } - - // Join all chunks with double newline separator - return textChunks.join('\n\n') - } catch (error) { - // Fail gracefully - log warning and return undefined - logger.warn('[Guardrails] Failed to extract text from sourceDocuments', { error }) - return undefined - } -} - export { shouldAutoPlayTTS, generateTTSForResponseStream } diff --git a/packages/server/test/services/guardrails/index.test.ts b/packages/server/test/services/guardrails/index.test.ts new file mode 100644 index 00000000000..c669d0061f0 --- /dev/null +++ b/packages/server/test/services/guardrails/index.test.ts @@ -0,0 +1,226 @@ +/** + * FiddlerGuardrailsService — plan-tier capability handling. + * + * These tests pin the behavior we ship for the freemium-PII bug: + * 1. A Fiddler 404 with "not supported" body maps to FiddlerUnsupportedError. + * 2. Unsupported endpoints are cached per (apiKey, endpoint) and short-circuit + * future calls without hitting the network or tripping the circuit breaker. + * 3. Unsupported is reflected as ok=true, degraded=false (no per-message + * banner) and the merged stage status stays clean. + * 4. Faithfulness uses the wire shape Fiddler actually accepts: + * `{ data: { prompt, response, context } }` (NOT { input, output }). + */ + +const mockPost = jest.fn() + +jest.mock('axios', () => ({ + create: jest.fn(() => ({ + post: mockPost + })), + default: { + create: jest.fn(() => ({ post: mockPost })) + } +})) + +// Stub the express-app accessor and the config loader. FiddlerGuardrailsService +// imports both at module-load time; without these stubs, requiring the service +// boots the entire express app (including Auth0 jwt verifier setup) and fails. +jest.mock('../../../src/utils/getRunningExpressApp', () => ({ + getRunningExpressApp: () => ({ AppDataSource: { getRepository: () => ({}) } }) +})) + +jest.mock('../../../src/services/guardrails/config', () => ({ + getGuardrailsConfig: jest.fn() +})) + +// Avoid pulling flowise-components (and transitively langfuse, which uses +// dynamic ESM imports that don't work under ts-jest) just to get +// decryptCredentialData. We don't exercise it here. +jest.mock('../../../src/utils', () => ({ + decryptCredentialData: jest.fn().mockResolvedValue({ fiddlerApiKey: 'stub', fiddlerApiUrl: 'stub' }) +})) + +jest.mock('../../../src/database/entities/Credential', () => ({ + Credential: class {}, + CredentialVisibility: { ORGANIZATION: 'Organization', PLATFORM: 'Platform' } +})) + +// Suppress logger output during tests; we don't assert on log lines. +jest.mock('../../../src/utils/logger', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + log: jest.fn() + } +})) + +import { FiddlerGuardrailsService } from '../../../src/services/guardrails/FiddlerGuardrailsService' +import { FiddlerUnsupportedError, FiddlerError } from '../../../src/services/guardrails/errors' +import type { GuardrailsConfig } from '../../../src/types/guardrails' + +const baseConfig: GuardrailsConfig = { + enabled: true, + failureMode: 'open', + safety: { enabled: true, threshold: 0.1, action: 'block' }, + pii: { enabled: true, confidenceThreshold: 0.8, action: 'redact' }, + faithfulness: { enabled: true, threshold: 0.005, action: 'warn' }, + circuitBreaker: { failureThreshold: 5, resetTimeout: 30_000, successThreshold: 3 } +} + +const buildService = (apiKey = 'test-key') => + new FiddlerGuardrailsService({ apiKey, apiUrl: 'https://guardrails.cloud.fiddler.ai' }, baseConfig) + +const freemium404 = (endpoint: string) => { + const error: any = new Error(`Guardrail '${endpoint}' is not supported by the freemium guardrails API.`) + error.isAxiosError = true + error.config = { url: endpoint } + error.response = { + status: 404, + data: { + error: { + code: 404, + message: `Guardrail '${endpoint}' is not supported by the freemium guardrails API.`, + errors: [{ reason: 'NotFound', message: 'Not supported', help: '' }] + } + } + } + return error +} + +describe('FiddlerGuardrailsService — unsupported plan-tier handling', () => { + beforeEach(() => { + mockPost.mockReset() + FiddlerGuardrailsService.__resetUnsupportedCacheForTests() + }) + + it('classifies a "not supported by freemium" 404 as unsupported (ok=true, degraded=false)', async () => { + mockPost.mockRejectedValueOnce(freemium404('/v3/guardrails/sensitive-information')) + + const svc = buildService() + const { status } = await svc.detectPII('hello world') + + expect(status.reason).toBe('unsupported') + expect(status.degraded).toBe(false) + expect(status.ok).toBe(true) + expect(status.httpStatus).toBe(404) + }) + + it('caches unsupported endpoint per (apiKey, endpoint) and short-circuits on subsequent calls', async () => { + mockPost.mockRejectedValueOnce(freemium404('/v3/guardrails/sensitive-information')) + + const svc = buildService('cache-key') + + const first = await svc.detectPII('hi') + const second = await svc.detectPII('hi again') + const third = await svc.detectPII('and again') + + // First call hit the network, the next two short-circuited via cache. + expect(mockPost).toHaveBeenCalledTimes(1) + + // All three return the same unsupported status shape. + for (const r of [first.status, second.status, third.status]) { + expect(r.reason).toBe('unsupported') + expect(r.degraded).toBe(false) + } + + expect(svc.isEndpointUnsupported('/v3/guardrails/sensitive-information')).toBe(true) + }) + + it('separates the unsupported cache by API key (different orgs / plan tiers)', async () => { + // Org A: PII unsupported. + mockPost.mockRejectedValueOnce(freemium404('/v3/guardrails/sensitive-information')) + const orgA = buildService('org-a-key') + await orgA.detectPII('hello') + expect(orgA.isEndpointUnsupported('/v3/guardrails/sensitive-information')).toBe(true) + + // Org B: same endpoint, different key, must not inherit Org A's cache miss. + const orgB = buildService('org-b-key') + expect(orgB.isEndpointUnsupported('/v3/guardrails/sensitive-information')).toBe(false) + }) + + it('does NOT trip the circuit breaker when an endpoint is plan-unsupported', async () => { + // Even if the cache did not short-circuit, repeated unsupported responses + // are treated as breaker SUCCESS — the API answered, the caller's plan + // just doesn't include the endpoint. Penalizing the breaker on these + // would trip it and break sibling endpoints (e.g. safety) on the same + // Fiddler account. + for (let i = 0; i < 7; i++) { + mockPost.mockRejectedValueOnce(freemium404('/v3/guardrails/sensitive-information')) + } + + const svc = buildService('breaker-key') + await svc.detectPII('a') + await svc.detectPII('b') + await svc.detectPII('c') + + expect(svc.getCircuitStatus().state).toBe('CLOSED') + }) + + it('uses the correct faithfulness payload shape: { data: { prompt, response, context } }', async () => { + mockPost.mockResolvedValueOnce({ data: { fdl_faithful_score: 0.985 } }) + + const svc = buildService('faith-key') + await svc.evaluateFaithfulness('Paris is the capital.', 'The capital of France is Paris.', 'What is the capital of France?') + + expect(mockPost).toHaveBeenCalledTimes(1) + const [endpoint, body] = mockPost.mock.calls[0] + expect(endpoint).toBe('/v3/guardrails/ftl-response-faithfulness') + expect(body).toEqual({ + data: { + prompt: 'What is the capital of France?', + response: 'Paris is the capital.', + context: 'The capital of France is Paris.' + } + }) + }) + + it('skips faithfulness without a prompt (Fiddler requires all three fields)', async () => { + // Stub safety + pii so validateOutput's parallel calls resolve cleanly. + mockPost.mockResolvedValueOnce({ data: { fdl_harmful: 0.01 } }) // safety + mockPost.mockResolvedValueOnce({ data: { fdl_sensitive_information_scores: [] } }) // pii + + const svc = buildService('skip-key') + const result = await svc.validateOutput( + 'Some response', + 'Some context', + undefined as unknown as string // simulate caller that did not plumb prompt + ) + + // Only safety + pii hit the network; faithfulness was skipped because + // Fiddler's faithfulness endpoint requires all three of {prompt, response, context}. + expect(mockPost).toHaveBeenCalledTimes(2) + expect(result.violations.faithfulness).toBeUndefined() + expect(result.status.degraded).toBe(false) + }) + + it('exports FiddlerUnsupportedError as a FiddlerError subclass', () => { + const err = new FiddlerUnsupportedError('nope', '/v3/guardrails/sensitive-information') + expect(err).toBeInstanceOf(FiddlerError) + expect(err.reason).toBe('unsupported') + expect(err.endpoint).toBe('/v3/guardrails/sensitive-information') + }) +}) + +describe('FiddlerGuardrailsService — capabilityMatrix', () => { + beforeEach(() => { + mockPost.mockReset() + FiddlerGuardrailsService.__resetUnsupportedCacheForTests() + }) + + it('reports per-endpoint plan capability (ok / unsupported / degraded)', async () => { + // safety: 200, pii: 404 unsupported, faithfulness: 200 + mockPost.mockResolvedValueOnce({ data: { fdl_harmful: 0.01 } }) + mockPost.mockRejectedValueOnce(freemium404('/v3/guardrails/sensitive-information')) + mockPost.mockResolvedValueOnce({ data: { fdl_faithful_score: 0.9 } }) + + const svc = buildService('cap-key') + const matrix = await svc.capabilityMatrix() + + expect(matrix.safety.reason).toBe('ok') + expect(matrix.pii.reason).toBe('unsupported') + expect(matrix.faithfulness.reason).toBe('ok') + }) +}) diff --git a/packages/ui/src/api/guardrails.js b/packages/ui/src/api/guardrails.js index b53e518bec7..462ae4f5be0 100644 --- a/packages/ui/src/api/guardrails.js +++ b/packages/ui/src/api/guardrails.js @@ -10,9 +10,24 @@ const getGuardrailsConfig = (orgId) => client.get(`/organizations/${orgId}/confi const updateGuardrailsConfig = (orgId, guardrails) => client.put(`/organizations/${orgId}/config/guardrails`, { guardrails }) +/** + * Diagnostic selftest. Returns: + * { config, credentialSource, healthCheck, capabilities, circuitState, notes } + * Capabilities is a per-endpoint plan-tier matrix so the admin UI can render + * which guardrails the connected Fiddler key actually supports. + */ +const getSelftest = (params = {}) => { + const search = new URLSearchParams() + if (params.chatflowId) search.set('chatflowId', params.chatflowId) + if (params.workspaceId) search.set('workspaceId', params.workspaceId) + const qs = search.toString() + return client.get(`/guardrails/selftest${qs ? `?${qs}` : ''}`) +} + export default { getOrgConfig, updateOrgConfig, getGuardrailsConfig, - updateGuardrailsConfig + updateGuardrailsConfig, + getSelftest } diff --git a/packages/ui/src/ui-component/extended/ChatflowGuardrails.jsx b/packages/ui/src/ui-component/extended/ChatflowGuardrails.jsx index ce8ef3c3391..56cbe4a5c8b 100644 --- a/packages/ui/src/ui-component/extended/ChatflowGuardrails.jsx +++ b/packages/ui/src/ui-component/extended/ChatflowGuardrails.jsx @@ -82,15 +82,20 @@ const ChatflowGuardrails = ({ dialogProps }) => { setError(null) setSuccess(false) + const mergedGuardrails = newConfig == null ? undefined : { ...(guardrailsConfig || organizationConfig || {}), ...newConfig } + // Update chatbotConfig with guardrails const updatedChatbotConfig = { ...chatbotConfig, - guardrails: overrideEnabled ? newConfig : undefined + guardrails: overrideEnabled ? mergedGuardrails : undefined } if (!dialogProps.chatflow.id && dialogProps.handleSaveFlow) { return dialogProps.handleSaveFlow(dialogProps.chatflow.name, { - chatbotConfig: JSON.stringify(updatedChatbotConfig) + chatbotConfig: JSON.stringify({ + ...chatbotConfig, + guardrails: overrideEnabled ? mergedGuardrails : undefined + }) }) } @@ -99,7 +104,18 @@ const ChatflowGuardrails = ({ dialogProps }) => { }) if (saveResp.data) { - setGuardrailsConfig(overrideEnabled ? newConfig : null) + let parsed = {} + try { + parsed = saveResp.data.chatbotConfig ? JSON.parse(saveResp.data.chatbotConfig) : {} + } catch (e) { + parsed = {} + } + setChatbotConfig(parsed) + if (overrideEnabled) { + setGuardrailsConfig(parsed.guardrails != null ? parsed.guardrails : mergedGuardrails) + } else { + setGuardrailsConfig(null) + } setSuccess(true) enqueueSnackbar({ message: 'Guardrails Configuration Saved', @@ -260,6 +276,70 @@ const ChatflowGuardrails = ({ dialogProps }) => { {/* Configuration UI (only shown when override is enabled) */} {overrideEnabled && ( + {/* Per-chatflow failure-mode override. Leaving this untouched means the + chatflow inherits the organization-level failureMode. */} + + + Failure Mode (override) + + + Organization default: {organizationConfig?.failureMode || 'open'}. Override below to change how + THIS chatflow reacts when guardrails cannot be evaluated. + + { + if (!value) return + const next = { ...(guardrailsConfig || {}), failureMode: value } + setGuardrailsConfig(next) + handleSave(next) + }} + aria-label='Chatflow failure mode' + > + Fail Open + Fail Closed + + + + + + Observability-only / shadow (override) + + + Organization default: {organizationConfig?.observabilityOnly ? 'On' : 'Off'}. When on, + violations are recorded but never block. Use the switch to pin a value for this chatflow, or keep it aligned + with the org (inherit). + + { + const next = { ...(guardrailsConfig || {}), observabilityOnly: e.target.checked } + setGuardrailsConfig(next) + handleSave(next) + }} + /> + } + label={ + + + {guardrailsConfig?.observabilityOnly !== undefined + ? 'Custom value for this chatflow' + : 'Following organization default (toggle to override)'} + + + } + /> + + {/* Mode Tabs */} Date: Mon, 4 May 2026 15:22:29 -0400 Subject: [PATCH 2/2] fix(csv-transformer): preserve AI-generated columns and surface cron status (#1062) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - **Bug fix — `generateCsv` dropped AI-introduced columns.** `convertToCSV` was building the header from `Object.keys(data[0])`. If row 0's `generatedData` was empty, errored, or shaped differently from later rows, every column that appeared only later was silently dropped from the export. Replaced with a stable union of keys across all rows (first-seen order); fields containing newlines/CR are now quoted as well. Mirror fix applied in the `packages-answers/scripts/generateCsv.ts` Prisma copy. - **CSV Transformer reachable from the sidebar.** Added a "CSV Transformer" entry under Agent Studio (and the public-org Answer Studio submenu) in `AppDrawer.tsx`, linking to `/sidekick-studio/csv-transformer`. Always visible (no env gate). - **Cron-status banner on the page.** The Next.js page is now a server component that reads `ENABLE_CSV_RUN_CRON` and passes a `cronEnabled` prop into the client. Renders a green `success` banner when enabled, and an orange `warning` banner when disabled (explaining that runs will be created but not processed until the env is set + worker restarted). No `NEXT_PUBLIC_` duplication required. The original `dynamic(..., { ssr: false })` behavior is preserved via a thin `CsvTransformerClient.tsx` wrapper. ## Why - Users were submitting CSV runs and getting back files with the four enrichment columns empty even though the chatbot was returning data — root cause was a combination of (a) an output-parser schema mismatch on the chatflow side and (b) the `convertToCSV` first-row-only header bug. This PR fixes (b). - The CSV Transformer page existed at `/sidekick-studio/csv-transformer` but was unreachable from the main sidebar navigation in most environments. - It was not obvious whether a given environment had the CSV cron worker turned on, so users could submit runs that would never be processed and have no visible indication why. ## Test plan - [ ] Build succeeds (`pnpm build`). - [ ] Sidebar: open Agent Studio → "CSV Transformer" appears below Credentials and links to `/sidekick-studio/csv-transformer`. - [ ] CSV Transformer page renders the **warning** banner when `ENABLE_CSV_RUN_CRON` is unset or not `true`. - [ ] Set `ENABLE_CSV_RUN_CRON=true`, restart server, and confirm the page renders the **success** banner. - [ ] Submit a small CSV through a CSV-tagged chatflow whose output schema returns a key that the original CSV does not have. Confirm that the processed CSV download contains the new column header and values for all rows (regression test for the dropped-column bug). - [ ] Existing CSV Transformer flow (Process tab, History tab, download) still works. --- .../csv-transformer/CsvTransformerClient.tsx | 14 ++++++ .../(main-layout)/csv-transformer/page.tsx | 8 ++-- packages-answers/scripts/generateCsv.ts | 38 +++++++++++----- packages-answers/ui/src/AppDrawer.tsx | 12 +++++ .../CsvTransfomer/CsvTransformer.Client.tsx | 20 ++++++++- packages/server/src/jobs/generateCsv.ts | 44 ++++++++++++++----- packages/server/src/jobs/processCsvRows.ts | 27 ++++++++++-- 7 files changed, 129 insertions(+), 34 deletions(-) create mode 100644 apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/CsvTransformerClient.tsx diff --git a/apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/CsvTransformerClient.tsx b/apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/CsvTransformerClient.tsx new file mode 100644 index 00000000000..da2e37f7ac5 --- /dev/null +++ b/apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/CsvTransformerClient.tsx @@ -0,0 +1,14 @@ +'use client' +import dynamic from 'next/dynamic' + +const View = dynamic(() => import('@ui/CsvTransfomer'), { ssr: false }) + +interface Props { + cronEnabled: boolean +} + +const CsvTransformerClient = ({ cronEnabled }: Props) => { + return +} + +export default CsvTransformerClient diff --git a/apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/page.tsx b/apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/page.tsx index 4d455eb5e81..9a8b64bb941 100644 --- a/apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/page.tsx +++ b/apps/web/app/(Main UI)/sidekick-studio/(main-layout)/csv-transformer/page.tsx @@ -1,12 +1,12 @@ import React from 'react' -import dynamic from 'next/dynamic' - -const View = dynamic(() => import('@ui/CsvTransfomer'), { ssr: false }) +import CsvTransformerClient from './CsvTransformerClient' const Page = () => { + const cronEnabled = process.env.ENABLE_CSV_RUN_CRON === 'true' + return ( <> - + ) } diff --git a/packages-answers/scripts/generateCsv.ts b/packages-answers/scripts/generateCsv.ts index a75985a4f0f..bf929bfe05d 100644 --- a/packages-answers/scripts/generateCsv.ts +++ b/packages-answers/scripts/generateCsv.ts @@ -12,33 +12,47 @@ const s3 = new S3({ } }) +function collectAllColumnKeys(rows: Array>): string[] { + const seen = new Set() + const ordered: string[] = [] + for (const row of rows) { + if (!row || typeof row !== 'object') continue + for (const key of Object.keys(row)) { + if (!seen.has(key)) { + seen.add(key) + ordered.push(key) + } + } + } + return ordered +} + function convertToCSV(data: T[]): string { if (data.length === 0) { return '' } - // Extract the keys from the first object to use as CSV headers - const keys = Object.keys(data[0]) + const records = data as Array> + const keys = collectAllColumnKeys(records) + if (keys.length === 0) { + return '' + } - // Create the CSV header row const header = keys.join(',') - // Create rows by mapping through the array of objects - const rows = data.map((row) => { + const lines = records.map((row) => { return keys .map((key) => { - // Convert undefined or null to empty strings - const value = (row as Record)[key] ?? '' - // Escape quotes by replacing " with "" - // Wrap fields containing commas or quotes in double quotes + const value = row[key] ?? '' const stringValue = String(value).replace(/"/g, '""') - return stringValue.includes(',') || stringValue.includes('"') ? `"${stringValue}"` : stringValue + const needsQuotes = + stringValue.includes(',') || stringValue.includes('"') || stringValue.includes('\n') || stringValue.includes('\r') + return needsQuotes ? `"${stringValue}"` : stringValue }) .join(',') }) - // Combine header and rows - return [header, ...rows].join('\n') + return [header, ...lines].join('\n') } const generateCsv = async (csvParseRun: AppCsvParseRuns) => { diff --git a/packages-answers/ui/src/AppDrawer.tsx b/packages-answers/ui/src/AppDrawer.tsx index 7dfce43dcfb..49079084394 100644 --- a/packages-answers/ui/src/AppDrawer.tsx +++ b/packages-answers/ui/src/AppDrawer.tsx @@ -317,6 +317,12 @@ export const AppDrawer = ({ session }: AppDrawerProps) => { link: '/sidekick-studio/credentials', icon: }, + { + id: 'csv-transformer', + text: 'CSV Transformer', + link: '/sidekick-studio/csv-transformer', + icon: + }, { id: 'datasets', text: 'Datasets', @@ -508,6 +514,12 @@ export const AppDrawer = ({ session }: AppDrawerProps) => { link: '/sidekick-studio/credentials', icon: }, + { + id: 'csv-transformer', + text: 'CSV Transformer', + link: '/sidekick-studio/csv-transformer', + icon: + }, { id: 'datasets', text: 'Datasets', diff --git a/packages-answers/ui/src/CsvTransfomer/CsvTransformer.Client.tsx b/packages-answers/ui/src/CsvTransfomer/CsvTransformer.Client.tsx index 99ebdf2d237..597fdbb7b94 100644 --- a/packages-answers/ui/src/CsvTransfomer/CsvTransformer.Client.tsx +++ b/packages-answers/ui/src/CsvTransfomer/CsvTransformer.Client.tsx @@ -14,11 +14,15 @@ declare module '@/api/chatflows' { import chatflowsApi from '@/api/chatflows' // material-ui -import { Container, Box, Stack, Tabs, Tab, Typography } from '@mui/material' +import { Container, Box, Stack, Tabs, Tab, Typography, Alert, AlertTitle } from '@mui/material' import ProcessCsv from './ProcessCsv' import ProcessingHistory from './ProcessingHistory' +interface CsvTransformerProps { + cronEnabled?: boolean +} + function TabPanel(props: any) { const { children, currentValue, value, ...other } = props return ( @@ -34,7 +38,7 @@ function TabPanel(props: any) { ) } -const CsvTransformer = () => { +const CsvTransformer = ({ cronEnabled = false }: CsvTransformerProps) => { const { user, isLoading } = useUser() const [chatflows, setChatflows] = useState([]) const searchParams = useSearchParams() @@ -96,6 +100,18 @@ const CsvTransformer = () => { AI CSV Transformer + {cronEnabled ? ( + + Background processing is enabled + Submitted CSVs will be picked up by the cron worker on this environment. + + ) : ( + + Background processing is disabled + CSV runs can be created but they will not be processed until ENABLE_CSV_RUN_CRON=true is set on the + server (then the worker is restarted). + + )} diff --git a/packages/server/src/jobs/generateCsv.ts b/packages/server/src/jobs/generateCsv.ts index de3c64b9a1e..271557c3b38 100644 --- a/packages/server/src/jobs/generateCsv.ts +++ b/packages/server/src/jobs/generateCsv.ts @@ -23,33 +23,53 @@ const ENABLE_GENERATE_CSV_CRON = process.env.ENABLE_GENERATE_CSV_CRON !== 'false const { s3Client } = getS3Config() +/** + * Stable union of all object keys across rows: first row's key order first, + * then any keys that appear only on later rows (first-seen order). + * Using only data[0] drops columns that exist on later rows but not row 0 + * (e.g. when row 0's generatedData is empty/errored or shaped differently). + */ +function collectAllColumnKeys(rows: Array>): string[] { + const seen = new Set() + const ordered: string[] = [] + for (const row of rows) { + if (!row || typeof row !== 'object') continue + for (const key of Object.keys(row)) { + if (!seen.has(key)) { + seen.add(key) + ordered.push(key) + } + } + } + return ordered +} + function convertToCSV(data: T[]): string { if (data.length === 0) { return '' } - // Extract the keys from the first object to use as CSV headers - const keys = Object.keys(data[0]) + const records = data as Array> + const keys = collectAllColumnKeys(records) + if (keys.length === 0) { + return '' + } - // Create the CSV header row const header = keys.join(',') - // Create rows by mapping through the array of objects - const rows = data.map((row) => { + const lines = records.map((row) => { return keys .map((key) => { - // Convert undefined or null to empty strings - const value = (row as Record)[key] ?? '' - // Escape quotes by replacing " with "" - // Wrap fields containing commas or quotes in double quotes + const value = row[key] ?? '' const stringValue = String(value).replace(/"/g, '""') - return stringValue.includes(',') || stringValue.includes('"') ? `"${stringValue}"` : stringValue + const needsQuotes = + stringValue.includes(',') || stringValue.includes('"') || stringValue.includes('\n') || stringValue.includes('\r') + return needsQuotes ? `"${stringValue}"` : stringValue }) .join(',') }) - // Combine header and rows - return [header, ...rows].join('\n') + return [header, ...lines].join('\n') } const generateCsv = async (csvParseRun: AppCsvParseRuns) => { diff --git a/packages/server/src/jobs/processCsvRows.ts b/packages/server/src/jobs/processCsvRows.ts index 0c4ae5b5f9b..6e85c8611d9 100644 --- a/packages/server/src/jobs/processCsvRows.ts +++ b/packages/server/src/jobs/processCsvRows.ts @@ -50,6 +50,22 @@ const runChatFlow = async (row: IAppCsvParseRows, chatflowChatId: string) => { const { config: csvConfig } = safeParseCsvConfiguration(csvParseRun.configuration) const additionalContext = csvConfig?.context || '' const combinedQuestion = `### ${JSON.stringify(row.rowData)} ### ${additionalContext}`.trim() + + // Cron synthesizes its own request context (no req.user.activeWorkspaceId), + // so we must derive workspace/org from durable sources or executeFlow's + // workspace-scoped queries (e.g. getWorkspaceSearchOptions) will throw + // `Workspace ID is required` and the row will silently fail. + const workspaceId = chatflow.workspaceId || (user as any).activeWorkspaceId || '' + if (!workspaceId) { + throw new Error( + `Cannot run CSV row ${row.id}: chatflow ${chatflow.id} has no workspaceId and user ${user.id} has no activeWorkspaceId.` + ) + } + const orgId = user.organizationId || chatflow.organizationId || '' + if (!orgId) { + throw new Error(`Cannot run CSV row ${row.id}: neither user ${user.id} nor chatflow ${chatflow.id} has an organizationId.`) + } + const response = await executeFlow({ user: user as any, incomingInput: { @@ -66,8 +82,8 @@ const runChatFlow = async (row: IAppCsvParseRows, chatflowChatId: string) => { telemetry: appServer.telemetry, cachePool: appServer.cachePool, usageCacheManager: appServer.usageCacheManager, - orgId: user.organizationId || '', - workspaceId: '', + orgId, + workspaceId, subscriptionId: '', productId: '' }) @@ -209,9 +225,12 @@ const parseCsvRun = async (csvParseRun: IAppCsvParseRuns): Promise => { results.forEach((result, index) => { if (result.status === 'fulfilled') { - logger.info(`Row ${rows[index].id} completed`) + // processRow swallows row-level errors and only marks the row + // COMPLETE_WITH_ERRORS, so a fulfilled promise here does not + // necessarily mean success — keep the message neutral. + logger.info(`Row ${rows[index].id} processed`) } else { - logger.error(`Row ${rows[index].id} failed`) + logger.error(`Row ${rows[index].id} failed`, result.reason) } }) }