Fail loud when false-positive filtering is unavailable (remove the fixed-model probe)

README.md

@@ -54,7 +54,7 @@ This action is not hardened against prompt injection attacks and should only be
 | `comment-pr` | Whether to comment on PRs with findings | `true` | No |
 | `upload-results` | Whether to upload results as artifacts | `true` | No |
 | `exclude-directories` | Comma-separated list of directories to exclude from scanning | None | No |
-| `claude-model` | Claude [model name](https://docs.anthropic.com/en/docs/about-claude/models/overview#model-names) to use. Defaults to Opus 4.1. | `claude-opus-4-1-20250805` | No |
+| `claude-model` | Claude [model name](https://docs.anthropic.com/en/docs/about-claude/models/overview#model-names) to use. Defaults to Opus 5. | `claude-opus-5` | No |
 | `claudecode-timeout` | Timeout for ClaudeCode analysis in minutes | `20` | No |
 | `run-every-commit` | Run ClaudeCode on every commit (skips cache check). Warning: May increase false positives on PRs with many commits. | `false` | No |
 | `false-positive-filtering-instructions` | Path to custom false positive filtering instructions text file | None | No |

action.yml

@@ -29,7 +29,7 @@ inputs:
     default: ''
 
   claude-model:
-    description: 'Claude model to use for security analysis (e.g., claude-sonnet-4-20250514)'
+    description: 'Claude model to use for security analysis (e.g., claude-sonnet-5)'
     required: false
     default: ''
 
@@ -246,6 +246,23 @@ runs:
         else
           echo "ClaudeCode scan completed successfully"
         fi
+
+        # Exit 2 is EXIT_CONFIGURATION_ERROR: the audit could not run AS CONFIGURED —
+        # unreadable environment, or (since the false-positive filter fails loud) a model
+        # the key cannot reach. Exit 1 stays swallowed: that is the auditor's "found a HIGH
+        # severity finding" signal, which is a RESULT, not a failure, and callers gate on
+        # it themselves.
+        #
+        # Without this the step exits 0 and reports findings_count=0 whatever the auditor
+        # said, so "the filter could not run" is indistinguishable from "there was nothing
+        # to report" — the exact defect the filter's own fail-loud change exists to fix,
+        # left in place one layer up. Deferred to the caller, the property only holds for
+        # callers that parse the results JSON themselves.
+        CLAUDECODE_CONFIG_ERROR=0
+        if [ "${CLAUDECODE_EXIT_CODE:-0}" = "2" ]; then
+          CLAUDECODE_CONFIG_ERROR=1
+          echo "::error::ClaudeCode could not run as configured (exit 2). The results file below carries the reason; a FALSE_POSITIVE_FILTER_UNAVAILABLE error means the scan ran but its false-positive filter did not, so any findings are UNFILTERED."
+        fi
 
         # Parse ClaudeCode results and count findings regardless of exit code
         if [ -f claudecode/claudecode-results.json ]; then
@@ -303,8 +320,14 @@ runs:
         fi
 
         echo "::endgroup::"
-
-
+
+        # Fail LAST, after the results/findings outputs and the workspace copies above, so
+        # the artifact upload and the PR-comment step still see everything they need.
+        if [ "$CLAUDECODE_CONFIG_ERROR" = "1" ]; then
+          exit 1
+        fi
+
+
     - name: Upload scan results
       if: always() && inputs.upload-results == 'true'
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 pinned to commit hash

claudecode/claude_api_client.py

@@ -2,6 +2,7 @@
 
 import os
 import json
+import re
 import time
 from typing import Dict, Any, Tuple, Optional
 from pathlib import Path
@@ -18,6 +19,38 @@
 logger = get_logger(__name__)
 
 
+# Configuration-class API errors: the false-positive filter cannot run AT ALL as
+# configured (unknown/retired model id, bad or revoked key, key without access to the
+# model). These are never transient — retrying cannot fix them, and continuing without
+# the filter produces output indistinguishable from output the filter actually passed.
+#
+# Deliberately NARROW. Rate limits, overload, timeouts and spend/usage-cap 400s are NOT
+# in this set: they are transient or billing conditions, already handled by the retry
+# path below (and, for callers running this in CI, by their own cap classification).
+_CONFIGURATION_ERROR_PATTERNS = re.compile(
+    r"not_found_error"
+    r"|authentication_error"
+    r"|permission_error"
+    r"|invalid[ _-](x[ _-])?api[ _-]key"
+    r"|error code:\s*(401|403|404)\b",
+    re.IGNORECASE,
+)
+
+
+class ClaudeFilteringUnavailableError(RuntimeError):
+    """Claude-based false-positive filtering cannot run as configured.
+
+    Raised in place of silently degrading to hard-rules-only filtering. Callers are
+    expected to FAIL the run: an UNFILTERED result must never be reported as a
+    filtered one.
+    """
+
+
+def is_configuration_error(error_message: str) -> bool:
+    """True when an API error means filtering can never succeed as configured."""
+    return bool(_CONFIGURATION_ERROR_PATTERNS.search(error_message or ""))
+
+
 class ClaudeAPIClient:
     """Client for calling Claude API directly for security analysis tasks."""
 
@@ -50,27 +83,6 @@ def __init__(self,
         self.client = Anthropic(api_key=self.api_key)
         logger.info("Claude API client initialized successfully")
 
-    def validate_api_access(self) -> Tuple[bool, str]:
-        """Validate that API access is working.
-
-        Returns:
-            Tuple of (success, error_message)
-        """
-        try:
-            # Simple test call to verify API access
-            self.client.messages.create(
-                model="claude-3-5-haiku-20241022",
-                max_tokens=10,
-                messages=[{"role": "user", "content": "Hello"}],
-                timeout=10
-            )
-            logger.info("Claude API access validated successfully")
-            return True, ""
-        except Exception as e:
-            error_msg = str(e)
-            logger.error(f"Claude API validation failed: {error_msg}")
-            return False, f"API validation failed: {error_msg}"
-
     def call_with_retry(self, 
                        prompt: str,
                        system_prompt: Optional[str] = None,
@@ -124,7 +136,16 @@ def call_with_retry(self,
                 error_msg = str(e)
                 last_error = error_msg
                 logger.error(f"Claude API call failed: {error_msg}")
-
+
+                # Fail LOUD and immediately on a configuration-class error. Retrying an
+                # unknown model id only multiplies failed requests, and the old code path
+                # ended in a silently-unfiltered scan either way.
+                if is_configuration_error(error_msg):
+                    raise ClaudeFilteringUnavailableError(
+                        f"Claude false-positive filtering is unavailable with model "
+                        f"'{self.model}': {error_msg}"
+                    ) from e
+
                 # Check if it's a rate limit error
                 if "rate limit" in error_msg.lower() or "429" in error_msg:
                     logger.warning("Rate limit detected, increasing backoff")
@@ -179,6 +200,10 @@ def analyze_single_finding(self,
                 # Fallback: return error
                 return False, {}, "Failed to parse JSON response"
 
+        except ClaudeFilteringUnavailableError:
+            # Configuration-class failure — must not be flattened into a per-finding
+            # "keep it anyway" result; the whole run has to fail.
+            raise
         except Exception as e:
             logger.exception(f"Error during single finding security analysis: {str(e)}")
             return False, {}, f"Single finding security analysis failed: {str(e)}"

claudecode/constants.py

@@ -5,13 +5,13 @@
 import os
 
 # API Configuration
-DEFAULT_CLAUDE_MODEL = os.environ.get('CLAUDE_MODEL') or 'claude-opus-4-1-20250805'
+DEFAULT_CLAUDE_MODEL = os.environ.get('CLAUDE_MODEL') or 'claude-opus-5'
 DEFAULT_TIMEOUT_SECONDS = 180  # 3 minutes
 DEFAULT_MAX_RETRIES = 3
 RATE_LIMIT_BACKOFF_MAX = 30  # Maximum backoff time for rate limits
 
 # Token Limits
-PROMPT_TOKEN_LIMIT = 16384  # 16k tokens max for claude-opus-4
+PROMPT_TOKEN_LIMIT = 16384  # 16k tokens max
 
 # Exit Codes
 EXIT_SUCCESS = 0

claudecode/findings_filter.py

@@ -5,7 +5,10 @@
 import time
 from dataclasses import dataclass, field
 
-from claudecode.claude_api_client import ClaudeAPIClient
+from claudecode.claude_api_client import (
+    ClaudeAPIClient,
+    ClaudeFilteringUnavailableError,
+)
 from claudecode.constants import DEFAULT_CLAUDE_MODEL
 from claudecode.logger import get_logger
 
@@ -18,6 +21,7 @@ class FilterStats:
     total_findings: int = 0
     hard_excluded: int = 0
     claude_excluded: int = 0
+    claude_api_failures: int = 0
     kept_findings: int = 0
     exclusion_breakdown: Dict[str, int] = field(default_factory=dict)
     confidence_scores: List[float] = field(default_factory=list)
@@ -184,15 +188,21 @@ def __init__(self,
                     model=model,
                     api_key=api_key
                 )
-                # Validate API access
-                valid, error = self.claude_client.validate_api_access()
-                if not valid:
-                    logger.warning(f"Claude API validation failed: {error}")
-                    self.claude_client = None
-                    self.use_claude_filtering = False
+                # NO fixed-model probe here. The removed code called
+                # validate_api_access(), which hard-coded a model id of its own; once that
+                # id was retired the probe failed on every run, this branch set
+                # use_claude_filtering = False, and every scan afterwards reported
+                # hard-rules-only output exactly as if the Claude filter had passed it.
+                # The filter's own first real call now surfaces any problem, against the
+                # CONFIGURED model, and a configuration-class failure raises rather than
+                # silently disabling filtering.
+            except ClaudeFilteringUnavailableError:
+                raise
             except Exception as e:
-                logger.error(f"Failed to initialize Claude client: {str(e)}")
-                self.use_claude_filtering = False
+                raise ClaudeFilteringUnavailableError(
+                    f"Failed to initialize Claude client for false-positive "
+                    f"filtering: {str(e)}"
+                ) from e
 
     def filter_findings(self, 
                        findings: List[Dict[str, Any]],
@@ -296,6 +306,7 @@ def filter_findings(self,
                         stats.kept_findings += 1
                 else:
                     # Claude API call failed for this finding - keep it with warning
+                    stats.claude_api_failures += 1
                     logger.warning(f"Claude API call failed for finding {orig_idx}: {error_msg}")
                     enriched_finding = finding.copy()
                     enriched_finding['_filter_metadata'] = {
@@ -304,6 +315,22 @@ def filter_findings(self,
                     }
                     findings_after_claude.append(enriched_finding)
                     stats.kept_findings += 1
+            # A PARTIAL transient failure is a degraded-but-real filter pass, and the
+            # count reported below makes it visible. A TOTAL one is not: if every single
+            # finding fell back, the filter produced no verdict at all, and emitting that
+            # as a normal result is exactly the state ClaudeFilteringUnavailableError's
+            # docstring says must never happen — output indistinguishable from output the
+            # filter actually passed.
+            #
+            # A persistent 429/529/timeout reaches HERE rather than the configuration path,
+            # deliberately: those are transient, and failing a whole security job on a blip
+            # would be worse than the disease. But "nothing was filtered" is not a blip.
+            if stats.claude_api_failures and stats.claude_api_failures == len(findings_after_hard):
+                raise ClaudeFilteringUnavailableError(
+                    f"Claude false-positive filtering produced no verdicts: all "
+                    f"{stats.claude_api_failures} finding(s) failed against model "
+                    f"'{self.claude_client.model}'. Findings are UNFILTERED.")
+
         else:
             # Claude filtering disabled or no client - keep all findings from hard filter
             for orig_idx, finding in findings_after_hard:
@@ -331,13 +358,19 @@ def filter_findings(self,
                 "excluded_findings": len(all_excluded),
                 "hard_excluded": stats.hard_excluded,
                 "claude_excluded": stats.claude_excluded,
+                # Findings the Claude filter could not reach a verdict on (transient API
+                # failures) and which were therefore KEPT unjudged. Non-zero means the
+                # result is partially unfiltered — surfaced so a consumer can tell that
+                # apart from a clean pass instead of having to assume.
+                "claude_api_failures": stats.claude_api_failures,
                 "exclusion_breakdown": stats.exclusion_breakdown,
                 "average_confidence": sum(stats.confidence_scores) / len(stats.confidence_scores) if stats.confidence_scores else None,
                 "runtime_seconds": stats.runtime_seconds
             }
         }
 
         logger.info(f"Filtering completed: {stats.kept_findings}/{stats.total_findings} findings kept "
-                    f"({stats.runtime_seconds:.1f}s)")
+                    f"({stats.runtime_seconds:.1f}s); "
+                    f"{stats.claude_api_failures} finding(s) unjudged after API failures")
 
         return True, filtered_results, stats

claudecode/github_action_audit.py

@@ -16,6 +16,7 @@
 
 # Import existing components we can reuse
 from claudecode.prompts import get_security_audit_prompt
+from claudecode.claude_api_client import ClaudeFilteringUnavailableError
 from claudecode.findings_filter import FindingsFilter
 from claudecode.json_parser import parse_json_with_fallbacks
 from claudecode.constants import (
@@ -425,6 +426,9 @@ def initialize_findings_filter(custom_filtering_instructions: Optional[str] = No
                 use_hard_exclusions=True,
                 use_claude_filtering=False
             )
+    except ClaudeFilteringUnavailableError:
+        # Preserve the distinct filter-unavailable signal; main() classifies it.
+        raise
     except Exception as e:
         raise ConfigurationError(f'Failed to initialize findings filter: {str(e)}')
 
@@ -560,6 +564,12 @@ def main():
         # Initialize findings filter
         try:
             findings_filter = initialize_findings_filter(custom_filtering_instructions)
+        except ClaudeFilteringUnavailableError as e:
+            print(json.dumps({
+                'error': f'FALSE_POSITIVE_FILTER_UNAVAILABLE: {str(e)}',
+                'filter_unavailable': True,
+            }))
+            sys.exit(EXIT_CONFIGURATION_ERROR)
         except ConfigurationError as e:
             print(json.dumps({'error': str(e)}))
             sys.exit(EXIT_CONFIGURATION_ERROR)
@@ -609,10 +619,21 @@ def main():
             'description': pr_data.get('body', '')
         }
 
-        # Apply findings filter (including final directory exclusion)
-        kept_findings, excluded_findings, analysis_summary = apply_findings_filter(
-            findings_filter, original_findings, pr_context, github_client
-        )
+        # Apply findings filter (including final directory exclusion).
+        # A configuration-class filter failure FAILS the run: the audit did produce
+        # findings, but they are UNFILTERED, and emitting them as a normal result would
+        # be indistinguishable from a filtered one.
+        try:
+            kept_findings, excluded_findings, analysis_summary = apply_findings_filter(
+                findings_filter, original_findings, pr_context, github_client
+            )
+        except ClaudeFilteringUnavailableError as e:
+            print(json.dumps({
+                'error': f'FALSE_POSITIVE_FILTER_UNAVAILABLE: {str(e)}',
+                'filter_unavailable': True,
+                'unfiltered_findings_count': len(original_findings),
+            }))
+            sys.exit(EXIT_CONFIGURATION_ERROR)
 
         # Prepare output
         output = {