From ea97f12a3e9bd859162cdc7503fc3785407a85bc Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Thu, 8 Oct 2026 14:14:53 +0100 Subject: [PATCH 01/15] feat(forge-saw): add stable namespace-local openshell gateway alias Service Selects the agent VMI directly so Pipeline defaults need no deployment namespace. The gateway server leaf includes 'openshell' as a DNS SAN, letting the evaluate task keep OPENSHELL_GATEWAY_INSECURE=false. Co-Authored-By: Claude Code --- config/forge-saw/openshell-eval-alias.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 config/forge-saw/openshell-eval-alias.yaml diff --git a/config/forge-saw/openshell-eval-alias.yaml b/config/forge-saw/openshell-eval-alias.yaml new file mode 100644 index 0000000..d8ac338 --- /dev/null +++ b/config/forge-saw/openshell-eval-alias.yaml @@ -0,0 +1,20 @@ +# Stable, namespace-local endpoint for evaluation jobs. Selecting the agent +# VMI directly avoids baking a deployment namespace into Pipeline defaults. +# The gateway server leaf must include "openshell" as a DNS SAN before the +# evaluation task can disable OPENSHELL_GATEWAY_INSECURE. +apiVersion: v1 +kind: Service +metadata: + name: openshell + labels: + app.kubernetes.io/name: openshell-eval-alias + app.kubernetes.io/part-of: abevalflow +spec: + selector: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + ports: + - name: gateway + port: 17670 + protocol: TCP + targetPort: 17670 From ee282895f2bcdff72358780670d5a911ef3dbcca Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Thu, 8 Oct 2026 14:14:53 +0100 Subject: [PATCH 02/15] feat(litellm): add GLM-5-3-Flash routes for agent and judge traffic - openai/ route for rits/zai-org/GLM-5-3-Flash (agent model) - judge-glm-5-3 hosted_vllm route: LLM judges call /v1/messages (Anthropic dialect); with an openai/ model LiteLLM converts that to the upstream /responses API which answers 403, so every judge errors. hosted_vllm stays on /chat/completions. Select with aeh-judge-model-override=judge-glm-5-3. Also drop the hardcoded gz-forge-eval namespace so the ConfigMap applies anywhere. Co-Authored-By: Claude Code --- config/litellm/configmap.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/config/litellm/configmap.yaml b/config/litellm/configmap.yaml index b352a41..c01eca0 100644 --- a/config/litellm/configmap.yaml +++ b/config/litellm/configmap.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: litellm-config - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm @@ -15,6 +14,25 @@ data: api_base: os.environ/INFERENCE_ENDPOINT_URL api_key: os.environ/INFERENCE_API_KEY + - model_name: rits/zai-org/GLM-5-3-Flash + litellm_params: + model: openai/rits/zai-org/GLM-5-3-Flash + api_base: os.environ/INFERENCE_ENDPOINT_URL + api_key: os.environ/INFERENCE_API_KEY + + # Judge route. The LLM judges call /v1/messages (Anthropic dialect) — + # score.py has no OpenAI dialect. For an "openai/" model LiteLLM converts + # that to the upstream's /responses API, which this upstream rejects with + # 403 "Authentication parameters missing", so every scoring judge errors + # and mean_reward is unavailable. Routing the same upstream as hosted_vllm + # keeps the call on /chat/completions. Select it with + # aeh-judge-model-override=judge-glm-5-3. + - model_name: judge-glm-5-3 + litellm_params: + model: hosted_vllm/rits/zai-org/GLM-5-3-Flash + api_base: os.environ/INFERENCE_ENDPOINT_URL + api_key: os.environ/INFERENCE_API_KEY + litellm_settings: drop_params: true set_verbose: false From bad1b83af2f968e185351bd7b2b21b3ae079ccc9 Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Thu, 8 Oct 2026 14:14:54 +0100 Subject: [PATCH 03/15] feat(pipeline): add eval-stack-probe task Probes every dependency an eval pod needs: mlflow, litellm, minio, k8s api, github, pypi, github releases/raw, and TCP reachability of the agent/integ gateways and postgres. Deliberately not labelled part-of=abevalflow so NetworkPolicy sees it as an ordinary Tekton pod and reports real connectivity. Co-Authored-By: Claude Code --- pipeline/tasks/eval-stack-probe.yaml | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 pipeline/tasks/eval-stack-probe.yaml diff --git a/pipeline/tasks/eval-stack-probe.yaml b/pipeline/tasks/eval-stack-probe.yaml new file mode 100644 index 0000000..cee3d4c --- /dev/null +++ b/pipeline/tasks/eval-stack-probe.yaml @@ -0,0 +1,48 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: eval-stack-probe + # Deliberately not labelled app.kubernetes.io/part-of: abevalflow — the probe + # must be seen by NetworkPolicy as an ordinary Tekton pod, otherwise it passes + # through the stack's own allow-rules and reports connectivity the pipeline + # does not actually have. +spec: + description: >- + Probes every dependency an ABEvalFlow pipeline pod needs. Run it before and + after NetworkPolicy changes to see exactly which paths are blocked. + steps: + - name: probe + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + NS="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null || echo current)" + fail=0 + probe() { # name url + if curl -sS --max-time 8 -o /dev/null -w "%{http_code}" $2 >/tmp/code 2>/tmp/err; then + echo " PASS $1 (HTTP $(cat /tmp/code))" + else + echo " FAIL $1 -> $(tr -d '\n' /tmp/tcp-err; then + echo " PASS $1 (TCP $2:$3)" + else + echo " FAIL $1 -> $(tr -d '\n' Date: Thu, 8 Oct 2026 14:14:54 +0100 Subject: [PATCH 04/15] fix(cleanup): make PVC cleanup best-effort and use in-cluster CLI image - image-registry.openshift-image-registry openshift/cli instead of registry.redhat.io (avoids pull failures without a redhat registry secret on the pipeline SA) - --request-timeout on oc calls, 3 retries with backoff - onError: continue so the finally task never fails the PipelineRun Co-Authored-By: Claude Code --- pipeline/tasks/post/cleanup_pvc.yaml | 43 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/pipeline/tasks/post/cleanup_pvc.yaml b/pipeline/tasks/post/cleanup_pvc.yaml index 9ba6c65..c050958 100644 --- a/pipeline/tasks/post/cleanup_pvc.yaml +++ b/pipeline/tasks/post/cleanup_pvc.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: cleanup-pvc - namespace: ab-eval-flow spec: description: >- Deletes the workspace PVC after a pipeline run completes. @@ -11,36 +10,38 @@ spec: - name: pipelinerun-name type: string description: Name of the PipelineRun whose PVCs should be cleaned up - steps: - - name: delete-pvc - image: registry.redhat.io/openshift4/ose-cli:latest - script: | - #!/usr/bin/env bash - set -u + steps: + - name: delete-pvc + image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + onError: continue + script: | + #!/usr/bin/env bash + set -uo pipefail # Get current namespace from service account NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) echo "Cleaning up PVCs for PipelineRun $(params.pipelinerun-name) in namespace ${NAMESPACE}..." - PVC_LIST="" - for attempt in 1 2 3 4 5; do - if PVC_LIST=$(oc get pvc -n "${NAMESPACE}" \ - -o jsonpath="{range .items[?(@.metadata.ownerReferences[0].name=='$(params.pipelinerun-name)')]}{.metadata.name}{'\\n'}{end}" 2>/tmp/cleanup-pvc.err); then + pvcs="" + for attempt in 1 2 3; do + if pvcs=$(oc get pvc -n "${NAMESPACE}" --request-timeout=20s \ + -o jsonpath='{range .items[?(@.metadata.ownerReferences[0].name=="$(params.pipelinerun-name)")]}{.metadata.name}{"\n"}{end}'); then break fi - echo "WARNING: Kubernetes API unavailable (attempt ${attempt}/5); retrying..." >&2 - sleep 10 + echo "WARNING: PVC lookup attempt ${attempt}/3 failed" + sleep $((attempt * 5)) done - if [ -z "${PVC_LIST}" ] && [ -s /tmp/cleanup-pvc.err ]; then - cat /tmp/cleanup-pvc.err >&2 - echo "WARNING: PVC cleanup deferred because the Kubernetes API was unreachable" >&2 + + if [ -z "$pvcs" ]; then + echo "No owned PVCs found, or the API remained unavailable; cleanup is best-effort" exit 0 fi + while read -r pvc; do - if [ -n "${pvc}" ]; then - echo "Deleting PVC: ${pvc}" - oc delete pvc "${pvc}" -n "${NAMESPACE}" --wait=false || \ - echo "WARNING: failed to delete ${pvc}; it may be retried later" >&2 + if [ -n "$pvc" ]; then + echo "Deleting PVC: $pvc" + oc delete pvc "$pvc" -n "${NAMESPACE}" --wait=false --request-timeout=20s \ + || echo "WARNING: failed to delete PVC $pvc" fi - done <<< "${PVC_LIST}" + done <<< "$pvcs" echo "PVC cleanup complete" From 42812b82187cbecc045d16f6087ba9cef024d679 Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Thu, 8 Oct 2026 14:16:00 +0100 Subject: [PATCH 05/15] feat(pipeline): default to GLM-5-3-Flash with judge route and gateway alias - llm-model / aeh-model-override default to the Flash model; the submission's model entry carries the 128000-token budget it needs - aeh-judge-model-override defaults to judge-glm-5-3 (hosted_vllm route; a plain openai/ judge model 403s on /v1/messages) - openshell-gateway-endpoint default becomes https://openshell:17670, the alias Service from config/forge-saw/openshell-eval-alias.yaml, so the Pipeline applies to any namespace - example PipelineRun: short-name endpoints (namespace-agnostic, drops the gz-forge-eval hostAliases pin), main revisions, and timeouts sized for 15-minute cases (3h pipeline / 2h30m tasks) Co-Authored-By: Claude Code --- pipeline/pipelines/ci-pipeline-openshell.yaml | 14 +++++--- .../runs/openshell-openclaw-pipelinerun.yaml | 35 ++++++++++--------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index 2cbeb3e..ca817b1 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -39,7 +39,7 @@ spec: description: Must stay aeh_openshell_openclaw for this profile - name: llm-model type: string - default: "rits/zai-org/glm-5-3" + default: "rits/zai-org/GLM-5-3-Flash" - name: llm-api-base type: string default: "http://litellm.gz-forge-eval.svc:4000" @@ -51,7 +51,7 @@ spec: default: "http://litellm.ab-eval-flow.svc:4000/v1" - name: llm-generation-model type: string - default: "rits/zai-org/glm-5-3" + default: "rits/zai-org/GLM-5-3-Flash" - name: enable-ai-generation type: string default: "false" @@ -74,10 +74,13 @@ spec: default: "-1.0" - name: aeh-model-override type: string - default: "" + default: "inference/rits/zai-org/GLM-5-3-Flash" - name: aeh-judge-model-override type: string - default: "" + # Judges call /v1/messages (Anthropic dialect); see the judge-glm-5-3 + # entry in config/litellm/configmap.yaml for why a plain openai/ model + # 403s there and hosted_vllm does not. + default: "judge-glm-5-3" - name: aeh-mode type: string default: "single" @@ -112,7 +115,8 @@ spec: default: "0.0.116" - name: openshell-gateway-endpoint type: string - default: "https://openshell.openshell.svc.cluster.local:17670" + # Installed by config/forge-saw/openshell-eval-alias.yaml in the run namespace. + default: "https://openshell:17670" - name: openshell-sandbox-image type: string default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index 885d566..4fd3340 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -1,5 +1,10 @@ # OpenShell OpenClaw profile — prepare → evaluate → analyze → store (no test cube). -# Apply the Pipeline first: oc apply -f pipeline/pipelines/ci-pipeline-openshell.yaml +# +# oc apply -n -f pipeline/pipelines/ci-pipeline-openshell.yaml +# oc create -n -f pipeline/runs/openshell-openclaw-pipelinerun.yaml +# +# Service endpoints below use short names, which resolve inside the namespace +# the run is created in, so this file is namespace-agnostic. # Do not put M365 tokens in this file; use Secret openshell-credentials in the namespace. apiVersion: tekton.dev/v1 kind: PipelineRun @@ -28,12 +33,13 @@ spec: # - name: agent-eval-harness-repo-revision # value: "" - name: openshell-gateway-endpoint - # Certificate-valid hostname, resolved to this namespace's Service below. - value: "https://host.containers.internal:17670" + # Stable, certificate-valid alias installed by + # config/forge-saw/openshell-eval-alias.yaml in the run namespace. + value: "https://openshell:17670" - name: openshell-mtls-secret - value: "openshell-gateway-mtls" + value: "openshell-mtls" - name: openshell-ai-gateway-ca-secret - value: "forge-agent-upstream-tls" + value: "forge-ai-gateway-ca" # To use another sandbox image, replace the value below with . - name: openshell-sandbox-image value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" @@ -44,28 +50,25 @@ spec: - name: aeh-openshell-image value: "registry.access.redhat.com/ubi9/python-311:9.6" - name: llm-model - value: "rits/zai-org/glm-5-3" + value: "rits/zai-org/GLM-5-3-Flash" - name: llm-api-base - value: "http://litellm.gz-forge-eval.svc.cluster.local:4000" + value: "http://litellm:4000" - name: llm-api-key value: "mock" - name: aeh-model-override - value: "inference/rits/zai-org/glm-5-3" + value: "inference/rits/zai-org/GLM-5-3-Flash" - name: aeh-judge-model-override - value: "rits/zai-org/glm-5-3" + value: "judge-glm-5-3" - name: enable-mlflow value: "true" - name: mlflow-tracking-uri - value: "http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000" + value: "http://abevalflow-mlflow:5000" taskRunTemplate: serviceAccountName: pipeline - podTemplate: - hostAliases: - - ip: "172.30.183.80" - hostnames: ["host.containers.internal"] timeouts: - pipeline: 2h0m0s - tasks: 1h30m0s + pipeline: 3h0m0s + tasks: 2h30m0s + finally: 15m0s workspaces: - name: shared-workspace volumeClaimTemplate: From ec12e29df06a224b0fde9d877565423e3bf83cbf Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Thu, 8 Oct 2026 14:17:01 +0100 Subject: [PATCH 06/15] test: cover Flash defaults, judge route, alias Service, and cleanup hardening Extends the OpenShell profile tests for the carried forge-nommen changes: Flash model defaults, judge-glm-5-3 override, namespace- agnostic example run, gateway alias Service selector, LiteLLM Flash/judge routes, and best-effort PVC cleanup. Co-Authored-By: Claude Code --- tests/test_openshell_pipeline_profile.py | 48 +++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index 1875801..687fcfe 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -47,12 +47,58 @@ def test_openshell_defaults(self): assert defaults["openshell-sandbox-image"] == ( "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" ) + assert defaults["llm-model"] == "rits/zai-org/GLM-5-3-Flash" + assert defaults["llm-generation-model"] == "rits/zai-org/GLM-5-3-Flash" + assert defaults["aeh-model-override"] == "inference/rits/zai-org/GLM-5-3-Flash" + # A plain openai/ judge model 403s on /v1/messages; the judge-glm-5-3 + # hosted_vllm route in config/litellm/configmap.yaml stays on + # /chat/completions. See that file for the full rationale. + assert defaults["aeh-judge-model-override"] == "judge-glm-5-3" + # Alias Service from config/forge-saw/openshell-eval-alias.yaml; keeps + # the default namespace-agnostic. + assert defaults["openshell-gateway-endpoint"] == "https://openshell:17670" assert defaults["enable-mlflow"] == "true" - assert defaults["mlflow-tracking-uri"] == ("http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000") analyze = next(t for t in _load(PIPELINE)["spec"]["tasks"] if t["name"] == "analyze") scan = next(p for p in analyze["params"] if p["name"] == "security-scan-mode") assert scan["value"] == "disabled" + def test_litellm_configmap_has_flash_and_judge_routes(self): + config = _load(REPO / "config" / "litellm" / "configmap.yaml") + models = yaml.safe_load(config["data"]["config.yaml"])["model_list"] + names = {entry["model_name"] for entry in models} + assert "rits/zai-org/GLM-5-3-Flash" in names + assert "judge-glm-5-3" in names + judge = next(e for e in models if e["model_name"] == "judge-glm-5-3") + assert judge["litellm_params"]["model"].startswith("hosted_vllm/") + + def test_gateway_alias_service_selects_agent_vmi(self): + alias = _load(REPO / "config" / "forge-saw" / "openshell-eval-alias.yaml") + assert alias["spec"]["selector"] == { + "app.kubernetes.io/name": "openshell-saw-agent", + "vm.kubevirt.io/name": "openshell-saw-agent", + } + ports = alias["spec"]["ports"] + assert ports == [{"name": "gateway", "port": 17670, "protocol": "TCP", "targetPort": 17670}] + + def test_example_run_is_namespace_agnostic(self): + run = _load(REPO / "pipeline" / "runs" / "openshell-openclaw-pipelinerun.yaml") + params = {p["name"]: p["value"] for p in run["spec"]["params"]} + for name in ("openshell-gateway-endpoint", "llm-api-base", "mlflow-tracking-uri"): + value = params[name] + assert ".svc.cluster.local" not in value, f"{name} pins a namespace: {value}" + pod_template = run["spec"]["taskRunTemplate"].get("podTemplate", {}) + assert "hostAliases" not in pod_template + timeouts = run["spec"]["timeouts"] + assert timeouts["pipeline"] == "3h0m0s" + assert timeouts["tasks"] == "2h30m0s" + + def test_cleanup_task_is_best_effort(self): + task = _load(REPO / "pipeline" / "tasks" / "post" / "cleanup_pvc.yaml") + step = task["spec"]["steps"][0] + assert step.get("onError") == "continue" + assert "--request-timeout=20s" in step["script"] + assert step["image"].startswith("image-registry.openshift-image-registry.svc:5000/") + def test_evaluate_openshell_step_logs_mlflow(self): evaluate = (REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml").read_text() openshell = evaluate.split("name: aeh-openshell-eval", 1)[1].split("name: aeh-pairwise", 1)[0] From a79f6caff0a01864f7fa4dc87033e00c53eadfd4 Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Fri, 9 Oct 2026 10:31:27 +0100 Subject: [PATCH 07/15] chore: drop hardcoded namespaces from deployable manifests Pinned namespaces (ab-eval-flow, gz-forge-eval) made `oc apply -n ` fail with a namespace mismatch for anyone deploying the OpenShell profile into their own workspace (e.g. forge-nommen). Remove the namespace fields so the manifests land in whatever namespace they are applied to; the example PipelineRun already uses short service names that resolve in the run namespace. Co-Authored-By: Claude Code --- config/litellm/deployment.yaml | 1 - config/litellm/service.yaml | 1 - config/litellm/serviceaccount.yaml | 1 - pipeline/pipelines/ci-pipeline-openshell.yaml | 1 - pipeline/tasks/phases/evaluate.yaml | 1 - pipeline/tasks/phases/prepare.yaml | 1 - pipeline/tasks/post/analyze-and-check-degradation.yaml | 1 - pipeline/tasks/post/store.yaml | 1 - 8 files changed, 8 deletions(-) diff --git a/config/litellm/deployment.yaml b/config/litellm/deployment.yaml index 8146bab..46c015a 100644 --- a/config/litellm/deployment.yaml +++ b/config/litellm/deployment.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: Deployment metadata: name: litellm - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm diff --git a/config/litellm/service.yaml b/config/litellm/service.yaml index bc12954..885c92b 100644 --- a/config/litellm/service.yaml +++ b/config/litellm/service.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Service metadata: name: litellm - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm diff --git a/config/litellm/serviceaccount.yaml b/config/litellm/serviceaccount.yaml index 5ea3455..78ef60d 100644 --- a/config/litellm/serviceaccount.yaml +++ b/config/litellm/serviceaccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: litellm - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index ca817b1..b06026a 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: abevalflow-pipeline-openshell - namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow app.kubernetes.io/part-of: abevalflow diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 8ce8940..14c7c5e 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: evaluate - namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow app.kubernetes.io/component: phase diff --git a/pipeline/tasks/phases/prepare.yaml b/pipeline/tasks/phases/prepare.yaml index cdc7d79..86bc0be 100644 --- a/pipeline/tasks/phases/prepare.yaml +++ b/pipeline/tasks/phases/prepare.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: prepare - namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow app.kubernetes.io/component: phase diff --git a/pipeline/tasks/post/analyze-and-check-degradation.yaml b/pipeline/tasks/post/analyze-and-check-degradation.yaml index 3140f6e..10ab0ae 100644 --- a/pipeline/tasks/post/analyze-and-check-degradation.yaml +++ b/pipeline/tasks/post/analyze-and-check-degradation.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: analyze-and-check-degradation - namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow spec: diff --git a/pipeline/tasks/post/store.yaml b/pipeline/tasks/post/store.yaml index 32d27aa..bf0dcc3 100644 --- a/pipeline/tasks/post/store.yaml +++ b/pipeline/tasks/post/store.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: store - namespace: ab-eval-flow spec: description: >- Persists A/B evaluation results and publishes artifacts. Step 1 writes From fa6051d6aad57a2f3f16e2e93cfb5280910f385f Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Mon, 12 Oct 2026 11:47:05 +0100 Subject: [PATCH 08/15] fix(pipeline): default endpoints to namespace-local short names llm-api-base, llm-base-url and mlflow-tracking-uri defaulted to gz-forge-eval / ab-eval-flow FQDNs, so the Pipeline only worked in those namespaces. Short service names (litellm, abevalflow-mlflow) resolve inside whatever namespace the run is created in. The example PipelineRun no longer duplicates the endpoint defaults (single source of truth in the Pipeline) and keeps no hostAliases or namespace pins. Tests pin the Pipeline defaults and the example's non-duplication. Co-Authored-By: Claude Code --- pipeline/pipelines/ci-pipeline-openshell.yaml | 11 +++++++--- .../runs/openshell-openclaw-pipelinerun.yaml | 22 ++++++++++--------- tests/test_openshell_pipeline_profile.py | 20 +++++++++++++++-- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index b06026a..97102ad 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -41,13 +41,16 @@ spec: default: "rits/zai-org/GLM-5-3-Flash" - name: llm-api-base type: string - default: "http://litellm.gz-forge-eval.svc:4000" + # Short service name: resolves inside whatever namespace the run is + # created in (the stack's own LiteLLM from config/litellm/). + default: "http://litellm:4000" - name: llm-api-key type: string default: "mock" - name: llm-base-url type: string - default: "http://litellm.ab-eval-flow.svc:4000/v1" + # Short service name; used only when AI generation is enabled. + default: "http://litellm:4000/v1" - name: llm-generation-model type: string default: "rits/zai-org/GLM-5-3-Flash" @@ -146,7 +149,9 @@ spec: description: Export MLFLOW_TRACKING_URI for AEH and run log_aeh_mlflow.py after evaluate - name: mlflow-tracking-uri type: string - default: "http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000" + # Short service name: the MLflow deployment from config/mlflow/ in the + # run's own namespace. + default: "http://abevalflow-mlflow:5000" description: In-cluster MLflow tracking URI - name: enable-aeh-regression-check type: string diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index 4fd3340..0c10340 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -3,8 +3,10 @@ # oc apply -n -f pipeline/pipelines/ci-pipeline-openshell.yaml # oc create -n -f pipeline/runs/openshell-openclaw-pipelinerun.yaml # -# Service endpoints below use short names, which resolve inside the namespace -# the run is created in, so this file is namespace-agnostic. +# Service endpoints default to namespace-local short names (litellm, +# abevalflow-mlflow, openshell) in the Pipeline, so they resolve inside +# whatever namespace the run is created in; this file is namespace-agnostic +# and does not override them. # Do not put M365 tokens in this file; use Secret openshell-credentials in the namespace. apiVersion: tekton.dev/v1 kind: PipelineRun @@ -32,10 +34,11 @@ spec: # value: "" # - name: agent-eval-harness-repo-revision # value: "" - - name: openshell-gateway-endpoint - # Stable, certificate-valid alias installed by - # config/forge-saw/openshell-eval-alias.yaml in the run namespace. - value: "https://openshell:17670" + # Service endpoints (openshell gateway, litellm, mlflow) are not set here: + # their Pipeline defaults are namespace-local short names that resolve in + # this run's namespace. Override only for a non-default gateway. + # - name: openshell-gateway-endpoint + # value: "https://openshell:17670" - name: openshell-mtls-secret value: "openshell-mtls" - name: openshell-ai-gateway-ca-secret @@ -51,8 +54,9 @@ spec: value: "registry.access.redhat.com/ubi9/python-311:9.6" - name: llm-model value: "rits/zai-org/GLM-5-3-Flash" - - name: llm-api-base - value: "http://litellm:4000" + # The llm-api-key default is "mock" for LiteLLM deployments that accept + # it; put the real key in a Secret-referencing param override instead of + # committing it here. - name: llm-api-key value: "mock" - name: aeh-model-override @@ -61,8 +65,6 @@ spec: value: "judge-glm-5-3" - name: enable-mlflow value: "true" - - name: mlflow-tracking-uri - value: "http://abevalflow-mlflow:5000" taskRunTemplate: serviceAccountName: pipeline timeouts: diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index 687fcfe..3049b10 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -80,11 +80,27 @@ def test_gateway_alias_service_selects_agent_vmi(self): ports = alias["spec"]["ports"] assert ports == [{"name": "gateway", "port": 17670, "protocol": "TCP", "targetPort": 17670}] + def test_pipeline_defaults_are_namespace_agnostic(self): + defaults = _param_defaults(_load(PIPELINE)["spec"]) + for name in ("llm-api-base", "llm-base-url", "mlflow-tracking-uri", "openshell-gateway-endpoint"): + value = defaults[name] + assert ".svc" not in value, f"{name} pins a namespace: {value}" + assert defaults["llm-api-base"] == "http://litellm:4000" + assert defaults["llm-base-url"] == "http://litellm:4000/v1" + assert defaults["mlflow-tracking-uri"] == "http://abevalflow-mlflow:5000" + # Single source of truth: the example run must not duplicate the + # endpoint defaults the Pipeline already provides. + run = _load(REPO / "pipeline" / "runs" / "openshell-openclaw-pipelinerun.yaml") + example_params = {p["name"] for p in run["spec"]["params"]} + for name in ("llm-api-base", "llm-base-url", "mlflow-tracking-uri", "openshell-gateway-endpoint"): + assert name not in example_params, f"example run duplicates Pipeline default {name}" + def test_example_run_is_namespace_agnostic(self): run = _load(REPO / "pipeline" / "runs" / "openshell-openclaw-pipelinerun.yaml") params = {p["name"]: p["value"] for p in run["spec"]["params"]} - for name in ("openshell-gateway-endpoint", "llm-api-base", "mlflow-tracking-uri"): - value = params[name] + # Endpoint defaults live in the Pipeline and are namespace-local + # short names; the example must not reintroduce namespace pins. + for name, value in params.items(): assert ".svc.cluster.local" not in value, f"{name} pins a namespace: {value}" pod_template = run["spec"]["taskRunTemplate"].get("podTemplate", {}) assert "hostAliases" not in pod_template From 56af0b894b8ae4475ebc0a853ce7909fd67c17c7 Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Mon, 12 Oct 2026 11:48:33 +0100 Subject: [PATCH 09/15] feat(pipeline): select gateway auth path with openshell-auth-mode The evaluate Task inferred mTLS-vs-OIDC from the endpoint's hostname shape, which silently picks the wrong path for any endpoint that does not look like *.svc.cluster.local or host.containers.internal. Add an explicit openshell-auth-mode param, defaulting to mtls to match the certificate-valid openshell alias endpoint, and dispatch on it in the evaluate script; oidc without its Secret fails fast with a clear error. Tests pin the param plumbing through Pipeline and Task and that the endpoint-shape sniffing is gone. Co-Authored-By: Claude Code --- pipeline/pipelines/ci-pipeline-openshell.yaml | 9 ++ pipeline/tasks/phases/evaluate.yaml | 127 ++++++++++-------- tests/test_openshell_pipeline_profile.py | 28 ++++ 3 files changed, 111 insertions(+), 53 deletions(-) diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index 97102ad..ffefa0a 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -119,6 +119,13 @@ spec: type: string # Installed by config/forge-saw/openshell-eval-alias.yaml in the run namespace. default: "https://openshell:17670" + - name: openshell-auth-mode + type: string + # Explicit gateway authentication path. The default alias endpoint is + # certificate-valid for the name "openshell", so the default mode is + # mtls; choose oidc only when the gateway is registered for + # client-credentials OIDC instead of client certificates. + default: "mtls" - name: openshell-sandbox-image type: string default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" @@ -292,6 +299,8 @@ spec: value: $(params.openshell-cli-version) - name: openshell-gateway-endpoint value: $(params.openshell-gateway-endpoint) + - name: openshell-auth-mode + value: $(params.openshell-auth-mode) - name: openshell-sandbox-image value: $(params.openshell-sandbox-image) - name: openshell-provider diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 14c7c5e..4036e8e 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -193,6 +193,15 @@ spec: description: >- In-cluster NVIDIA OpenShell gateway (namespace openshell, gRPC :8080). Override only if using a different gateway (e.g. forge-saw :17670). + - name: openshell-auth-mode + type: string + default: "mtls" + description: >- + Explicit gateway authentication path, replacing endpoint-shape + sniffing. mtls registers the gateway with the staged client + certificate and requires a server-name-valid HTTPS endpoint (such as + the openshell alias Service, https://openshell:17670). oidc registers + with client-credentials OIDC from the openshell OIDC Secret. - name: openshell-sandbox-image type: string default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" @@ -1676,6 +1685,8 @@ spec: value: $(workspaces.source.path)/reports - name: OPENSHELL_GATEWAY_ENDPOINT value: "$(params.openshell-gateway-endpoint)" + - name: OPENSHELL_AUTH_MODE + value: "$(params.openshell-auth-mode)" - name: OPENSHELL_GATEWAY_NAME value: "ci-gateway" - name: AGENT_EVAL_OPENSHELL_WORKSPACE @@ -1767,60 +1778,70 @@ spec: exit 1 fi - # Native mTLS requires server-name verification AND a client identity. - # --gateway-insecure in this CLI drops client authentication. - if [[ "${OPENSHELL_GATEWAY_ENDPOINT}" == *".svc.cluster.local:"* ]] \ - || [[ "${OPENSHELL_GATEWAY_ENDPOINT}" == "https://host.containers.internal:"* ]]; then - unset OPENSHELL_GATEWAY_INSECURE - CLI_GATEWAY_ENDPOINT="$OPENSHELL_GATEWAY_ENDPOINT" - # Use the CLI's supported local/mTLS registration path. This lets - # the installed CLI create metadata in the exact shape it expects - # and associate the staged client bundle with the gateway profile. - openshell gateway add "$CLI_GATEWAY_ENDPOINT" \ - --local \ - --name "$GW_NAME" \ - >/tmp/openshell-gateway-add.log 2>&1 || { - grep -vE 'client_secret|token|secret' /tmp/openshell-gateway-add.log || true + # The authentication path is selected by the explicit openshell-auth-mode + # parameter, never by the endpoint's hostname shape. Native mTLS requires + # server-name verification AND a client identity; --gateway-insecure in + # this CLI drops client authentication. + case "${OPENSHELL_AUTH_MODE:-mtls}" in + mtls) + unset OPENSHELL_GATEWAY_INSECURE + CLI_GATEWAY_ENDPOINT="$OPENSHELL_GATEWAY_ENDPOINT" + # Use the CLI's supported local/mTLS registration path. This lets + # the installed CLI create metadata in the exact shape it expects + # and associate the staged client bundle with the gateway profile. + openshell gateway add "$CLI_GATEWAY_ENDPOINT" \ + --local \ + --name "$GW_NAME" \ + >/tmp/openshell-gateway-add.log 2>&1 || { + grep -vE 'client_secret|token|secret' /tmp/openshell-gateway-add.log || true + exit 1 + } + POST_ADD_MTLS_DST="${XDG_CONFIG_HOME}/openshell/gateways/${GW_NAME}/mtls" + mkdir -p "$POST_ADD_MTLS_DST" + cp -f "$MTLS_SRC/ca.crt" "$POST_ADD_MTLS_DST/" + cp -f "$MTLS_SRC/tls.crt" "$POST_ADD_MTLS_DST/" + cp -f "$MTLS_SRC/tls.key" "$POST_ADD_MTLS_DST/" + chmod 600 "$POST_ADD_MTLS_DST"/* + echo "OpenShell mTLS material restaged after gateway registration" + export OPENSHELL_GATEWAY="$GW_NAME" + export OPENSHELL_GATEWAY_NAME="$GW_NAME" + export OPENSHELL_GATEWAY_ENDPOINT="$CLI_GATEWAY_ENDPOINT" + echo "OpenShell mTLS gateway configured: ${GW_NAME}" + ;; + oidc) + if [ -z "${OPENSHELL_OIDC_CLIENT_SECRET:-}" ] || [ -z "${OPENSHELL_OIDC_CLIENT_ID:-}" ] \ + || [ -z "${OPENSHELL_OIDC_ISSUER:-}" ]; then + echo "ERROR: openshell-auth-mode=oidc requires the openshell OIDC Secret (issuer, client-id, client-secret)" >&2 exit 1 - } - POST_ADD_MTLS_DST="${XDG_CONFIG_HOME}/openshell/gateways/${GW_NAME}/mtls" - mkdir -p "$POST_ADD_MTLS_DST" - cp -f "$MTLS_SRC/ca.crt" "$POST_ADD_MTLS_DST/" - cp -f "$MTLS_SRC/tls.crt" "$POST_ADD_MTLS_DST/" - cp -f "$MTLS_SRC/tls.key" "$POST_ADD_MTLS_DST/" - chmod 600 "$POST_ADD_MTLS_DST"/* - echo "OpenShell mTLS material restaged after gateway registration" - export OPENSHELL_GATEWAY="$GW_NAME" - export OPENSHELL_GATEWAY_NAME="$GW_NAME" - export OPENSHELL_GATEWAY_ENDPOINT="$CLI_GATEWAY_ENDPOINT" - echo "OpenShell mTLS gateway configured: ${GW_NAME}" - elif [ -n "${OPENSHELL_OIDC_CLIENT_SECRET:-}" ] && [ -n "${OPENSHELL_OIDC_CLIENT_ID:-}" ] \ - && [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then - openshell gateway add "$OPENSHELL_GATEWAY_ENDPOINT" \ - --name "$GW_NAME" \ - --oidc-issuer "$OPENSHELL_OIDC_ISSUER" \ - --oidc-client-id "$OPENSHELL_OIDC_CLIENT_ID" \ - --oidc-audience "${OPENSHELL_OIDC_AUDIENCE:-openshell-cli}" \ - >/tmp/openshell-gateway-add.log 2>&1 || { - grep -vE 'client.secret|secret|token' /tmp/openshell-gateway-add.log || true - exit 1 - } - OIDC_TOKEN_CACHE="$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" - OIDC_REFRESH_HELPER="$(workspaces.source.path)/_pipeline/scripts/refresh_openshell_oidc.py" - python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" - echo "OpenShell OIDC token cache populated via client credentials" - # A case can run for 15 minutes, while the client-credentials token - # expires after roughly five. Refresh the CLI cache throughout this - # step so later artifact downloads and sandbox creation stay authorized. - python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" --interval 120 & - OIDC_REFRESH_PID=$! - trap 'kill "$OIDC_REFRESH_PID" 2>/dev/null || true' EXIT - echo "OpenShell OIDC token refresher started" - export OPENSHELL_GATEWAY="$GW_NAME" - echo "OpenShell OIDC gateway configured: ${GW_NAME}" - else - echo "OpenShell OIDC Secret not configured; using existing gateway auth" - fi + fi + openshell gateway add "$OPENSHELL_GATEWAY_ENDPOINT" \ + --name "$GW_NAME" \ + --oidc-issuer "$OPENSHELL_OIDC_ISSUER" \ + --oidc-client-id "$OPENSHELL_OIDC_CLIENT_ID" \ + --oidc-audience "${OPENSHELL_OIDC_AUDIENCE:-openshell-cli}" \ + >/tmp/openshell-gateway-add.log 2>&1 || { + grep -vE 'client.secret|secret|token' /tmp/openshell-gateway-add.log || true + exit 1 + } + OIDC_TOKEN_CACHE="$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" + OIDC_REFRESH_HELPER="$(workspaces.source.path)/_pipeline/scripts/refresh_openshell_oidc.py" + python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" + echo "OpenShell OIDC token cache populated via client credentials" + # A case can run for 15 minutes, while the client-credentials token + # expires after roughly five. Refresh the CLI cache throughout this + # step so later artifact downloads and sandbox creation stay authorized. + python3 "$OIDC_REFRESH_HELPER" --cache "$OIDC_TOKEN_CACHE" --interval 120 & + OIDC_REFRESH_PID=$! + trap 'kill "$OIDC_REFRESH_PID" 2>/dev/null || true' EXIT + echo "OpenShell OIDC token refresher started" + export OPENSHELL_GATEWAY="$GW_NAME" + echo "OpenShell OIDC gateway configured: ${GW_NAME}" + ;; + *) + echo "ERROR: openshell-auth-mode must be 'mtls' or 'oidc' (got '${OPENSHELL_AUTH_MODE}')" >&2 + exit 1 + ;; + esac # Secret mounts flat at $HOME/.config/openshell. OpenShell CLI expects # gateways//mtls/{ca.crt,tls.crt,tls.key}. diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index 3049b10..a1bec0c 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -115,6 +115,34 @@ def test_cleanup_task_is_best_effort(self): assert "--request-timeout=20s" in step["script"] assert step["image"].startswith("image-registry.openshift-image-registry.svc:5000/") + def test_explicit_auth_mode_selects_gateway_path(self): + # The default alias endpoint must select the mTLS path through an + # explicit parameter, never through endpoint hostname shape. + pipeline_defaults = _param_defaults(_load(PIPELINE)["spec"]) + assert pipeline_defaults["openshell-auth-mode"] == "mtls" + assert pipeline_defaults["openshell-gateway-endpoint"] == "https://openshell:17670" + evaluate = next(t for t in _load(PIPELINE)["spec"]["tasks"] if t["name"] == "evaluate") + forwarded = {p["name"]: p["value"] for p in evaluate["params"]} + assert forwarded["openshell-auth-mode"] == "$(params.openshell-auth-mode)" + + task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + assert _param_defaults(task["spec"])["openshell-auth-mode"] == "mtls" + step = next(s for s in task["spec"]["steps"] if s["name"] == "aeh-openshell-eval") + env = next(e for e in step["env"] if e["name"] == "OPENSHELL_AUTH_MODE") + assert env["value"] == "$(params.openshell-auth-mode)" + script = step["script"] + case_head = 'case "${OPENSHELL_AUTH_MODE:-mtls}" in' + assert case_head in script + mtls_branch = script.split(case_head, 1)[1].split("oidc)", 1)[0] + assert "unset OPENSHELL_GATEWAY_INSECURE" in mtls_branch + assert "--local" in mtls_branch + oidc_branch = script.split("oidc)", 1)[1].split("*)", 1)[0] + assert "openshell-auth-mode=oidc requires" in oidc_branch + assert "OPENSHELL_OIDC_ISSUER" in oidc_branch + # The endpoint-shape sniffing that used to pick the path is gone. + assert '*".svc.cluster.local:"*' not in script + assert '"https://host.containers.internal:"*' not in script + def test_evaluate_openshell_step_logs_mlflow(self): evaluate = (REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml").read_text() openshell = evaluate.split("name: aeh-openshell-eval", 1)[1].split("name: aeh-pairwise", 1)[0] From 71b8ea1cb8d1be0f93260b27abf8ac4c89d4b171 Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Mon, 12 Oct 2026 11:49:23 +0100 Subject: [PATCH 10/15] feat(probe): check the pipeline's gateway contract layer by layer eval-stack-probe targeted the VMI's raw openshell-saw-agent-gateway service, which is not the endpoint the Pipeline uses, and reported a single curl status line. Point it at the same openshell-gateway-endpoint param and default as the Pipeline and probe three layers separately: TCP reachability, TLS handshake with SAN display (full chain validation when the forge-ai-gateway-ca Secret exists, optional so absent Secrets do not fail the probe), and an HTTP status. A mismatch is now attributable to one layer instead of a generic connection error. Co-Authored-By: Claude Code --- pipeline/tasks/eval-stack-probe.yaml | 67 +++++++++++++++++++++--- tests/test_openshell_pipeline_profile.py | 22 ++++++++ 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/pipeline/tasks/eval-stack-probe.yaml b/pipeline/tasks/eval-stack-probe.yaml index cee3d4c..acf19ae 100644 --- a/pipeline/tasks/eval-stack-probe.yaml +++ b/pipeline/tasks/eval-stack-probe.yaml @@ -2,17 +2,41 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: eval-stack-probe - # Deliberately not labelled app.kubernetes.io/part-of: abevalflow — the probe - # must be seen by NetworkPolicy as an ordinary Tekton pod, otherwise it passes - # through the stack's own allow-rules and reports connectivity the pipeline - # does not actually have. + # A baseline connectivity probe: it runs as an ordinary Tekton pod, without + # the pipeline's network identity (the canonical NetworkPolicies select the + # evaluate Task's pods, e.g. by tekton.dev/task=evaluate), so it verifies + # the stack's baseline reachability rather than the exact allowances the + # pipeline runs under. Diagnose pipeline-path connectivity from the evaluate + # Task's own logs; use this probe before/after infra changes. spec: description: >- Probes every dependency an ABEvalFlow pipeline pod needs. Run it before and - after NetworkPolicy changes to see exactly which paths are blocked. + after NetworkPolicy changes to see exactly which paths are blocked. The + agent-gateway probe targets the same openshell alias endpoint the Pipeline + uses, in three layers: TCP reachability, TLS handshake/validation, and + HTTP response. + params: + - name: openshell-gateway-endpoint + type: string + # Keep in sync with the abevalflow-pipeline-openshell default: the probe + # must test the endpoint the Pipeline actually uses (the openshell alias + # Service), not the VMI's raw service name. tests assert the two + # defaults stay identical. + default: "https://openshell:17670" steps: - name: probe image: registry.access.redhat.com/ubi9/python-311:9.6 + env: + # When present, TLS validation and the HTTP probe verify the gateway + # certificate chain; without it, the TLS layer reports handshake + + # SANs only. Optional so the probe still runs in namespaces that do + # not have this Secret. + - name: GATEWAY_CA_CERT + valueFrom: + secretKeyRef: + name: forge-ai-gateway-ca + key: ca.crt + optional: true script: | #!/usr/bin/env bash NS="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null || echo current)" @@ -31,6 +55,35 @@ spec: echo " FAIL $1 -> $(tr -d '\n' /dev/null; then + echo " PASS $name tcp ($host:$port)" + else + echo " FAIL $name tcp -> $host:$port not reachable (missing Service or blocked egress)"; fail=1; return + fi + if [ -n "${GATEWAY_CA_CERT:-}" ]; then + printf '%s\n' "$GATEWAY_CA_CERT" >/tmp/gateway-ca.pem + tls_extra="-CAfile /tmp/gateway-ca.pem -verify_return_error" + curl_ca="--cacert /tmp/gateway-ca.pem" + tls_note="chain validated against the mounted gateway CA" + fi + if echo | timeout 12 openssl s_client -connect "$host:$port" -servername "$host" $tls_extra 2>/dev/null \ + | openssl x509 -noout -ext subjectAltName >/tmp/gateway-san 2>/dev/null; then + echo " PASS $name tls ($tls_note; $(sed -n '2p' /tmp/gateway-san | cut -c1-120))" + else + echo " FAIL $name tls -> handshake or certificate validation failed for $host:$port"; fail=1; return + fi + local code + if code=$(curl -sS --max-time 10 -o /dev/null -w "%{http_code}" $curl_ca "$endpoint" 2>/tmp/gw-err); then + echo " PASS $name http (HTTP $code)" + else + echo " FAIL $name http -> $(tr -d '\n' Date: Mon, 12 Oct 2026 11:50:24 +0100 Subject: [PATCH 11/15] feat(cleanup): surface PVC cleanup outcome instead of silent skip The finally cleanup Task is best-effort, so a failed delete was invisible: the PipelineRun stayed green with a leaked PVC. Report a cleanup-status task and PipelineRun result (deleted / none / api-unavailable / failed) and label a PVC that could not be deleted abevalflow.io/cleanup-failed=true so leaks are findable across runs. Labelling needs the patch verb on persistentvolumeclaims in RBAC. Also strip the leftover ab-eval-flow namespace pins from rbac.yaml so the manifests apply into any namespace with oc -n "$NS" apply, which the manual trigger guide's install step relies on. Co-Authored-By: Claude Code --- config/rbac.yaml | 132 +++++++++--------- pipeline/pipelines/ci-pipeline-openshell.yaml | 5 + pipeline/tasks/post/cleanup_pvc.yaml | 80 ++++++++--- tests/test_openshell_pipeline_profile.py | 30 ++++ 4 files changed, 158 insertions(+), 89 deletions(-) diff --git a/config/rbac.yaml b/config/rbac.yaml index 8aec87f..91791df 100644 --- a/config/rbac.yaml +++ b/config/rbac.yaml @@ -1,68 +1,64 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: pipeline - namespace: ab-eval-flow - labels: - app.kubernetes.io/part-of: abevalflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: pipeline-image-builder - namespace: ab-eval-flow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:image-builder -subjects: - - kind: ServiceAccount - name: pipeline - namespace: ab-eval-flow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: pipeline-trial-manager - namespace: ab-eval-flow -rules: - # Pod management for Harbor trial pods and AEH KubernetesEnvironment - # AEH uses: create_namespaced_pod, read_namespaced_pod, delete_namespaced_pod - - apiGroups: [""] - resources: [pods] - verbs: [create, get, list, watch, delete] - # Pod exec for Harbor/AEH to run commands in trial pods - # AEH uses: connect_get_namespaced_pod_exec - - apiGroups: [""] - resources: [pods/exec] - verbs: [create] - - apiGroups: [""] - resources: [pods/log] - verbs: [get] - - apiGroups: [""] - resources: [events] - verbs: [get, list] - - apiGroups: [""] - resources: [secrets] - verbs: [get] - resourceNames: [ab-eval-db-credentials, litellm-credentials, minio-credentials, github-token, llm-credentials] - - apiGroups: [""] - resources: [configmaps] - verbs: [get, list] - - apiGroups: [""] - resources: [persistentvolumeclaims] - verbs: [get, list, create, delete] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: pipeline-trial-manager - namespace: ab-eval-flow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: pipeline-trial-manager -subjects: - - kind: ServiceAccount - name: pipeline - namespace: ab-eval-flow +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pipeline + labels: + app.kubernetes.io/part-of: abevalflow +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pipeline-image-builder +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:image-builder +subjects: + - kind: ServiceAccount + name: pipeline +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pipeline-trial-manager +rules: + # Pod management for Harbor trial pods and AEH KubernetesEnvironment + # AEH uses: create_namespaced_pod, read_namespaced_pod, delete_namespaced_pod + - apiGroups: [""] + resources: [pods] + verbs: [create, get, list, watch, delete] + # Pod exec for Harbor/AEH to run commands in trial pods + # AEH uses: connect_get_namespaced_pod_exec + - apiGroups: [""] + resources: [pods/exec] + verbs: [create] + - apiGroups: [""] + resources: [pods/log] + verbs: [get] + - apiGroups: [""] + resources: [events] + verbs: [get, list] + - apiGroups: [""] + resources: [secrets] + verbs: [get] + resourceNames: [ab-eval-db-credentials, litellm-credentials, minio-credentials, github-token, llm-credentials] + - apiGroups: [""] + resources: [configmaps] + verbs: [get, list] + - apiGroups: [""] + resources: [persistentvolumeclaims] + # patch labels a PVC abevalflow.io/cleanup-failed=true when the finally + # cleanup task cannot delete it, so leaks are operator-visible. + verbs: [get, list, create, delete, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pipeline-trial-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pipeline-trial-manager +subjects: + - kind: ServiceAccount + name: pipeline diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index ffefa0a..9dc697d 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -205,6 +205,11 @@ spec: - name: scorecard-recommendation description: "Unified scorecard recommendation (pass/warn/fail)" value: $(tasks.analyze.results.scorecard-recommendation) + - name: cleanup-status + description: >- + Finally PVC cleanup outcome: deleted / none / api-unavailable / + failed. Non-blocking: a failed cleanup never fails the run. + value: $(finally.cleanup.results.cleanup-status) workspaces: - name: shared-workspace diff --git a/pipeline/tasks/post/cleanup_pvc.yaml b/pipeline/tasks/post/cleanup_pvc.yaml index c050958..5054659 100644 --- a/pipeline/tasks/post/cleanup_pvc.yaml +++ b/pipeline/tasks/post/cleanup_pvc.yaml @@ -1,47 +1,85 @@ -apiVersion: tekton.dev/v1 -kind: Task -metadata: - name: cleanup-pvc -spec: - description: >- - Deletes the workspace PVC after a pipeline run completes. - Runs as a finally task to prevent PVC quota exhaustion. - params: - - name: pipelinerun-name - type: string - description: Name of the PipelineRun whose PVCs should be cleaned up +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: cleanup-pvc +spec: + description: >- + Deletes the workspace PVC after a pipeline run completes. Runs as a + finally task to prevent PVC quota exhaustion. Non-blocking by design: + a cleanup failure never fails the PipelineRun. The outcome is reported + in the cleanup-status result (deleted / none / api-unavailable / + failed), and a PVC that could not be deleted is labelled + abevalflow.io/cleanup-failed=true so operators can find and alert on it. + params: + - name: pipelinerun-name + type: string + description: Name of the PipelineRun whose PVCs should be cleaned up + results: + - name: cleanup-status + description: >- + deleted: all owned PVCs deleted; none: no owned PVCs found; + api-unavailable: the lookup could not reach the API, deletion + unverified; failed: one or more deletes failed (PVCs labelled + abevalflow.io/cleanup-failed=true). steps: - name: delete-pvc - image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + image: image-registry.openshift-image-registry.svc:5000/openshell/cli:latest onError: continue script: | #!/usr/bin/env bash set -uo pipefail - - # Get current namespace from service account - NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - - echo "Cleaning up PVCs for PipelineRun $(params.pipelinerun-name) in namespace ${NAMESPACE}..." + + # Get current namespace from service account + NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + RESULT_PATH="$(results.cleanup-status.path)" + + echo "Cleaning up PVCs for PipelineRun $(params.pipelinerun-name) in namespace ${NAMESPACE}..." pvcs="" + lookup_ok=0 for attempt in 1 2 3; do if pvcs=$(oc get pvc -n "${NAMESPACE}" --request-timeout=20s \ -o jsonpath='{range .items[?(@.metadata.ownerReferences[0].name=="$(params.pipelinerun-name)")]}{.metadata.name}{"\n"}{end}'); then + lookup_ok=1 break fi echo "WARNING: PVC lookup attempt ${attempt}/3 failed" sleep $((attempt * 5)) done + # Non-blocking by design: the run stays green, but the outcome is + # reported so an operator can see the PVC may still exist. + if [ "$lookup_ok" -ne 1 ]; then + echo "api-unavailable" > "$RESULT_PATH" + echo "WARNING: PVC lookup failed; the API remained unavailable, deletion unverified" + exit 0 + fi + if [ -z "$pvcs" ]; then - echo "No owned PVCs found, or the API remained unavailable; cleanup is best-effort" + echo "none" > "$RESULT_PATH" + echo "No owned PVCs found; nothing to clean up" exit 0 fi + failed=0 while read -r pvc; do if [ -n "$pvc" ]; then echo "Deleting PVC: $pvc" - oc delete pvc "$pvc" -n "${NAMESPACE}" --wait=false --request-timeout=20s \ - || echo "WARNING: failed to delete PVC $pvc" + if ! oc delete pvc "$pvc" -n "${NAMESPACE}" --wait=false --request-timeout=20s; then + failed=1 + echo "WARNING: failed to delete PVC $pvc" + # Operator-visible marker on the leaked PVC: alert on the + # abevalflow.io/cleanup-failed label instead of a log line. + oc label pvc "$pvc" -n "${NAMESPACE}" \ + abevalflow.io/cleanup-failed=true --overwrite --request-timeout=20s \ + || echo "WARNING: could not label PVC $pvc (needs patch on persistentvolumeclaims)" + fi fi done <<< "$pvcs" + + if [ "$failed" -ne 0 ]; then + echo "failed" > "$RESULT_PATH" + echo "WARNING: PVC cleanup failed for this run; labelled PVCs remain for an operator" + exit 0 + fi + echo "deleted" > "$RESULT_PATH" echo "PVC cleanup complete" diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index 267f3bd..3420cc4 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -115,6 +115,36 @@ def test_cleanup_task_is_best_effort(self): assert "--request-timeout=20s" in step["script"] assert step["image"].startswith("image-registry.openshift-image-registry.svc:5000/") + def test_cleanup_failures_are_operator_visible(self): + task = _load(REPO / "pipeline" / "tasks" / "post" / "cleanup_pvc.yaml") + assert "cleanup-status" in {r["name"] for r in task["spec"]["results"]} + script = task["spec"]["steps"][0]["script"] + assert "$(results.cleanup-status.path)" in script + for outcome in ("api-unavailable", "none", "failed", "deleted"): + assert f'echo "{outcome}" > "$RESULT_PATH"' in script + # A failed delete labels the leaked PVC instead of failing the run. + assert "abevalflow.io/cleanup-failed=true" in script + pipeline = _load(PIPELINE) + result = next(r for r in pipeline["spec"]["results"] if r["name"] == "cleanup-status") + assert result["value"] == "$(finally.cleanup.results.cleanup-status)" + # Labelling the PVC needs patch on persistentvolumeclaims. + rbac = list(yaml.safe_load_all((REPO / "config" / "rbac.yaml").read_text())) + role = next(o for o in rbac if o and o["kind"] == "Role" and o["metadata"]["name"] == "pipeline-trial-manager") + pvc_rule = next(r for r in role["rules"] if "persistentvolumeclaims" in r["resources"]) + assert "patch" in pvc_rule["verbs"] + + def test_rbac_applies_into_any_namespace(self): + # metadata.namespace would make `oc apply -n X` fail, and a subject + # namespace would bind another namespace's ServiceAccount; subjects + # without a namespace default to the RoleBinding's own namespace. + rbac = list(yaml.safe_load_all((REPO / "config" / "rbac.yaml").read_text())) + for obj in rbac: + assert obj is not None + assert "namespace" not in obj["metadata"], obj["metadata"]["name"] + if obj["kind"] == "RoleBinding": + for subject in obj["subjects"]: + assert "namespace" not in subject + def test_explicit_auth_mode_selects_gateway_path(self): # The default alias endpoint must select the mTLS path through an # explicit parameter, never through endpoint hostname shape. From 8c8acb46f36c6acfa09a28754be65051af6b5503 Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Mon, 12 Oct 2026 11:50:47 +0100 Subject: [PATCH 12/15] docs(guide): install the namespace-agnostic profile directly Replace the render-and-patch installer (namespace string replacement, host.containers.internal endpoint, hostAliases with a gateway IP) with plain oc -n "$NS" apply of the now namespace-agnostic manifests and a revision-only run render. Document the openshell alias Service and its SAN check, the openshell-auth-mode choice, the cleanup-status result with the abevalflow.io/cleanup-failed label, and correct the note on the storage/MLflow/LiteLLM deployables: they still pin ab-eval-flow and need their namespace lines adjusted. Co-Authored-By: Claude Code --- Docs/manual_trigger_guide.md | 214 +++++++++++++++++++---------------- 1 file changed, 119 insertions(+), 95 deletions(-) diff --git a/Docs/manual_trigger_guide.md b/Docs/manual_trigger_guide.md index 1398974..aba5510 100644 --- a/Docs/manual_trigger_guide.md +++ b/Docs/manual_trigger_guide.md @@ -14,26 +14,24 @@ The commands below are operator instructions; reading this guide does not deploy ### 1. Choose the namespace and matching source revisions -Run from an Agentic Eval Flow checkout containing the native-mTLS fixes. The -older TLS bridge implementation is not the validated configuration. Use an -isolated checkout if your current checkout has local changes. Requirements: -authenticated `oc`, OpenShift Pipelines, Python 3 with PyYAML, a writable storage -class, sufficient quota, and permission to manage resources in your namespace. +Run from an Agentic Eval Flow checkout containing the namespace-agnostic +OpenShell profile. Use an isolated checkout if your current checkout has local +changes. Requirements: authenticated `oc`, OpenShift Pipelines, Python 3 with +PyYAML, a writable storage class, sufficient quota, and permission to manage +resources in your namespace. ```bash export NS="[NAMESPACE]" # replace this literal before running -export FLOW_REV="[FLOW_REVISION]" # branch/tag containing the fixes, or commit SHA -export HARNESS_REV="[HARNESS_REVISION]" # OpenShell-capable AEH fork revision +export FLOW_REV="[FLOW_REVISION]" # branch/tag or commit SHA to evaluate +export HARNESS_REV="[HARNESS_REVISION]" # optional; defaults to the Pipeline's pinned harness commit oc get namespace "$NS" oc -n "$NS" get vm,svc,pvc -oc -n "$NS" get endpointslices -l kubernetes.io/service-name=openshell-saw-agent-gateway -export GATEWAY_IP=$(oc -n "$NS" get svc openshell-saw-agent-gateway -o jsonpath='{.spec.clusterIP}') ``` -At the time of the fixes, both repositories used `feat/aeh-openshell-openclaw` -(AEH repository: `GuyZivRH/agent-eval-harness`). Use `main` only after the relevant -changes are merged. Pin a tested commit for reproducibility. A branch reference -clones its current tip when a new run starts; an existing run is not updated. +The Pipeline pins the AEH harness repository (`GuyZivRH/agent-eval-harness`) +to a verified commit by default; override `agent-eval-harness-repo-revision` +only for development runs. A branch reference clones its current tip when a +new run starts; an existing run is not updated. There are two independently versioned layers: installed Tekton Pipeline/Task YAML and Git-cloned Python scripts/submission files. Reapplying the matching @@ -45,30 +43,37 @@ does not update the installed Tasks. Forge SAW does not by itself provision the CI results stack. Before running, provision PostgreSQL (`config/storage/postgres.yaml`), MinIO (`config/storage/minio.yaml`), MLflow (`config/mlflow/`), and the judge's LiteLLM -service (`config/litellm/`) if they are absent. Render their namespace, internal -URLs, storage class, credentials and model upstream for your installation first. +service (`config/litellm/`) if they are absent. Render their internal URLs, +storage class, credentials and model upstream for your installation first. Do not apply sample Secret values or overwrite an existing Forge deployment. The expected service ports are PostgreSQL 5432, MinIO S3 9000 (console 9001), -MLflow 5000 and LiteLLM 4000. The source manifests contain deployment-specific -namespaces and endpoints: `oc -n` does not override `metadata.namespace`, -RoleBinding subject namespaces, or embedded DNS names. +MLflow 5000 and LiteLLM 4000. The Pipeline's endpoint defaults are +namespace-agnostic: short service names such as `http://litellm:4000`, +`http://abevalflow-mlflow:5000` and `https://openshell:17670` resolve inside +whatever namespace the run is created in, so the PipelineRun needs no endpoint +overrides. The checked-in storage, MLflow and LiteLLM deployables are a +different story: they still pin `ab-eval-flow` in `metadata.namespace` and in +MLflow's host allowlist. When installing them into `$NS`, remove the +`namespace:` lines (and adjust the MLflow `--allowed-hosts` FQDNs) so +`oc -n "$NS" apply` places them correctly. Required Secret contracts, all in `[NAMESPACE]`: -- `openshell-gateway-mtls`: `ca.crt`, `tls.crt`, `tls.key`. The client leaf must +- `openshell-mtls`: `ca.crt`, `tls.crt`, `tls.key`. The client leaf must be trusted by the running gateway and authorized for sandbox/provider operations. Preserve Forge's existing CA and server/client bundle; never generate an unrelated CI CA. The Forge bootstrap may also keep `server.crt`/`server.key` here. -- `forge-agent-upstream-tls`: `ca.crt` trusted by the agent's Forge upstreams. -- `inference`: `api_key`, required by the evaluate Task. Configure the actual - model credentials in namespace Secrets and Forge providers. +- `forge-ai-gateway-ca`: `ca.crt` trusted by the agent's Forge upstreams. - `openshell-credentials`: `M365_ACCESS_TOKEN`, `M365_USER`; optional refresh configuration `M365_TENANT_ID`, `M365_CLIENT_ID`, `M365_CLIENT_SECRET`. With real values already exported, run `EVAL_NS="$NS" ./config/forge-saw/create-openshell-credentials.sh`. -- `minio-credentials`: `endpoint-url` (for example - `http://minio.[NAMESPACE].svc.cluster.local:9000`), `root-user`, `root-password`. +- `openshell-oidc-credentials`: issuer, client-id and client-secret for + client-credentials OIDC. Required only when the Pipeline runs with + `openshell-auth-mode: oidc`; the default `mtls` mode does not read it. +- `minio-credentials`: `endpoint-url` (for example `http://minio:9000`, + the namespace-local short name), `root-user`, `root-password`. - `ab-eval-db-credentials`: `database-url` for the CI results database and `postgres-password` if using the checked-in PostgreSQL deployment. Its default user/database are `abevalflow`; keep the URL and database password consistent. @@ -77,9 +82,15 @@ Required Secret contracts, all in `[NAMESPACE]`: - Private Git repositories additionally require Git credentials on the pipeline ServiceAccount. `github-token` key `token` is optional for publication features. -`openshell-oidc-credentials` is optional for this native-mTLS path. A successful -Keycloak login alone does not supply the TLS client certificate required by the -gateway. Kubernetes admin privileges also do not grant OpenShell application roles. +The model API key is not a Secret contract of the evaluate Task: it reaches the +run through the PipelineRun `llm-api-key` parameter (default `mock`, valid only +when your LiteLLM explicitly accepts that value). The namespace's own LiteLLM +deployment keeps its upstream key in its own Secret; keeping that in sync with +Vault is an ops concern outside this repository. + +A successful Keycloak login alone does not supply the TLS client certificate +required by the default `mtls` path. Kubernetes admin privileges also do not +grant OpenShell application roles. For MLflow, enable artifact serving (`mlflow-artifacts:/`) so remote clients do not try to write the server's filesystem path locally. Configure its backend and @@ -180,14 +191,30 @@ so a Helm redeploy does not remove them. The validated CLI setup stages the client bundle in `$XDG_CONFIG_HOME/openshell/gateways/ci-gateway/mtls/`, registers via `openshell gateway add ENDPOINT --local --name ci-gateway`, and restages the -bundle afterward. The corrected evaluate Task already does this. It unsets -`OPENSHELL_GATEWAY_INSECURE`; keep certificate verification enabled. +bundle afterward. The evaluate Task does this whenever `openshell-auth-mode` +is `mtls` (the Pipeline default). It unsets `OPENSHELL_GATEWAY_INSECURE`; +keep certificate verification enabled. + +The Pipeline reaches the gateway through the namespace-local `openshell` +alias Service (installed in step 5 from +`config/forge-saw/openshell-eval-alias.yaml`), so the default endpoint is +`https://openshell:17670` — no PipelineRun `hostAliases`, no cross-namespace +DNS, and no dependency on a gateway Service cluster IP. The gateway's server +certificate must include the SAN `openshell` for that name to validate. +Check what your deployment serves: + +```bash +oc -n "$NS" run cert-check --rm -i --restart=Never \ + --labels="app.kubernetes.io/part-of=abevalflow" \ + --image=registry.access.redhat.com/ubi9/python-311:9.6 \ + -- bash -c 'echo | openssl s_client -connect openshell:17670 -servername openshell 2>/dev/null | openssl x509 -noout -subject -ext subjectAltName' +``` -The working Forge server certificate includes `host.containers.internal`. -Use `https://host.containers.internal:17670` and a PipelineRun `hostAliases` -mapping to **your** `$GATEWAY_IP`. Verify your server SAN before using that name. -This alias is for CI pods; the inner sandbox's host bridge is a separate network. -An external OpenShift Route is not required for this in-cluster connection. +If your gateway serves a different name, add a matching alias Service or +override `openshell-gateway-endpoint`. An external OpenShift Route is not +required for this in-cluster connection. For gateways registered for +client-credentials OIDC instead of client certificates, run the Pipeline +with `openshell-auth-mode: oidc` and provide `openshell-oidc-credentials`. If certificates change on service restart, inspect the gateway's systemd unit and cloud-init configuration on the SAW VM. Remove unconditional @@ -197,8 +224,8 @@ them. Do not reset the state database or rotate the CA as a routine CI setup ste Compare the served certificate and trust chain to the mounted client bundle after any service restart. Use the deployment's actual unit name and TLS paths. -Before a full run, validate from a pod with the same CI label, Secret mounts and -host alias: an authenticated OpenShell operation, temporary sandbox creation, +Before a full run, validate from a pod with the same CI label and Secret +mounts: an authenticated OpenShell operation, temporary sandbox creation, readable image-owned `AGENTS.md` and skill `SKILL.md` files, and an actual GLM response. A TCP connection or working Forge web chat does not prove this CI path. Use the normal Forge startup/profile and providers; a bare `sleep infinity` @@ -213,70 +240,61 @@ persona/skills. Preserve the Forge filesystem policy and existing providers Agent GLM access uses these providers; the judge separately calls LiteLLM from the Tekton pod. Validate both paths and the upstream CA. -### 5. Render and install the matching Pipeline and Tasks +### 5. Install the Pipeline, Tasks and gateway alias -The example below renders only the required Tekton resources and RBAC into a -private temporary directory. It also renders the PipelineRun's namespace DNS -and gateway alias. Run this from the checkout of `$FLOW_REV`, not an older checkout. -It requires PyYAML. Review the files before applying. +The manifests are namespace-agnostic: apply them into `$NS` directly, from a +checkout of `$FLOW_REV`. Review the files before applying. The RBAC grants the +`pipeline` ServiceAccount pod and PVC management, including `patch`, which the +finally cleanup Task uses to label a PVC it could not delete. + +```bash +oc -n "$NS" apply -f config/rbac.yaml +oc -n "$NS" apply -f config/forge-saw/openshell-eval-alias.yaml +oc -n "$NS" apply -f pipeline/tasks/phases/prepare.yaml +oc -n "$NS" apply -f pipeline/tasks/phases/evaluate.yaml +oc -n "$NS" apply -f pipeline/tasks/post/analyze-and-check-degradation.yaml +oc -n "$NS" apply -f pipeline/tasks/post/store.yaml +oc -n "$NS" apply -f pipeline/tasks/post/cleanup_pvc.yaml +oc -n "$NS" apply -f pipeline/pipelines/ci-pipeline-openshell.yaml +oc -n "$NS" auth can-i list persistentvolumeclaims --as="system:serviceaccount:$NS:pipeline" +oc -n "$NS" auth can-i delete persistentvolumeclaims --as="system:serviceaccount:$NS:pipeline" +``` + +Optionally install the standalone connectivity probe Task +(`pipeline/tasks/eval-stack-probe.yaml`) as well; it checks the same +`https://openshell:17670` gateway endpoint the Pipeline uses, probing the TCP, +TLS and HTTP layers separately. + +The example PipelineRun runs as-is against `main` revisions. To pin revisions, +render the run with only those overrides: ```bash export CI_RENDER_DIR=$(mktemp -d) python3 - <<'PY' import json, os, pathlib, yaml -ns = os.environ['NS'] -assert ns and '[' not in ns, 'Replace [NAMESPACE] first' +assert os.environ['NS'] and '[' not in os.environ['NS'], 'Replace [NAMESPACE] first' out = pathlib.Path(os.environ['CI_RENDER_DIR']) -files = ['config/rbac.yaml', - 'pipeline/tasks/phases/prepare.yaml', - 'pipeline/tasks/phases/evaluate.yaml', - 'pipeline/tasks/post/analyze-and-check-degradation.yaml', - 'pipeline/tasks/post/store.yaml', 'pipeline/tasks/post/cleanup_pvc.yaml', - 'pipeline/pipelines/ci-pipeline-openshell.yaml'] -items = [] -for path in files: - text = pathlib.Path(path).read_text() - for old in ('ab-eval-flow', 'gz-forge-eval'): - text = text.replace(old, ns) - for obj in yaml.safe_load_all(text): - if obj: - assert obj['kind'] in ('Task', 'Pipeline', 'ServiceAccount', 'Role', 'RoleBinding') - obj.setdefault('metadata', {})['namespace'] = ns - items.append(obj) -(out / 'resources.json').write_text(json.dumps({'apiVersion': 'v1', 'kind': 'List', 'items': items})) run = yaml.safe_load(pathlib.Path('pipeline/runs/openshell-openclaw-pipelinerun.yaml').read_text()) -run['metadata']['namespace'] = ns -overrides = { - 'revision': os.environ['FLOW_REV'], 'pipeline-repo-revision': os.environ['FLOW_REV'], - 'agent-eval-harness-repo-revision': os.environ['HARNESS_REV'], - 'openshell-gateway-endpoint': 'https://host.containers.internal:17670', - 'openshell-mtls-secret': 'openshell-gateway-mtls', - 'openshell-ai-gateway-ca-secret': 'forge-agent-upstream-tls', - 'llm-api-base': f'http://litellm.{ns}.svc.cluster.local:4000', - 'llm-base-url': f'http://litellm.{ns}.svc.cluster.local:4000/v1', - 'mlflow-tracking-uri': f'http://abevalflow-mlflow.{ns}.svc.cluster.local:5000', -} params = {p['name']: p['value'] for p in run['spec']['params']} -params.update(overrides) +params.update({ + 'revision': os.environ['FLOW_REV'], + 'pipeline-repo-revision': os.environ['FLOW_REV'], +}) +if os.environ.get('HARNESS_REV'): + params['agent-eval-harness-repo-revision'] = os.environ['HARNESS_REV'] run['spec']['params'] = [{'name': k, 'value': v} for k, v in params.items()] -run['spec']['taskRunTemplate']['podTemplate']['hostAliases'] = [ - {'ip': os.environ['GATEWAY_IP'], 'hostnames': ['host.containers.internal']}] (out / 'run.json').write_text(json.dumps(run, indent=2)) print(out) PY -oc -n "$NS" apply --dry-run=server -f "$CI_RENDER_DIR/resources.json" -# Review resources.json and run.json, then install the namespaced resources. -oc -n "$NS" apply -f "$CI_RENDER_DIR/resources.json" -oc -n "$NS" auth can-i list persistentvolumeclaims --as="system:serviceaccount:$NS:pipeline" -oc -n "$NS" auth can-i delete persistentvolumeclaims --as="system:serviceaccount:$NS:pipeline" oc -n "$NS" create --dry-run=server -f "$CI_RENDER_DIR/run.json" ``` -The renderer does not provision Forge, Secrets, storage services, network policies -or database migrations; finish steps 2–4 first. If model names, providers or -Secret names differ, adjust the rendered params. `llm-api-key=mock` is valid only -when your LiteLLM explicitly accepts that value; configure real credentials -through Secrets otherwise. Keep the template's GHCR image digest and GLM model +The installer does not provision Forge, Secrets, storage services, network +policies or database migrations; finish steps 2–4 first. If model names, +providers or Secret names differ, adjust the run params. `llm-api-key=mock` +is valid only when your LiteLLM explicitly accepts that value; pass real +credentials through the param from your own Secret handling instead of +committing them. Keep the template's GHCR image digest and GLM model overrides only if they match the deployment you validated. ### 6. Trigger, inspect and confirm completion @@ -324,17 +342,23 @@ PipelineRun. Open MinIO at ; use that namespace's MinIO credentials. Connection refused usually means the local forward stopped. MinIO 9001 is the web console; application uploads use 9000. -Cleanup requires API connectivity even after evaluation succeeds. The corrected -Task retries and may defer cleanup with a warning; verify PVC deletion rather -than assuming green means it was deleted. If store fails, preserve its workspace -before cleanup and run a store-only TaskRun with that existing PVC and the -original run ID/results paths. A normal PipelineRun rerun executes evaluation -again; it does not resume at store. - -Before any rerun, verify both Git revisions, installed Task versions, namespace -URLs and the current gateway Service IP. Service recreation can invalidate a -saved host alias. Helm redeployment can revert manual firewall or TLS changes; -retain the validated settings in the namespace's deployment source. +Cleanup requires API connectivity even after evaluation succeeds. The finally +Task is non-blocking (`onError: continue`) and reports its outcome as the +`cleanup-status` result on the PipelineRun: `deleted`, `none`, `api-unavailable` +or `failed`. Check that result rather than assuming green means the PVC was +deleted; a `failed` cleanup also labels the leaked PVC +`abevalflow.io/cleanup-failed=true`, so `oc -n "$NS" get pvc -l +abevalflow.io/cleanup-failed=true` finds leftovers across runs. If store fails, +preserve its workspace before cleanup and run a store-only TaskRun with that +existing PVC and the original run ID/results paths. A normal PipelineRun rerun +executes evaluation again; it does not resume at store. + +Before any rerun, verify both Git revisions, installed Task versions and +namespace URLs. The openshell gateway address is the `openshell` Service, whose +selector tracks the agent VMI, so it keeps pointing at the VM across VM +restarts; no host alias or IP pinning is maintained. Helm redeployment can +revert manual firewall or TLS changes; retain the validated settings in the +namespace's deployment source. --- From 6c34a77cde2dae923515ef947b722c5f6917f1e3 Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Mon, 12 Oct 2026 14:18:42 +0100 Subject: [PATCH 13/15] fix(evaluate): scope mTLS client-certificate staging to the mtls branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The staging of the openshell-mtls client certificate and its required-file check ran before the auth-mode dispatch, so openshell-auth-mode=oidc aborted in an OIDC-only deployment (which has no openshell-mtls Secret) before the oidc branch could validate its own credentials. Move the staging block and its required-file check inside the mtls branch of the case; the oidc branch still validates only the OIDC issuer, client ID and client secret. The post-case diagnostics referenced MTLS_DST unconditionally — guard that reference, since the step runs under set -u and oidc mode now leaves the variable unset. test_mtls_staging_is_scoped_to_mtls_branch pins the scoping: the staging vars and the mTLS required-file error must not appear before the case or inside the oidc branch. Co-Authored-By: Claude Code --- pipeline/tasks/phases/evaluate.yaml | 42 ++++++++++++++---------- tests/test_openshell_pipeline_profile.py | 28 ++++++++++++++++ 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 4036e8e..a834ce2 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -1759,24 +1759,11 @@ spec: export PATH="/tmp/bin:$PATH" fi - # Materialize the CI client certificate before registering the gateway. - # gateway add inspects this directory when selecting mTLS transport. + # Gateway profile shared by both auth modes; mode-specific credentials + # are staged inside their case branches below. export XDG_CONFIG_HOME="${HOME}/.config" GW_NAME="${OPENSHELL_GATEWAY_NAME:-ci-gateway}" - MTLS_SRC="/tmp/.config/openshell" - MTLS_DST="${XDG_CONFIG_HOME}/openshell/gateways/${GW_NAME}/mtls" mkdir -p "${XDG_CONFIG_HOME}/openshell" - if [ -f "$MTLS_SRC/tls.crt" ] && [ -f "$MTLS_SRC/tls.key" ]; then - mkdir -p "$MTLS_DST" - cp -f "$MTLS_SRC/ca.crt" "$MTLS_DST/" 2>/dev/null || true - cp -f "$MTLS_SRC/tls.crt" "$MTLS_DST/" - cp -f "$MTLS_SRC/tls.key" "$MTLS_DST/" - chmod 600 "$MTLS_DST"/* 2>/dev/null || true - echo "OpenShell mTLS material staged before gateway registration" - else - echo "ERROR: openshell-mtls client certificate is missing" >&2 - exit 1 - fi # The authentication path is selected by the explicit openshell-auth-mode # parameter, never by the endpoint's hostname shape. Native mTLS requires @@ -1784,6 +1771,24 @@ spec: # this CLI drops client authentication. case "${OPENSHELL_AUTH_MODE:-mtls}" in mtls) + # Materialize the CI client certificate before registering the + # gateway: gateway add inspects this directory when selecting mTLS + # transport. Scoped to this branch on purpose — the client + # certificate is an mTLS-only requirement, and an OIDC-only + # deployment has no openshell-mtls Secret to stage here. + MTLS_SRC="/tmp/.config/openshell" + MTLS_DST="${XDG_CONFIG_HOME}/openshell/gateways/${GW_NAME}/mtls" + if [ -f "$MTLS_SRC/tls.crt" ] && [ -f "$MTLS_SRC/tls.key" ]; then + mkdir -p "$MTLS_DST" + cp -f "$MTLS_SRC/ca.crt" "$MTLS_DST/" 2>/dev/null || true + cp -f "$MTLS_SRC/tls.crt" "$MTLS_DST/" + cp -f "$MTLS_SRC/tls.key" "$MTLS_DST/" + chmod 600 "$MTLS_DST"/* 2>/dev/null || true + echo "OpenShell mTLS material staged before gateway registration" + else + echo "ERROR: openshell-mtls client certificate is missing" >&2 + exit 1 + fi unset OPENSHELL_GATEWAY_INSECURE CLI_GATEWAY_ENDPOINT="$OPENSHELL_GATEWAY_ENDPOINT" # Use the CLI's supported local/mTLS registration path. This lets @@ -1844,8 +1849,11 @@ spec: esac # Secret mounts flat at $HOME/.config/openshell. OpenShell CLI expects - # gateways//mtls/{ca.crt,tls.crt,tls.key}. - if [ -f "$MTLS_DST/tls.crt" ] && [ -f "$MTLS_DST/tls.key" ]; then + # gateways//mtls/{ca.crt,tls.crt,tls.key}. MTLS_DST is set + # only by the mtls branch (and this step runs under set -u), and the + # oidc branch already logs its token-cache state, so this diagnostic + # block is mTLS-only. + if [ -n "${MTLS_DST:-}" ] && [ -f "$MTLS_DST/tls.crt" ] && [ -f "$MTLS_DST/tls.key" ]; then echo "OpenShell mTLS profile files:" find "$HOME/.config/openshell/gateways/$GW_NAME" -maxdepth 3 -type f -printf '%p %s bytes\n' 2>/dev/null || true openssl x509 -in "$MTLS_DST/tls.crt" -noout -subject -issuer -fingerprint -sha256 2>/dev/null || true diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index 3420cc4..f382116 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -173,6 +173,29 @@ def test_explicit_auth_mode_selects_gateway_path(self): assert '*".svc.cluster.local:"*' not in script assert '"https://host.containers.internal:"*' not in script + def test_mtls_staging_is_scoped_to_mtls_branch(self): + # The mTLS client certificate is an mTLS-only requirement: staging it + # and its required-file check must live inside the mtls branch, so an + # OIDC-only deployment (no openshell-mtls Secret) reaches the oidc + # branch without failing on it. + task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + step = next(s for s in task["spec"]["steps"] if s["name"] == "aeh-openshell-eval") + script = step["script"] + case_head = 'case "${OPENSHELL_AUTH_MODE:-mtls}" in' + before_case = script.split(case_head, 1)[0] + mtls_branch = script.split(case_head, 1)[1].split("oidc)", 1)[0] + oidc_branch = script.split("oidc)", 1)[1].split("*)", 1)[0] + for token in ("MTLS_SRC", "MTLS_DST", "openshell-mtls client certificate is missing"): + assert token not in before_case, token + assert token in mtls_branch, token + assert token not in oidc_branch, token + # The oidc branch depends only on the OIDC Secret's own fields. + for token in ("OPENSHELL_OIDC_ISSUER", "OPENSHELL_OIDC_CLIENT_ID", "OPENSHELL_OIDC_CLIENT_SECRET"): + assert token in oidc_branch + # Post-case diagnostics run under set -u with MTLS_DST unset in oidc + # mode, so they must guard the variable instead of referencing it. + assert '[ -n "${MTLS_DST:-}" ]' in script + def test_probe_targets_pipeline_gateway_contract(self): probe = _load(REPO / "pipeline" / "tasks" / "eval-stack-probe.yaml") pipeline_defaults = _param_defaults(_load(PIPELINE)["spec"]) @@ -189,6 +212,11 @@ def test_probe_targets_pipeline_gateway_contract(self): assert "openssl s_client" in script assert "subjectAltName" in script assert "curl -sS" in script + # A failed TLS handshake or certificate validation must fail the TLS + # layer: openssl s_client is piped into openssl x509, so pipefail must + # propagate the client's failure status — without it, x509 parsing + # the still-emitted certificate would pass the check. + assert "set -uo pipefail" in script # TLS chain validation is enabled only when the CA Secret exists. ca = next(e for e in step["env"] if e["name"] == "GATEWAY_CA_CERT") ref = ca["valueFrom"]["secretKeyRef"] From 22b1a90a7d1f8b5d7b56cf275054f4935c73beed Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Mon, 12 Oct 2026 14:18:51 +0100 Subject: [PATCH 14/15] fix(probe): propagate TLS client failures through the s_client|x509 pipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe's TLS check pipes openssl s_client into openssl x509. With -verify_return_error a failing handshake or certificate validation still emits the peer certificate, so x509 parsed it and exited 0 — the pipeline's status came from the last command and the check printed PASS despite the failed validation. Enable set -uo pipefail so the TLS client's failure status propagates through the pipe. No -e: every probe must run and the script exits with the aggregate fail count. The gateway_probe locals already default tls_extra, curl_ca and tls_note, so -u stays safe when the gateway CA Secret is not mounted. The probe test now asserts the script enables pipefail. Co-Authored-By: Claude Code --- pipeline/tasks/eval-stack-probe.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pipeline/tasks/eval-stack-probe.yaml b/pipeline/tasks/eval-stack-probe.yaml index acf19ae..fbba2d5 100644 --- a/pipeline/tasks/eval-stack-probe.yaml +++ b/pipeline/tasks/eval-stack-probe.yaml @@ -39,6 +39,12 @@ spec: optional: true script: | #!/usr/bin/env bash + # pipefail: the TLS check pipes openssl s_client into openssl x509 — + # without it, a failed handshake or certificate validation whose + # certificate is still emitted would pass on x509's exit status. No + # -e: every probe must run and the script exits with the aggregate + # $fail below. + set -uo pipefail NS="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null || echo current)" fail=0 probe() { # name url From 691bf1c0be82e62c7c489faecf60ae83924800ac Mon Sep 17 00:00:00 2001 From: Shri Charan Date: Mon, 12 Oct 2026 14:54:03 +0100 Subject: [PATCH 15/15] fix(probe): validate the gateway chain against its own CA and fail on HTTP 5xx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipefail fix from the previous commit let the TLS check's real result through, and live verification against the gateway showed it failing: the probe mounted the forge service CA (forge-ai-gateway-ca) while the openshell gateway's serving certificate is signed by openshell-ca, so chain validation could never succeed — the old script masked that as PASS, which is exactly the false-pass class the review flagged. Mount the CA that actually signed the gateway's certificate instead: openshell-mtls/ca.crt (openshell-ca), still optional so the probe runs in namespaces without the bundle. With it, the TLS layer validates the chain and the HTTP layer verifies it via curl --cacert. While here, close the same false-pass class in the HTTP layers: a 5xx response now fails the probe (reachability is not service health). 4xx stays a pass because bare endpoints legitimately answer 404. Co-Authored-By: Claude Code --- pipeline/tasks/eval-stack-probe.yaml | 25 +++++++++++++++++++----- tests/test_openshell_pipeline_profile.py | 11 +++++++++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/pipeline/tasks/eval-stack-probe.yaml b/pipeline/tasks/eval-stack-probe.yaml index fbba2d5..d2f4934 100644 --- a/pipeline/tasks/eval-stack-probe.yaml +++ b/pipeline/tasks/eval-stack-probe.yaml @@ -29,12 +29,14 @@ spec: env: # When present, TLS validation and the HTTP probe verify the gateway # certificate chain; without it, the TLS layer reports handshake + - # SANs only. Optional so the probe still runs in namespaces that do - # not have this Secret. + # SANs only. The chain must be checked against the CA that signed the + # gateway's serving certificate (openshell-ca, shipped in the + # openshell-mtls bundle) — the forge service CA cannot validate it. + # Optional so the probe still runs in namespaces without the bundle. - name: GATEWAY_CA_CERT valueFrom: secretKeyRef: - name: forge-ai-gateway-ca + name: openshell-mtls key: ca.crt optional: true script: | @@ -49,7 +51,16 @@ spec: fail=0 probe() { # name url if curl -sS --max-time 8 -o /dev/null -w "%{http_code}" $2 >/tmp/code 2>/tmp/err; then - echo " PASS $1 (HTTP $(cat /tmp/code))" + code="$(cat /tmp/code)" + # Reachability is not service health: a 5xx means the dependency + # answered but is not serving. 4xx can be a valid answer for a + # bare endpoint (e.g. 404 on a CDN root), so only server errors + # fail the probe. + if [ "${code:-0}" -ge 500 ] 2>/dev/null; then + echo " FAIL $1 -> HTTP $code (server error)"; fail=1 + else + echo " PASS $1 (HTTP $code)" + fi else echo " FAIL $1 -> $(tr -d '\n' /tmp/gw-err); then - echo " PASS $name http (HTTP $code)" + if [ "${code:-0}" -ge 500 ] 2>/dev/null; then + echo " FAIL $name http -> HTTP $code (server error)"; fail=1 + else + echo " PASS $name http (HTTP $code)" + fi else echo " FAIL $name http -> $(tr -d '\n'