diff --git a/config/forge-saw/README.md b/config/forge-saw/README.md index 6f1ea23..b2b566c 100644 --- a/config/forge-saw/README.md +++ b/config/forge-saw/README.md @@ -148,6 +148,24 @@ oc get secret openshell-credentials -n guy-ziv-evalflow -o json \ `aeh-openshell-eval` TCP-checks `:17670` and optionally runs `openshell sandbox list`. If SAW is down the **run fails**. That step does not install SAW. +## NetworkPolicy (Forge shared namespace) + +When SAW VMs and Tekton share a namespace with default-deny policies, evaluate +pods must be allowed to reach the agent gateway on `:17670`. Use the canonical +Pipeline name `abevalflow-pipeline-openshell` (so `tekton.dev/pipeline` matches) +and label PipelineRuns with `app.kubernetes.io/part-of: abevalflow`. + +Same-namespace template: + +```bash +sed "s/NAMESPACE/${EVAL_NS}/g" config/forge-saw/networkpolicy-ci-openshell.yaml \ + | oc apply -f - +``` + +Do not create ad-hoc copies of the Pipeline under a different name unless those +PipelineRuns also carry the `part-of=abevalflow` label and the NetworkPolicies +above are applied — otherwise gateway preflight times out. + ## Image Stock `agent-eval-harness:v1.0.x` cannot import `agent_eval.openshell`. Point diff --git a/config/forge-saw/networkpolicy-ci-openshell.yaml b/config/forge-saw/networkpolicy-ci-openshell.yaml new file mode 100644 index 0000000..5bd6f18 --- /dev/null +++ b/config/forge-saw/networkpolicy-ci-openshell.yaml @@ -0,0 +1,213 @@ +# NetworkPolicies for OpenShell OpenClaw evals when SAW VMs and the Tekton +# pipeline share one namespace (Forge workspace style). +# +# Select evaluate pods by either: +# - tekton.dev/pipeline=abevalflow-pipeline-openshell (canonical Pipeline name) +# - app.kubernetes.io/part-of=abevalflow (set on PipelineRun labels) +# +# Do NOT rename/copy the Pipeline to an ad-hoc name without also labeling the +# PipelineRun with app.kubernetes.io/part-of=abevalflow — otherwise default-deny +# blocks TCP to the gateway on :17670 and evaluate fails preflight. +# +# Apply after substituting NAMESPACE (and AGENT_VM / INTEG_VM if different): +# sed "s/NAMESPACE/forge-nommen/g" networkpolicy-ci-openshell.yaml | oc apply -f - +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-agent-gateway + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + - podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 17670 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-integ-gateway + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + - podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-egress-by-pipeline + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: litellm + ports: + - protocol: TCP + port: 4000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: mlflow + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 5000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 9000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: abevalflow-allow-ci-egress-by-part-of + namespace: NAMESPACE + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/component: forge-saw +spec: + podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell-saw-integ + vm.kubevirt.io/name: openshell-saw-integ + ports: + - protocol: TCP + port: 17670 + - protocol: TCP + port: 18082 + - protocol: TCP + port: 8443 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: litellm + ports: + - protocol: TCP + port: 4000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: mlflow + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 5000 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: abevalflow + ports: + - protocol: TCP + port: 9000 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 diff --git a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml index af1aaef..9071391 100644 --- a/config/forge-saw/networkpolicy-gateway-from-abeval.yaml +++ b/config/forge-saw/networkpolicy-gateway-from-abeval.yaml @@ -1,6 +1,11 @@ # Allow evaluate Task pods to reach the OpenShell gateway on :17670. # bootstrap.sh rewrites namespace / from-namespace (SAW_NS, EVAL_NS). # Apply in the SAW namespace, not from the evaluate PipelineRun. +# +# Prefer matching Tekton pods by the canonical Pipeline name or the +# app.kubernetes.io/part-of=abevalflow label on PipelineRuns. A bare +# podSelector: {} from the whole EVAL_NS is broader than needed when SAW +# and eval share a Forge workspace namespace — see networkpolicy-ci-openshell.yaml. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -20,7 +25,15 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: guy-ziv-evalflow - - podSelector: {} + podSelector: + matchLabels: + tekton.dev/pipeline: abevalflow-pipeline-openshell + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: guy-ziv-evalflow + podSelector: + matchLabels: + app.kubernetes.io/part-of: abevalflow ports: - protocol: TCP port: 17670 diff --git a/config/forge-saw/openshell-eval-alias.yaml b/config/forge-saw/openshell-eval-alias.yaml new file mode 100644 index 0000000..d8ac338 --- /dev/null +++ b/config/forge-saw/openshell-eval-alias.yaml @@ -0,0 +1,20 @@ +# Stable, namespace-local endpoint for evaluation jobs. Selecting the agent +# VMI directly avoids baking a deployment namespace into Pipeline defaults. +# The gateway server leaf must include "openshell" as a DNS SAN before the +# evaluation task can disable OPENSHELL_GATEWAY_INSECURE. +apiVersion: v1 +kind: Service +metadata: + name: openshell + labels: + app.kubernetes.io/name: openshell-eval-alias + app.kubernetes.io/part-of: abevalflow +spec: + selector: + app.kubernetes.io/name: openshell-saw-agent + vm.kubevirt.io/name: openshell-saw-agent + ports: + - name: gateway + port: 17670 + protocol: TCP + targetPort: 17670 diff --git a/config/litellm/configmap.yaml b/config/litellm/configmap.yaml index b352a41..c01eca0 100644 --- a/config/litellm/configmap.yaml +++ b/config/litellm/configmap.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: litellm-config - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm @@ -15,6 +14,25 @@ data: api_base: os.environ/INFERENCE_ENDPOINT_URL api_key: os.environ/INFERENCE_API_KEY + - model_name: rits/zai-org/GLM-5-3-Flash + litellm_params: + model: openai/rits/zai-org/GLM-5-3-Flash + api_base: os.environ/INFERENCE_ENDPOINT_URL + api_key: os.environ/INFERENCE_API_KEY + + # Judge route. The LLM judges call /v1/messages (Anthropic dialect) — + # score.py has no OpenAI dialect. For an "openai/" model LiteLLM converts + # that to the upstream's /responses API, which this upstream rejects with + # 403 "Authentication parameters missing", so every scoring judge errors + # and mean_reward is unavailable. Routing the same upstream as hosted_vllm + # keeps the call on /chat/completions. Select it with + # aeh-judge-model-override=judge-glm-5-3. + - model_name: judge-glm-5-3 + litellm_params: + model: hosted_vllm/rits/zai-org/GLM-5-3-Flash + api_base: os.environ/INFERENCE_ENDPOINT_URL + api_key: os.environ/INFERENCE_API_KEY + litellm_settings: drop_params: true set_verbose: false diff --git a/config/litellm/deployment.yaml b/config/litellm/deployment.yaml index 8146bab..46c015a 100644 --- a/config/litellm/deployment.yaml +++ b/config/litellm/deployment.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: Deployment metadata: name: litellm - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm diff --git a/config/litellm/service.yaml b/config/litellm/service.yaml index bc12954..885c92b 100644 --- a/config/litellm/service.yaml +++ b/config/litellm/service.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Service metadata: name: litellm - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm diff --git a/config/litellm/serviceaccount.yaml b/config/litellm/serviceaccount.yaml index 5ea3455..78ef60d 100644 --- a/config/litellm/serviceaccount.yaml +++ b/config/litellm/serviceaccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: litellm - namespace: gz-forge-eval labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: litellm diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index 07a41c5..65d7d85 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -2,9 +2,9 @@ apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: abevalflow-pipeline-openshell - namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow + app.kubernetes.io/part-of: abevalflow app.kubernetes.io/version: "2.0" app.kubernetes.io/profile: aeh-openshell-openclaw spec: @@ -38,7 +38,7 @@ spec: description: Must stay aeh_openshell_openclaw for this profile - name: llm-model type: string - default: "rits/zai-org/glm-5-3" + default: "rits/zai-org/GLM-5-3-Flash" - name: llm-api-base type: string default: "http://litellm.gz-forge-eval.svc:4000" @@ -50,7 +50,7 @@ spec: default: "http://litellm.ab-eval-flow.svc:4000/v1" - name: llm-generation-model type: string - default: "rits/zai-org/glm-5-3" + default: "rits/zai-org/GLM-5-3-Flash" - name: enable-ai-generation type: string default: "false" @@ -73,10 +73,13 @@ spec: default: "-1.0" - name: aeh-model-override type: string - default: "" + default: "inference/rits/zai-org/GLM-5-3-Flash" - name: aeh-judge-model-override type: string - default: "" + # Judges call /v1/messages (Anthropic dialect); see the judge-glm-5-3 + # entry in config/litellm/configmap.yaml for why a plain openai/ model + # 403s there and hosted_vllm does not. + default: "judge-glm-5-3" - name: aeh-mode type: string default: "single" @@ -102,17 +105,20 @@ spec: default: "https://github.com/GuyZivRH/agent-eval-harness.git" - name: agent-eval-harness-repo-revision type: string - default: "feat/aeh-openshell-openclaw" - description: Harness revision with agent_eval.openshell (feature-branch pin) + default: "main" + description: >- + GuyZivRH/agent-eval-harness main includes the OpenClaw brief-reader and + gateway fixes merged in #9. Pin a commit SHA for reproducible runs. - name: openshell-cli-version type: string default: "0.0.116" - name: openshell-gateway-endpoint type: string - default: "https://openshell.openshell.svc.cluster.local:17670" + # Installed by config/forge-saw/openshell-eval-alias.yaml in the run namespace. + default: "https://openshell:17670" - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider type: string default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index f46241e..fd5f8c2 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -1,10 +1,20 @@ # OpenShell OpenClaw profile — prepare → evaluate → analyze → store (no test cube). -# Apply the Pipeline first: oc apply -f pipeline/pipelines/ci-pipeline-openshell.yaml +# +# oc apply -n -f pipeline/pipelines/ci-pipeline-openshell.yaml +# oc create -n -f pipeline/runs/openshell-openclaw-pipelinerun.yaml +# +# Service endpoints below use short names, which resolve inside the namespace +# the run is created in, so this file is namespace-agnostic. # Do not put M365 tokens in this file; use Secret openshell-credentials in the namespace. apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: aeh-openshell-openclaw- + labels: + # Required when NetworkPolicies select on part-of (and recommended always). + # Keep pipelineRef.name=abevalflow-pipeline-openshell so tekton.dev/pipeline + # also matches the canonical CI NetworkPolicies. + app.kubernetes.io/part-of: abevalflow spec: pipelineRef: name: abevalflow-pipeline-openshell @@ -16,54 +26,52 @@ spec: value: "openclaw-forge" - name: eval-engine value: "aeh_openshell_openclaw" - # Until this branch is merged, pass the feature revision so store/parse scripts match. - # After merge to main, omit these (Pipeline defaults are main). + # This example targets the merged upstream main for both submission and + # Pipeline helper source. Replace either value with a commit SHA to pin a run. - name: revision - value: "feat/aeh-openshell-openclaw" + value: "main" - name: pipeline-repo-revision - value: "feat/aeh-openshell-openclaw" + value: "main" # [Optional] Add a harness repository or revision for this run; these replace Pipeline defaults. # - name: agent-eval-harness-repo-url # value: "" # - name: agent-eval-harness-repo-revision # value: "" - name: openshell-gateway-endpoint - # Certificate-valid hostname, resolved to this namespace's Service below. - value: "https://host.containers.internal:17670" + # Stable, certificate-valid alias installed by + # config/forge-saw/openshell-eval-alias.yaml in the run namespace. + value: "https://openshell:17670" - name: openshell-mtls-secret - value: "openshell-gateway-mtls" + value: "openshell-mtls" - name: openshell-ai-gateway-ca-secret - value: "forge-agent-upstream-tls" + value: "forge-ai-gateway-ca" # To use another sandbox image, replace the value below with . - name: openshell-sandbox-image - value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider value: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" - name: aeh-openshell-image value: "registry.access.redhat.com/ubi9/python-311:9.6" - name: llm-model - value: "rits/zai-org/glm-5-3" + value: "rits/zai-org/GLM-5-3-Flash" - name: llm-api-base - value: "http://litellm.gz-forge-eval.svc.cluster.local:4000" + value: "http://litellm:4000" - name: llm-api-key value: "mock" - name: aeh-model-override - value: "inference/rits/zai-org/glm-5-3" + value: "inference/rits/zai-org/GLM-5-3-Flash" - name: aeh-judge-model-override - value: "rits/zai-org/glm-5-3" + value: "judge-glm-5-3" - name: enable-mlflow value: "true" - name: mlflow-tracking-uri - value: "http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000" + value: "http://abevalflow-mlflow:5000" taskRunTemplate: serviceAccountName: pipeline - podTemplate: - hostAliases: - - ip: "172.30.183.80" - hostnames: ["host.containers.internal"] timeouts: - pipeline: 2h0m0s - tasks: 1h30m0s + pipeline: 3h0m0s + tasks: 2h30m0s + finally: 15m0s workspaces: - name: shared-workspace volumeClaimTemplate: diff --git a/pipeline/tasks/eval-stack-probe.yaml b/pipeline/tasks/eval-stack-probe.yaml new file mode 100644 index 0000000..cee3d4c --- /dev/null +++ b/pipeline/tasks/eval-stack-probe.yaml @@ -0,0 +1,48 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: eval-stack-probe + # Deliberately not labelled app.kubernetes.io/part-of: abevalflow — the probe + # must be seen by NetworkPolicy as an ordinary Tekton pod, otherwise it passes + # through the stack's own allow-rules and reports connectivity the pipeline + # does not actually have. +spec: + description: >- + Probes every dependency an ABEvalFlow pipeline pod needs. Run it before and + after NetworkPolicy changes to see exactly which paths are blocked. + steps: + - name: probe + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + NS="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null || echo current)" + fail=0 + probe() { # name url + if curl -sS --max-time 8 -o /dev/null -w "%{http_code}" $2 >/tmp/code 2>/tmp/err; then + echo " PASS $1 (HTTP $(cat /tmp/code))" + else + echo " FAIL $1 -> $(tr -d '\n' /tmp/tcp-err; then + echo " PASS $1 (TCP $2:$3)" + else + echo " FAIL $1 -> $(tr -d '\n' - Branch or tag of GuyZivRH/agent-eval-harness to clone. Must include the OpenShell create keep-alive fix (`--detach` + `sleep infinity`). @@ -196,7 +195,7 @@ spec: Override only if using a different gateway (e.g. forge-saw :17670). - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" description: >- Immutable SAW Chief-of-Staff image with the daily-briefing skill and governed M365/Slack tools. @@ -1817,6 +1816,46 @@ spec: PY chmod 600 "$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" echo "OpenShell OIDC token cache populated via client credentials" + # A case can run for 15 minutes, while the client-credentials token + # expires after roughly five. Refresh the CLI cache throughout this + # step so later artifact downloads and sandbox creation stay authorized. + OIDC_TOKEN_CACHE="$HOME/.config/openshell/gateways/$GW_NAME/oidc_token.json" + ( + umask 077 + trap 'rm -f "${OIDC_TOKEN_JSON}.refresh"' EXIT + while sleep 120; do + if curl -ksSf --max-time 20 \ + -u "${OPENSHELL_OIDC_CLIENT_ID}:${OPENSHELL_OIDC_CLIENT_SECRET}" \ + -d grant_type=client_credentials \ + "${OPENSHELL_OIDC_ISSUER%/}/protocol/openid-connect/token" \ + >"${OIDC_TOKEN_JSON}.refresh" 2>/dev/null; then + python3 - "${OIDC_TOKEN_JSON}.refresh" "$OIDC_TOKEN_CACHE" \ + "$OPENSHELL_OIDC_ISSUER" "$OPENSHELL_OIDC_CLIENT_ID" <<'REFRESH' || true + import json, os, sys, tempfile, time + src, dst, issuer, client_id = sys.argv[1:] + with open(src, encoding="utf-8") as stream: + payload = json.load(stream) + cache = {"access_token": payload["access_token"], + "expires_at": int(time.time()) + int(payload.get("expires_in", 300)), + "issuer": issuer, "client_id": client_id} + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(dst), prefix=".oidc-token-") + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(cache, stream) + os.chmod(tmp, 0o600) + os.replace(tmp, dst) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + REFRESH + else + echo "WARN: OpenShell OIDC token refresh failed; retrying" >&2 + fi + done + ) & + OIDC_REFRESH_PID=$! + trap 'kill "$OIDC_REFRESH_PID" 2>/dev/null || true' EXIT + echo "OpenShell OIDC token refresher started" export OPENSHELL_GATEWAY="$GW_NAME" echo "OpenShell OIDC gateway configured: ${GW_NAME}" else @@ -2061,6 +2100,13 @@ spec: echo "Model: $MODEL" export AGENT_EVAL_HARNESS_ROOT="$HARNESS_DIR" + # A published Forge briefing needs installation identity. Keep this + # synthetic fixture scoped to submissions that supply one; an explicit + # harness setting takes precedence for other deployments. + if [ -z "${AGENT_EVAL_FORGE_USER_FILE:-}" ] && [ -f "$SUBMISSION_DIR/fixtures/USER.md" ]; then + export AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md" + echo "Using submission installation profile fixture" + fi if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" echo "MLFLOW_TRACKING_URI set for AEH harness + CI logger" diff --git a/pipeline/tasks/phases/prepare.yaml b/pipeline/tasks/phases/prepare.yaml index 33f0f21..7e85c46 100644 --- a/pipeline/tasks/phases/prepare.yaml +++ b/pipeline/tasks/phases/prepare.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: prepare - namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow app.kubernetes.io/component: phase @@ -122,16 +121,21 @@ spec: echo "=== PREPARE PHASE: Clone Pipeline Repo ===" PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + PIPELINE_REV="$(params.pipeline-repo-revision)" if [ -d "$PIPELINE_DIR/.git" ]; then echo "Pipeline repo already cloned, updating..." cd "$PIPELINE_DIR" - git fetch origin "$(params.pipeline-repo-revision)" - git checkout "$(params.pipeline-repo-revision)" 2>/dev/null || git checkout FETCH_HEAD + git fetch origin "$PIPELINE_REV" --depth 1 2>/dev/null \ + || git fetch origin "$PIPELINE_REV" + git checkout "$PIPELINE_REV" 2>/dev/null || git checkout FETCH_HEAD else - git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \ - "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + if ! git clone --depth 1 --branch "$PIPELINE_REV" \ + "$(params.pipeline-repo-url)" "$PIPELINE_DIR" 2>/dev/null; then + git clone "$(params.pipeline-repo-url)" "$PIPELINE_DIR" + git -C "$PIPELINE_DIR" checkout "$PIPELINE_REV" + fi fi - echo "Pipeline repo ready at $(params.pipeline-repo-revision)" + echo "Pipeline repo ready at $(git -C "$PIPELINE_DIR" rev-parse --short HEAD)" # Step 3: Generate tests (conditional) - name: generate-tests diff --git a/pipeline/tasks/post/analyze-and-check-degradation.yaml b/pipeline/tasks/post/analyze-and-check-degradation.yaml index 3140f6e..10ab0ae 100644 --- a/pipeline/tasks/post/analyze-and-check-degradation.yaml +++ b/pipeline/tasks/post/analyze-and-check-degradation.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: analyze-and-check-degradation - namespace: ab-eval-flow labels: app.kubernetes.io/name: abevalflow spec: diff --git a/pipeline/tasks/post/cleanup_pvc.yaml b/pipeline/tasks/post/cleanup_pvc.yaml index 9ba6c65..c050958 100644 --- a/pipeline/tasks/post/cleanup_pvc.yaml +++ b/pipeline/tasks/post/cleanup_pvc.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: cleanup-pvc - namespace: ab-eval-flow spec: description: >- Deletes the workspace PVC after a pipeline run completes. @@ -11,36 +10,38 @@ spec: - name: pipelinerun-name type: string description: Name of the PipelineRun whose PVCs should be cleaned up - steps: - - name: delete-pvc - image: registry.redhat.io/openshift4/ose-cli:latest - script: | - #!/usr/bin/env bash - set -u + steps: + - name: delete-pvc + image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + onError: continue + script: | + #!/usr/bin/env bash + set -uo pipefail # Get current namespace from service account NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) echo "Cleaning up PVCs for PipelineRun $(params.pipelinerun-name) in namespace ${NAMESPACE}..." - PVC_LIST="" - for attempt in 1 2 3 4 5; do - if PVC_LIST=$(oc get pvc -n "${NAMESPACE}" \ - -o jsonpath="{range .items[?(@.metadata.ownerReferences[0].name=='$(params.pipelinerun-name)')]}{.metadata.name}{'\\n'}{end}" 2>/tmp/cleanup-pvc.err); then + pvcs="" + for attempt in 1 2 3; do + if pvcs=$(oc get pvc -n "${NAMESPACE}" --request-timeout=20s \ + -o jsonpath='{range .items[?(@.metadata.ownerReferences[0].name=="$(params.pipelinerun-name)")]}{.metadata.name}{"\n"}{end}'); then break fi - echo "WARNING: Kubernetes API unavailable (attempt ${attempt}/5); retrying..." >&2 - sleep 10 + echo "WARNING: PVC lookup attempt ${attempt}/3 failed" + sleep $((attempt * 5)) done - if [ -z "${PVC_LIST}" ] && [ -s /tmp/cleanup-pvc.err ]; then - cat /tmp/cleanup-pvc.err >&2 - echo "WARNING: PVC cleanup deferred because the Kubernetes API was unreachable" >&2 + + if [ -z "$pvcs" ]; then + echo "No owned PVCs found, or the API remained unavailable; cleanup is best-effort" exit 0 fi + while read -r pvc; do - if [ -n "${pvc}" ]; then - echo "Deleting PVC: ${pvc}" - oc delete pvc "${pvc}" -n "${NAMESPACE}" --wait=false || \ - echo "WARNING: failed to delete ${pvc}; it may be retried later" >&2 + if [ -n "$pvc" ]; then + echo "Deleting PVC: $pvc" + oc delete pvc "$pvc" -n "${NAMESPACE}" --wait=false --request-timeout=20s \ + || echo "WARNING: failed to delete PVC $pvc" fi - done <<< "${PVC_LIST}" + done <<< "$pvcs" echo "PVC cleanup complete" diff --git a/pipeline/tasks/post/store.yaml b/pipeline/tasks/post/store.yaml index 32d27aa..bf0dcc3 100644 --- a/pipeline/tasks/post/store.yaml +++ b/pipeline/tasks/post/store.yaml @@ -2,7 +2,6 @@ apiVersion: tekton.dev/v1 kind: Task metadata: name: store - namespace: ab-eval-flow spec: description: >- Persists A/B evaluation results and publishes artifacts. Step 1 writes diff --git a/submissions/openclaw-forge/fixtures/USER.md b/submissions/openclaw-forge/fixtures/USER.md new file mode 100644 index 0000000..e88fe80 --- /dev/null +++ b/submissions/openclaw-forge/fixtures/USER.md @@ -0,0 +1,7 @@ +# Synthetic evaluation identity + +- Display name: Alex Rivera +- Role: Engineering executive +- Initials: AR +- Primary email: tbx-demo2@dev.mscloud.ibm.com +- Time zone: America/New_York diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index b44f222..e3640ff 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -42,14 +42,60 @@ def test_openshell_defaults(self): assert defaults["aeh-runner"] == "openshell" assert defaults["aeh-openshell-image"] == "registry.access.redhat.com/ubi9/python-311:9.6" assert defaults["openshell-sandbox-image"] == ( - "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" ) + assert defaults["llm-model"] == "rits/zai-org/GLM-5-3-Flash" + assert defaults["llm-generation-model"] == "rits/zai-org/GLM-5-3-Flash" + assert defaults["aeh-model-override"] == "inference/rits/zai-org/GLM-5-3-Flash" + # A plain openai/ judge model 403s on /v1/messages; the judge-glm-5-3 + # hosted_vllm route in config/litellm/configmap.yaml stays on + # /chat/completions. See that file for the full rationale. + assert defaults["aeh-judge-model-override"] == "judge-glm-5-3" + # Alias Service from config/forge-saw/openshell-eval-alias.yaml; keeps + # the default namespace-agnostic. + assert defaults["openshell-gateway-endpoint"] == "https://openshell:17670" assert defaults["enable-mlflow"] == "true" - assert defaults["mlflow-tracking-uri"] == ("http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000") analyze = next(t for t in _load(PIPELINE)["spec"]["tasks"] if t["name"] == "analyze") scan = next(p for p in analyze["params"] if p["name"] == "security-scan-mode") assert scan["value"] == "disabled" + def test_litellm_configmap_has_flash_and_judge_routes(self): + config = _load(REPO / "config" / "litellm" / "configmap.yaml") + models = yaml.safe_load(config["data"]["config.yaml"])["model_list"] + names = {entry["model_name"] for entry in models} + assert "rits/zai-org/GLM-5-3-Flash" in names + assert "judge-glm-5-3" in names + judge = next(e for e in models if e["model_name"] == "judge-glm-5-3") + assert judge["litellm_params"]["model"].startswith("hosted_vllm/") + + def test_gateway_alias_service_selects_agent_vmi(self): + alias = _load(REPO / "config" / "forge-saw" / "openshell-eval-alias.yaml") + assert alias["spec"]["selector"] == { + "app.kubernetes.io/name": "openshell-saw-agent", + "vm.kubevirt.io/name": "openshell-saw-agent", + } + ports = alias["spec"]["ports"] + assert ports == [{"name": "gateway", "port": 17670, "protocol": "TCP", "targetPort": 17670}] + + def test_example_run_is_namespace_agnostic(self): + run = _load(REPO / "pipeline" / "runs" / "openshell-openclaw-pipelinerun.yaml") + params = {p["name"]: p["value"] for p in run["spec"]["params"]} + for name in ("openshell-gateway-endpoint", "llm-api-base", "mlflow-tracking-uri"): + value = params[name] + assert ".svc.cluster.local" not in value, f"{name} pins a namespace: {value}" + pod_template = run["spec"]["taskRunTemplate"].get("podTemplate", {}) + assert "hostAliases" not in pod_template + timeouts = run["spec"]["timeouts"] + assert timeouts["pipeline"] == "3h0m0s" + assert timeouts["tasks"] == "2h30m0s" + + def test_cleanup_task_is_best_effort(self): + task = _load(REPO / "pipeline" / "tasks" / "post" / "cleanup_pvc.yaml") + step = task["spec"]["steps"][0] + assert step.get("onError") == "continue" + assert "--request-timeout=20s" in step["script"] + assert step["image"].startswith("image-registry.openshift-image-registry.svc:5000/") + def test_evaluate_openshell_step_logs_mlflow(self): evaluate = (REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml").read_text() openshell = evaluate.split("name: aeh-openshell-eval", 1)[1].split("name: aeh-pairwise", 1)[0] @@ -80,6 +126,23 @@ def test_openshell_eval_uses_llm_param_instead_of_inference_secret(self): assert env["value"] == "$(params.llm-api-key)" assert "valueFrom" not in env + def test_forge_briefing_has_installation_user_fixture(self): + fixture = REPO / "submissions" / "openclaw-forge" / "fixtures" / "USER.md" + fields = {} + for line in fixture.read_text().splitlines(): + if line.startswith("- ") and ": " in line: + key, value = line[2:].split(": ", 1) + fields[key.lower()] = value.strip() + assert all(fields.get(key) and not fields[key].startswith("<") for key in ("display name", "role", "initials")) + scene = _load(REPO / "submissions" / "openclaw-forge" / "scenes" / "monday-acquisition.yaml") + assert fields["primary email"] == scene["m365"]["user"] + + task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + step = next(s for s in task["spec"]["steps"] if s["name"] == "aeh-openshell-eval") + script = step["script"] + assert 'AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md"' in script + assert script.index('[ -f "$SUBMISSION_DIR/fixtures/USER.md" ]') < script.index("scripts/run_aeh.py") + def test_harbor_profiles_still_include_test(self): for path in (CI, CI_DEV): names = _task_names(_load(path)["spec"])