Harden OpenShell eval pipeline: judge-safe model routing, gateway alias, best-effort cleanup

config/forge-saw/README.md

@@ -148,6 +148,24 @@ oc get secret openshell-credentials -n guy-ziv-evalflow -o json \
 `aeh-openshell-eval` TCP-checks `:17670` and optionally runs `openshell sandbox list`.
 If SAW is down the **run fails**. That step does not install SAW.
 
+## NetworkPolicy (Forge shared namespace)
+
+When SAW VMs and Tekton share a namespace with default-deny policies, evaluate
+pods must be allowed to reach the agent gateway on `:17670`. Use the canonical
+Pipeline name `abevalflow-pipeline-openshell` (so `tekton.dev/pipeline` matches)
+and label PipelineRuns with `app.kubernetes.io/part-of: abevalflow`.
+
+Same-namespace template:
+
+```bash
+sed "s/NAMESPACE/${EVAL_NS}/g" config/forge-saw/networkpolicy-ci-openshell.yaml \
+  | oc apply -f -
+```
+
+Do not create ad-hoc copies of the Pipeline under a different name unless those
+PipelineRuns also carry the `part-of=abevalflow` label and the NetworkPolicies
+above are applied — otherwise gateway preflight times out.
+
 ## Image
 
 Stock `agent-eval-harness:v1.0.x` cannot import `agent_eval.openshell`. Point

config/forge-saw/networkpolicy-ci-openshell.yaml

@@ -0,0 +1,213 @@
+# NetworkPolicies for OpenShell OpenClaw evals when SAW VMs and the Tekton
+# pipeline share one namespace (Forge workspace style).
+#
+# Select evaluate pods by either:
+#   - tekton.dev/pipeline=abevalflow-pipeline-openshell  (canonical Pipeline name)
+#   - app.kubernetes.io/part-of=abevalflow               (set on PipelineRun labels)
+#
+# Do NOT rename/copy the Pipeline to an ad-hoc name without also labeling the
+# PipelineRun with app.kubernetes.io/part-of=abevalflow — otherwise default-deny
+# blocks TCP to the gateway on :17670 and evaluate fails preflight.
+#
+# Apply after substituting NAMESPACE (and AGENT_VM / INTEG_VM if different):
+#   sed "s/NAMESPACE/forge-nommen/g" networkpolicy-ci-openshell.yaml | oc apply -f -
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-agent-gateway
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: openshell-saw-agent
+      vm.kubevirt.io/name: openshell-saw-agent
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 17670
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-integ-gateway
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: openshell-saw-integ
+      vm.kubevirt.io/name: openshell-saw-integ
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-egress-by-pipeline
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      tekton.dev/pipeline: abevalflow-pipeline-openshell
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-agent
+              vm.kubevirt.io/name: openshell-saw-agent
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-integ
+              vm.kubevirt.io/name: openshell-saw-integ
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: litellm
+      ports:
+        - protocol: TCP
+          port: 4000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: mlflow
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 5000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: minio
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: openshift-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+        - protocol: UDP
+          port: 5353
+        - protocol: TCP
+          port: 5353
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: abevalflow-allow-ci-egress-by-part-of
+  namespace: NAMESPACE
+  labels:
+    app.kubernetes.io/part-of: abevalflow
+    app.kubernetes.io/component: forge-saw
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/part-of: abevalflow
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-agent
+              vm.kubevirt.io/name: openshell-saw-agent
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: openshell-saw-integ
+              vm.kubevirt.io/name: openshell-saw-integ
+      ports:
+        - protocol: TCP
+          port: 17670
+        - protocol: TCP
+          port: 18082
+        - protocol: TCP
+          port: 8443
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: litellm
+      ports:
+        - protocol: TCP
+          port: 4000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: mlflow
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 5000
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: minio
+              app.kubernetes.io/part-of: abevalflow
+      ports:
+        - protocol: TCP
+          port: 9000
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: openshift-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+        - protocol: UDP
+          port: 5353
+        - protocol: TCP
+          port: 5353

config/forge-saw/networkpolicy-gateway-from-abeval.yaml

@@ -1,6 +1,11 @@
 # Allow evaluate Task pods to reach the OpenShell gateway on :17670.
 # bootstrap.sh rewrites namespace / from-namespace (SAW_NS, EVAL_NS).
 # Apply in the SAW namespace, not from the evaluate PipelineRun.
+#
+# Prefer matching Tekton pods by the canonical Pipeline name or the
+# app.kubernetes.io/part-of=abevalflow label on PipelineRuns. A bare
+# podSelector: {} from the whole EVAL_NS is broader than needed when SAW
+# and eval share a Forge workspace namespace — see networkpolicy-ci-openshell.yaml.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -20,7 +25,15 @@ spec:
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: guy-ziv-evalflow
-        - podSelector: {}
+          podSelector:
+            matchLabels:
+              tekton.dev/pipeline: abevalflow-pipeline-openshell
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: guy-ziv-evalflow
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: abevalflow
       ports:
         - protocol: TCP
           port: 17670

config/forge-saw/openshell-eval-alias.yaml

@@ -0,0 +1,20 @@
+# Stable, namespace-local endpoint for evaluation jobs.  Selecting the agent
+# VMI directly avoids baking a deployment namespace into Pipeline defaults.
+# The gateway server leaf must include "openshell" as a DNS SAN before the
+# evaluation task can disable OPENSHELL_GATEWAY_INSECURE.
+apiVersion: v1
+kind: Service
+metadata:
+  name: openshell
+  labels:
+    app.kubernetes.io/name: openshell-eval-alias
+    app.kubernetes.io/part-of: abevalflow
+spec:
+  selector:
+    app.kubernetes.io/name: openshell-saw-agent
+    vm.kubevirt.io/name: openshell-saw-agent
+  ports:
+    - name: gateway
+      port: 17670
+      protocol: TCP
+      targetPort: 17670

config/litellm/configmap.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: litellm-config
-  namespace: gz-forge-eval
   labels:
     app.kubernetes.io/part-of: abevalflow
     app.kubernetes.io/name: litellm
@@ -15,6 +14,25 @@ data:
           api_base: os.environ/INFERENCE_ENDPOINT_URL
           api_key: os.environ/INFERENCE_API_KEY
 
+      - model_name: rits/zai-org/GLM-5-3-Flash
+        litellm_params:
+          model: openai/rits/zai-org/GLM-5-3-Flash
+          api_base: os.environ/INFERENCE_ENDPOINT_URL
+          api_key: os.environ/INFERENCE_API_KEY
+
+      # Judge route. The LLM judges call /v1/messages (Anthropic dialect) —
+      # score.py has no OpenAI dialect. For an "openai/" model LiteLLM converts
+      # that to the upstream's /responses API, which this upstream rejects with
+      # 403 "Authentication parameters missing", so every scoring judge errors
+      # and mean_reward is unavailable. Routing the same upstream as hosted_vllm
+      # keeps the call on /chat/completions. Select it with
+      # aeh-judge-model-override=judge-glm-5-3.
+      - model_name: judge-glm-5-3
+        litellm_params:
+          model: hosted_vllm/rits/zai-org/GLM-5-3-Flash
+          api_base: os.environ/INFERENCE_ENDPOINT_URL
+          api_key: os.environ/INFERENCE_API_KEY
+
     litellm_settings:
       drop_params: true
       set_verbose: false

config/litellm/deployment.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: litellm
-  namespace: gz-forge-eval
   labels:
     app.kubernetes.io/part-of: abevalflow
     app.kubernetes.io/name: litellm

config/litellm/service.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: Service
 metadata:
   name: litellm
-  namespace: gz-forge-eval
   labels:
     app.kubernetes.io/part-of: abevalflow
     app.kubernetes.io/name: litellm

config/litellm/serviceaccount.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: litellm
-  namespace: gz-forge-eval
   labels:
     app.kubernetes.io/part-of: abevalflow
     app.kubernetes.io/name: litellm