From 6fd46bd6c2262422e867890a30cfc117b74f224e Mon Sep 17 00:00:00 2001 From: Tarun Kumar Reddy Etikala Date: Thu, 1 Oct 2026 13:35:11 -0400 Subject: [PATCH 1/2] Seed synthetic Forge identity for briefing evals --- pipeline/tasks/phases/evaluate.yaml | 7 +++++++ submissions/openclaw-forge/fixtures/USER.md | 7 +++++++ tests/test_openshell_pipeline_profile.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 submissions/openclaw-forge/fixtures/USER.md diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index a50eff9..1f12869 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -2070,6 +2070,13 @@ spec: echo "Model: $MODEL" export AGENT_EVAL_HARNESS_ROOT="$HARNESS_DIR" + # A published Forge briefing needs installation identity. Keep this + # synthetic fixture scoped to submissions that supply one; an explicit + # harness setting takes precedence for other deployments. + if [ -z "${AGENT_EVAL_FORGE_USER_FILE:-}" ] && [ -f "$SUBMISSION_DIR/fixtures/USER.md" ]; then + export AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md" + echo "Using submission installation profile fixture" + fi if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" echo "MLFLOW_TRACKING_URI set for AEH harness + CI logger" diff --git a/submissions/openclaw-forge/fixtures/USER.md b/submissions/openclaw-forge/fixtures/USER.md new file mode 100644 index 0000000..e88fe80 --- /dev/null +++ b/submissions/openclaw-forge/fixtures/USER.md @@ -0,0 +1,7 @@ +# Synthetic evaluation identity + +- Display name: Alex Rivera +- Role: Engineering executive +- Initials: AR +- Primary email: tbx-demo2@dev.mscloud.ibm.com +- Time zone: America/New_York diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index f2a6f9e..3889e17 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -61,6 +61,24 @@ def test_evaluate_openshell_step_logs_mlflow(self): assert uri_note in openshell assert openshell.find(uri_note) < openshell.find("scripts/run_aeh.py") + def test_forge_briefing_has_installation_user_fixture(self): + fixture = REPO / "submissions" / "openclaw-forge" / "fixtures" / "USER.md" + fields = {} + for line in fixture.read_text().splitlines(): + if line.startswith("- ") and ": " in line: + key, value = line[2:].split(": ", 1) + fields[key.lower()] = value.strip() + assert all(fields.get(key) and not fields[key].startswith("<") + for key in ("display name", "role", "initials")) + scene = _load(REPO / "submissions" / "openclaw-forge" / "scenes" / "monday-acquisition.yaml") + assert fields["primary email"] == scene["m365"]["user"] + + task = _load(REPO / "pipeline" / "tasks" / "phases" / "evaluate.yaml") + step = next(s for s in task["spec"]["steps"] if s["name"] == "aeh-openshell-eval") + script = step["script"] + assert 'AGENT_EVAL_FORGE_USER_FILE="$SUBMISSION_DIR/fixtures/USER.md"' in script + assert script.index('[ -f "$SUBMISSION_DIR/fixtures/USER.md" ]') < script.index("scripts/run_aeh.py") + def test_harbor_profiles_still_include_test(self): for path in (CI, CI_DEV): names = _task_names(_load(path)["spec"]) From c1cf9d1d8b6617d06cfb47d73a72c74038243e9e Mon Sep 17 00:00:00 2001 From: Tarun Kumar Reddy Etikala Date: Thu, 1 Oct 2026 14:04:08 -0400 Subject: [PATCH 2/2] Pin verified SAW image for Forge briefing evals --- pipeline/pipelines/ci-pipeline-openshell.yaml | 2 +- pipeline/runs/openshell-openclaw-pipelinerun.yaml | 2 +- pipeline/tasks/phases/evaluate.yaml | 2 +- tests/test_openshell_pipeline_profile.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pipeline/pipelines/ci-pipeline-openshell.yaml b/pipeline/pipelines/ci-pipeline-openshell.yaml index 07a41c5..e0b4285 100644 --- a/pipeline/pipelines/ci-pipeline-openshell.yaml +++ b/pipeline/pipelines/ci-pipeline-openshell.yaml @@ -112,7 +112,7 @@ spec: default: "https://openshell.openshell.svc.cluster.local:17670" - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider type: string default: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" diff --git a/pipeline/runs/openshell-openclaw-pipelinerun.yaml b/pipeline/runs/openshell-openclaw-pipelinerun.yaml index 7fe633a..bdd5806 100644 --- a/pipeline/runs/openshell-openclaw-pipelinerun.yaml +++ b/pipeline/runs/openshell-openclaw-pipelinerun.yaml @@ -27,7 +27,7 @@ spec: - name: openshell-ai-gateway-ca-secret value: "forge-agent-upstream-tls" - name: openshell-sandbox-image - value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + value: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" - name: openshell-provider value: "forge-ai-gateway,m365-read-intervm,slack-read-proxy,drafts-service-agent" - name: aeh-openshell-image diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 1f12869..2d12775 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -196,7 +196,7 @@ spec: Override only if using a different gateway (e.g. forge-saw :17670). - name: openshell-sandbox-image type: string - default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + default: "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" description: >- Immutable SAW Chief-of-Staff image with the daily-briefing skill and governed M365/Slack tools. diff --git a/tests/test_openshell_pipeline_profile.py b/tests/test_openshell_pipeline_profile.py index 3889e17..79790cc 100644 --- a/tests/test_openshell_pipeline_profile.py +++ b/tests/test_openshell_pipeline_profile.py @@ -42,7 +42,7 @@ def test_openshell_defaults(self): assert defaults["aeh-runner"] == "openshell" assert defaults["aeh-openshell-image"] == "registry.access.redhat.com/ubi9/python-311:9.6" assert defaults["openshell-sandbox-image"] == ( - "ghcr.io/rh-forge/openclaw-saw-agent@sha256:bcc55e9b7a36d5f65e8ffc75962496f8b3617762a4cdb37fd1cf54611b72d41a" + "ghcr.io/rh-forge/openclaw-saw-agent@sha256:b47b92a6b3fd03327c1f2093a5c28aba0fdf3cb620e9154335688900191fe2b9" ) assert defaults["enable-mlflow"] == "true" assert defaults["mlflow-tracking-uri"] == ("http://abevalflow-mlflow.gz-forge-eval.svc.cluster.local:5000")