From 070a8875aac438fd9dc76e55ede7658d2b778d89 Mon Sep 17 00:00:00 2001 From: Adi El-Sammak <71227923+aelsammak@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:32:26 -0400 Subject: [PATCH 1/5] EBP-2849: Enforce plugin version bumps on release PRs and marketplace sync (#2) * EBP-2849: Enforce plugin version bumps on release PRs and marketplace sync - Add tools/check-plugin-versions.sh requiring a strict semver bump in plugin.json when plugin content (excluding the top-level evals/ corpus) changes relative to the PR base - Gate the version check to PRs targeting main, matching the EBP-2829 decision that bumps happen only in release PRs - Validate marketplace.json stays in sync with plugins/ on every run: entries resolve to existing plugin dirs, names match, no duplicates, every plugin dir listed - Wire both checks into the validate CI job as steps; checkout uses fetch-depth 0 so the merge base is reachable * EBP-2849: Address Claude PR review fixes * EBP-2849: Bump setup-node from Node 20 to Node 24 - Node 20 reached end of life in April 2026 and GitHub Actions runners remove it in fall 2026 - Node 24 is the active LTS and satisfies the Claude Code CLI requirement of Node 18 or newer - Applies to both setup-node usages (validate and trigger-evals jobs) --- .github/workflows/ci.yml | 48 ++++++- README.md | 2 +- tools/check-plugin-versions.sh | 255 +++++++++++++++++++++++++++++++++ 3 files changed, 300 insertions(+), 5 deletions(-) create mode 100755 tools/check-plugin-versions.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff80ce0..e89c22e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,10 +3,17 @@ name: ci # Parallel quality gates (D-16). No `needs:` between jobs so branch # protection on `main` can require each independently: # validate: CI-01: gh skill publish --dry-run + claude plugin validate (marketplace root) +# + marketplace.json/plugins/ sync + version bumps on release PRs +# + post-merge version backstop on pushes to main # link-check: CI-02: the tools/check-links.sh sweep # compile-check: EVAL-02 compile the baked JCSMP samples verbatim # trigger-evals: lightweight skill-triggering check; self-skips green without a key on: + # Deliberately no `edited` type: running on title/body edits either re-runs + # the paid eval matrix for a no-code change or, if jobs are gated, lets the + # edit supersede a red required check (skipped counts as success). Base + # retargeting to main is instead covered by the post-merge backstop below + # plus branch protection "require branches to be up to date" on main. pull_request: push: branches: [main] @@ -19,9 +26,12 @@ permissions: contents: read # Supersede stacked runs on the same ref: each trigger-evals leg spends real -# API budget, so consecutive pushes cancel the previous run instead of queuing. +# API budget, so consecutive PR pushes cancel the previous run instead of +# queuing. Pushes to main group by SHA instead, so a backstop run is never +# cancelled (or pending-replaced) by the next merge, which would leave its +# pushed range unchecked and grey instead of red. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: @@ -38,9 +48,11 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 # version check: merge-base + `git show :` need real history - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 24 - name: Install Claude Code CLI run: npm install -g @anthropic-ai/claude-code - name: Validate skills against the Agent Skills spec (gh skill publish dry run) @@ -50,6 +62,34 @@ jobs: # Validate the Claude plugin/marketplace manifests from the marketplace root ("."). - name: claude plugin validate (marketplace root) run: claude plugin validate . --strict + # Every marketplace entry must resolve to a real plugin dir and every + # plugin dir must be listed (pure tree check, runs on every event). + - name: Marketplace sync check + run: ./tools/check-plugin-versions.sh --sync-only + # Version bumps happen only in release PRs: fork dev -> upstream + # SolaceProducts main. github.base_ref is the PR's target branch in + # whichever repo hosts the PR, so this step fires on those upstream + # release PRs (the enforcement point) and on the fork's post-release + # main sync (where it passes, the bump already landed). It never runs + # on PRs into dev, and base_ref is empty on push and workflow_dispatch, + # so it self-skips there too. Claude Code caches installed plugins by + # the plugin.json version, so an unbumped release never reaches users. + # origin/, not the event's base.sha: the latter goes stale when + # the base branch advances after the last PR event, silently widening + # the diff to other PRs' commits. + - name: Check plugin version bumps (release PRs) + if: github.base_ref == 'main' + run: ./tools/check-plugin-versions.sh "origin/${{ github.base_ref }}" + # Post-merge backstop: merging one release PR does not re-run another's + # checks (synchronize fires only on head pushes), so a second PR can + # merge on a stale green with a version main already ships. Re-checking + # the pushed merge against event.before (the pre-push tip) turns that + # red instead of silent. Prevention needs branch protection (up-to-date + # branches or a merge queue); `!created` skips the branch-creation push, + # where event.before is the all-zeros SHA. + - name: Check plugin version bumps (post-merge backstop) + if: github.event_name == 'push' && !github.event.created + run: ./tools/check-plugin-versions.sh "${{ github.event.before }}" # CI-02: sweep the plugins/ tree for broken and soft-404 documentation links. link-check: @@ -104,7 +144,7 @@ jobs: - uses: actions/setup-node@v4 if: env.HAS_KEY == 'true' with: - node-version: 20 + node-version: 24 - name: Install Claude Code CLI if: env.HAS_KEY == 'true' run: npm install -g @anthropic-ai/claude-code diff --git a/README.md b/README.md index fbca63d..ae43d03 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ A collection of Solace AI agent plugins that package skills for building event-d ### `skills` CLI -This lets you pick specific skills to install and supports most agents. +This lets you pick specific skills to install. The skills in this repository follow the open [Agent Skills](https://agentskills.io) standard, and the [`skills` CLI](https://github.com/vercel-labs/skills) discovers skills in this format from any GitHub repository and installs them into a wide range of agents, including Claude Code, Cursor, Codex, GitHub Copilot, and Windsurf. ```shell npx skills add SolaceProducts/agent-plugins diff --git a/tools/check-plugin-versions.sh b/tools/check-plugin-versions.sh new file mode 100755 index 0000000..3c5613f --- /dev/null +++ b/tools/check-plugin-versions.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# +# check-plugin-versions.sh — enforce the plugin release versioning rules and +# keep marketplace.json in sync with the plugins/ tree. +# +# Claude Code caches an installed plugin by the "version" string in its +# plugin.json: pushing new commits alone is NOT enough for users to receive +# changes, the version must be bumped. Version bumps happen only in +# release PRs (PRs targeting main), so CI runs the version check on those PRs +# and the sync check on every run. +# +# Two checks: +# * Version bump — for every plugin whose content changed relative to the +# base (anything under plugins// except the top-level evals/ corpus, +# which does not ship functionality), plugin.json "version" must be strict +# semver MAJOR.MINOR.PATCH (no "v" prefix, no prerelease) and strictly +# greater than the version at the base. A plugin that is new relative to +# the base only needs a valid semver version. +# * Marketplace sync — every .claude-plugin/marketplace.json entry must +# resolve to an existing plugin directory whose plugin.json "name" matches +# the entry, and every plugins/*/ directory must be listed as an entry. +# +# Usage: tools/check-plugin-versions.sh [base-ref] (default base: origin/main) +# tools/check-plugin-versions.sh --sync-only (marketplace sync only) +# +# The version check diffs merge-base(base-ref, HEAD)..HEAD, so only this +# branch's own committed changes count; commit local changes before running. +# The old version comes from the base-ref tip, so a version the base +# already ships fails the comparison. +# In CI the base is origin/ and checkout uses fetch-depth: 0 +# so the merge base and the base-side plugin.json are reachable. +# +# Exit codes: 0 = all checks pass, 1 = violation(s) or unresolvable base. +# Requires jq (preinstalled on ubuntu-latest runners). + +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 1 + +MARKETPLACE=".claude-plugin/marketplace.json" + +command -v jq >/dev/null 2>&1 || { echo "jq is required but not found" >&2; exit 1; } + +SYNC_ONLY=0 +BASE_REF="origin/main" +if [[ "${1:-}" == "--sync-only" ]]; then + SYNC_ONLY=1 +elif [[ -n "${1:-}" ]]; then + BASE_REF="$1" +fi + +errors=() + +# fail — collect the failure and annotate so the +# message lands on the manifest in the PR Files view. +fail() { + echo "::error file=$1::$2" + errors+=("$2") +} + +# Strict MAJOR.MINOR.PATCH: no "v" prefix, no prerelease/build, no leading zeros. +is_semver() { + local re='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' + [[ "$1" =~ $re ]] +} + +# semver_gt — true only when a > b (equality fails; both pre-validated). +semver_gt() { + local a1 a2 a3 b1 b2 b3 + IFS=. read -r a1 a2 a3 <<<"$1" + IFS=. read -r b1 b2 b3 <<<"$2" + if [[ "$a1" -ne "$b1" ]]; then [[ "$a1" -gt "$b1" ]]; return; fi + if [[ "$a2" -ne "$b2" ]]; then [[ "$a2" -gt "$b2" ]]; return; fi + [[ "$a3" -gt "$b3" ]] +} + +# --- Part A: version bumps for changed plugin content ----------------------- + +if [[ "$SYNC_ONLY" -eq 0 ]]; then + if ! git rev-parse --verify --quiet "${BASE_REF}^{commit}" >/dev/null; then + echo "::error::cannot resolve base ref '${BASE_REF}'" + echo "Cannot resolve base ref '${BASE_REF}'." >&2 + echo "Locally: git fetch origin main (or pass a base ref). In CI: checkout needs fetch-depth: 0." >&2 + exit 1 + fi + # Unchecked, a merge-base failure would leave $merge_base empty, the diff + # below empty, and the whole check would silently pass. + if ! merge_base=$(git merge-base "$BASE_REF" HEAD); then + echo "::error::no common ancestor between '${BASE_REF}' and HEAD" + echo "No common ancestor between '${BASE_REF}' and HEAD (shallow clone or unrelated histories)." >&2 + echo "Locally: use a full clone (git fetch --unshallow)." >&2 + exit 1 + fi + + if [[ "$merge_base" == "$(git rev-parse HEAD)" ]]; then + echo "No commits beyond ${BASE_REF}; nothing to version-check." + else + # core.quotePath=false: git would otherwise C-quote non-ASCII paths + # ("plugins/x/caf\303\251.md"), which the sed extraction below would + # silently skip. --no-renames: a rename reports only its destination + # path, hiding the deletion side from the content filter below. + if ! changed=$(git -c core.quotePath=false diff --no-renames --name-only "$merge_base" HEAD -- plugins/); then + echo "::error::git diff ${merge_base}..HEAD failed" + exit 1 + fi + plugin_names=$(printf '%s\n' "$changed" | sed -n 's|^plugins/\([^/]*\)/.*|\1|p' | sort -u) + + # read -r, not `for ... in`: word splitting would break a plugin dir name + # containing whitespace into fragments and silently skip it. + while IFS= read -r name; do + [[ -n "$name" ]] || continue + manifest="plugins/${name}/.claude-plugin/plugin.json" + + # Deleted plugin: nothing to version; Part B flags the stale marketplace entry. + if [[ ! -d "plugins/${name}" ]]; then + echo "plugins/${name}: directory removed, skipping version check." + continue + fi + + # Content = changed files under the plugin minus its top-level evals/ + # corpus (the corpus does not ship functionality to users). + content="" + while IFS= read -r f; do + case "$f" in + "plugins/${name}/evals/"*) ;; + "plugins/${name}/"*) content+="${f}"$'\n' ;; + esac + done <<<"$changed" + + if [[ -z "$content" ]]; then + echo "plugins/${name}: only evals/ changed, no version bump required." + continue + fi + content_count=$(printf '%s' "$content" | grep -c .) + + if [[ ! -f "$manifest" ]]; then + fail "$manifest" "plugins/${name}: ${manifest} is missing" + continue + fi + new_ver=$(jq -r '.version // empty' "$manifest" 2>/dev/null) + if [[ -z "$new_ver" ]]; then + fail "$manifest" "plugins/${name}: plugin.json has no \"version\" field; set a semver version (new plugins start at 0.1.0)" + continue + fi + if ! is_semver "$new_ver"; then + fail "$manifest" "plugins/${name}: version '${new_ver}' is not strict semver MAJOR.MINOR.PATCH (no 'v' prefix, no prerelease)" + continue + fi + + # Old version from the base-ref tip, not the merge base: a version the + # base already ships would leave users' cached installs stale even if + # it beats the fork-point version. + old_manifest=$(git show "${BASE_REF}:${manifest}" 2>/dev/null) || old_manifest="" + if [[ -z "$old_manifest" ]]; then + echo "plugins/${name}: new plugin at ${new_ver}, OK." + continue + fi + old_ver=$(printf '%s' "$old_manifest" | jq -r '.version // empty' 2>/dev/null) + if [[ -z "$old_ver" ]] || ! is_semver "$old_ver"; then + echo "::warning file=${manifest}::plugins/${name}: base version '${old_ver}' is not strict semver; accepting ${new_ver} without comparison" + continue + fi + + if semver_gt "$new_ver" "$old_ver"; then + echo "plugins/${name}: ${old_ver} -> ${new_ver} covers ${content_count} changed content file(s), OK." + else + fail "$manifest" "plugins/${name}: ${content_count} content file(s) changed but version did not increase (${old_ver} -> ${new_ver}); bump \"version\" in ${manifest} so users receive the changes" + echo " changed content files (first 10):" + printf '%s' "$content" | head -n 10 | sed 's/^/ x /' + fi + done <<<"$plugin_names" + fi +fi + +# --- Part B: marketplace.json <-> plugins/ sync ------------------------------ + +if [[ ! -f "$MARKETPLACE" ]]; then + fail "$MARKETPLACE" "marketplace manifest ${MARKETPLACE} is missing" +# Reject an unparseable manifest up front with a specific error; the checked +# extraction below would also catch it, but with a less specific message. +elif ! jq -e '.plugins | type == "array"' "$MARKETPLACE" >/dev/null 2>&1; then + fail "$MARKETPLACE" "cannot parse ${MARKETPLACE} (invalid JSON or missing .plugins array)" +# Captured, not process-substituted: `done < <(jq ...)` hides a mid-stream +# @tsv failure (non-scalar name), silently passing on partial rows. +elif ! entries=$(jq -r '.plugins[] | [.name, (.source | type), (.source | tostring)] | @tsv' "$MARKETPLACE"); then + fail "$MARKETPLACE" "cannot extract entries from ${MARKETPLACE} (every entry \"name\" must be a JSON scalar)" +else + # Newline-separated source list for the reverse (directory -> entry) check. + sources="" + while IFS=$'\t' read -r ename etype esrc; do + # An empty .plugins array reaches the loop as one empty here-string line. + [[ -n "$etype" ]] || continue + if [[ "$etype" != "string" ]]; then + fail "$MARKETPLACE" "marketplace entry '${ename}': non-string source is not supported by this check; update tools/check-plugin-versions.sh if the format changed" + continue + fi + src="${esrc#./}"; src="${src%/}" + sources+="${src}"$'\n' + if [[ ! -d "$src" ]]; then + fail "$MARKETPLACE" "marketplace entry '${ename}': source '${esrc}' does not exist" + continue + fi + pmanifest="${src}/.claude-plugin/plugin.json" + if [[ ! -f "$pmanifest" ]]; then + fail "$MARKETPLACE" "marketplace entry '${ename}': ${pmanifest} is missing" + continue + fi + pname=$(jq -r '.name // empty' "$pmanifest" 2>/dev/null) + if [[ "$pname" != "$ename" ]]; then + fail "$pmanifest" "marketplace entry name '${ename}' does not match plugin.json name '${pname}'" + fi + done <<<"$entries" + + # Duplicate entry names or sources hide one plugin behind another. Checked + # jq exit so a failed query is not "no duplicates"; read -r so names with + # whitespace stay whole. + if ! dup_names=$(jq -r '[.plugins[].name] | group_by(.)[] | select(length > 1)[0]' "$MARKETPLACE"); then + fail "$MARKETPLACE" "duplicate-name check failed to run against ${MARKETPLACE}" + fi + while IFS= read -r d; do + [[ -n "$d" ]] || continue + fail "$MARKETPLACE" "marketplace has duplicate entries named '${d}'" + done <<<"$dup_names" + if ! dup_sources=$(jq -r '[.plugins[].source | tostring] | group_by(.)[] | select(length > 1)[0]' "$MARKETPLACE"); then + fail "$MARKETPLACE" "duplicate-source check failed to run against ${MARKETPLACE}" + fi + while IFS= read -r d; do + [[ -n "$d" ]] || continue + fail "$MARKETPLACE" "marketplace has duplicate entries with source '${d}'" + done <<<"$dup_sources" + + # Reverse direction: every plugin directory must be listed. + for dir in plugins/*/; do + [[ -d "$dir" ]] || continue # unexpanded glob when plugins/ is empty + dir="${dir%/}" + if ! printf '%s' "$sources" | grep -qxF "$dir"; then + fail "$MARKETPLACE" "plugin directory '${dir}' has no entry in ${MARKETPLACE}" + fi + done +fi + +# --- Summary ----------------------------------------------------------------- + +if [[ ${#errors[@]} -gt 0 ]]; then + echo "" + echo "Plugin version/marketplace violations found:" + printf ' x %s\n' "${errors[@]}" + exit 1 +fi + +if [[ "$SYNC_ONLY" -eq 1 ]]; then + echo "marketplace.json is in sync with plugins/." +else + echo "All changed plugins version-checked against ${BASE_REF}; marketplace.json is in sync with plugins/." +fi From 6b1fd62147e05cba3554dfc46a5f59d374e2e05a Mon Sep 17 00:00:00 2001 From: Adi El-Sammak <71227923+aelsammak@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:43:42 -0400 Subject: [PATCH 2/5] DATAGO-147743: Pin CI actions to commit SHAs and pin the Claude Code CLI install (#3) - Pin actions/checkout, setup-node, setup-java, and upload-artifact to the full commit SHAs their v4 tags currently resolve to, with the tag in a trailing comment - Pin the Claude Code CLI install to 2.1.231 via a workflow-level CLAUDE_CODE_VERSION env value shared by both install steps - Mutable tags and unpinned npm installs let a compromised upstream publish execute in CI; the trigger-evals job holds the ANTHROPIC_API_KEY secret --- .github/workflows/ci.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e89c22e..5b89401 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,12 @@ on: permissions: contents: read +# Pin the Claude Code CLI install to an exact version so CI never auto-pulls a +# fresh npm publish into a job that can hold the ANTHROPIC_API_KEY secret. +# Defined once so the two install steps cannot drift; bump deliberately. +env: + CLAUDE_CODE_VERSION: 2.1.231 + # Supersede stacked runs on the same ref: each trigger-evals leg spends real # API budget, so consecutive PR pushes cancel the previous run instead of # queuing. Pushes to main group by SHA instead, so a backstop run is never @@ -47,14 +53,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 # version check: merge-base + `git show :` need real history - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 24 - name: Install Claude Code CLI - run: npm install -g @anthropic-ai/claude-code + run: npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" - name: Validate skills against the Agent Skills spec (gh skill publish dry run) env: GH_TOKEN: ${{ github.token }} @@ -96,7 +102,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Link-check the plugins tree run: ./tools/check-links.sh plugins @@ -106,8 +112,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 25 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 with: distribution: temurin java-version: 11 @@ -140,14 +146,14 @@ jobs: - name: Warn when skipped (no ANTHROPIC_API_KEY) if: env.HAS_KEY != 'true' run: echo "::warning::trigger-evals self-skipped, the ANTHROPIC_API_KEY secret is not configured; this green check asserted nothing" - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: env.HAS_KEY == 'true' with: node-version: 24 - name: Install Claude Code CLI if: env.HAS_KEY == 'true' - run: npm install -g @anthropic-ai/claude-code + run: npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" - name: Run trigger evals for ${{ matrix.model }} if: env.HAS_KEY == 'true' env: @@ -158,7 +164,7 @@ jobs: # runner VM is destroyed, so upload them where they can be inspected. - name: Upload transcripts on failure if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: trigger-eval-transcripts-${{ matrix.model }} path: ${{ runner.temp }}/trigger-evals From b6eced58da7b5d038053aa231ce5b272317fc5e8 Mon Sep 17 00:00:00 2001 From: Adi El-Sammak <71227923+aelsammak@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:01:55 -0400 Subject: [PATCH 3/5] EBP-2889: Address session-analysis feedback on solace-application-development (#4) * EBP-2889: Address session-analysis feedback on solace-application-development - Promote buried mandates to SKILL.md invariants: AI-assisted disclaimer header, verification artifacts (checklist + verify.sh + hooks), and the live-broker heads-up; extend the doc-grounding invariant to ban substitute channels and the version invariant to ban the solrsearch index - Surface the Step 0 door-question gate and the environment-discovery rule (a found broker/config never answers a question) in jcsmp.md, design-mode.md, and implement-mode Step 0 - Split implement-mode.md into a core file plus three per-leaf wiring files so the core fits a single Read; state the generation file-output contract up front and move the checklist write into Step 4 generation output - Redesign verification to be shape-agnostic: verify.sh gains an app stage driven by a generated verify-hooks.sh (START_CMD/TRIGGER_CMD/READY_MARKER/PASS_MARKER), a warn-only conformance preflight, and a live-run heads-up line; curl may trigger but never render the verdict - Replace the log4j2 2.26.0 pin with outcome rules: resolve the highest stable 2.x from authoritative metadata, framework-owned SLF4J backends win, config file required, com.solacesystems loggers never silenced, and the 2.17.1 Log4Shell floor enforced as a check - Bind doc grounding to each design question and redefine the Grounding docs field to fetched-pages-only; add a Generation conformance group to the verification checklist - Remove the mandated Core Concepts intro from SKILL.md and prerequisites.md; prerequisites is now broker acquisition only - Solace Suggested decoupling: keep the two-project default but require a surfaced conflict with three resolutions on single-deliverable asks; never merge silently - Rename the Java API Release Notes link to JCSMP API Release Notes (the JAVA_API_RN download serves the JCSMP release notes) and point sample logging comments at the Step 3 rule - Root-caused from 10 analyzed sessions; violations and fixes tracked in EBP-2889 * EBP-2889: Route build requests through Design mode before Implement - Key mode routing on the design contract instead of request phrasing; a build prompt without a contract enters Design first, including web and embedded shapes - Add the design-contract gate to jcsmp.md and make the summary a hard precondition in Implement Step 2 (valid sources: same-session summary, explicit summary in chat, saved solace-design.md) - Add the topology rule for edits to existing apps: topology-changing edits re-enter Design, mechanical edits go straight to Implement - Add Design mode's fully-specified path: a prompt that answers every tree question gets one merged confirm at close-out - Dedupe the broker-access question in Implement Step 1 when Design Step 0 already confirmed it this session - Keep the door question an isolated Implement Step 0; a confirmed design never answers it * EBP-2889: Carry sample best-practice comments into generated code - Add the comments-follow-their-code contract to implement-mode Step 4: a construct carried from a sample keeps that construct's comment (names adapted), dropped demo harness takes its comments with it, and fresh messaging code that applies a documented practice gets a short comment naming it - Add a binary item to the checklist's Generation conformance group so a run that strips the comments fails its own checklist; keep it outside the preflight's first-four mechanical checks - Align the Step 6 group description with the new non-preflight item - Evidence: a generated embedded Direct Pub/Sub project carried sample constructs without their attached best-practice comments, while every explicitly mandated one-line comment survived (slot-bound rules get followed; unbound ones do not) * EBP-2889: Remove the Non-Secure sub-mode from Solace Suggested - Solace Suggested is TLS-secure by definition: tcps:// with server-certificate validation, requiring a TLS-configured broker; there is no non-secure variant and no Step 0 sub-question, so the door question stays a clean three-way choice - On a broker without TLS, the skill states the requirement and offers Custom with the equivalent knobs minus the secure session (the developer's explicit choice) or Quickstart; it never generates a plaintext session under the Solace Suggested name - Collapse the overlay's Step 1 fork to the TLS build only and delete the plaintext session section; the no-TLS case routes back to the Step 0 door - Update the checklist branch notes, the Step 6 tailoring examples, the Custom span description, and the comment-contract bullet accordingly - Feedback source: live session screenshot showed the door question flattened into four options with Non-Secure preselected * EBP-2889: State Quickstart's plaintext posture explicitly - Make explicit that Quickstart has no TLS requirement and connects with whatever scheme the host carries; a plaintext tcp:// dev broker is a normal Quickstart target - Guard against over-generalizing the new Solace Suggested TLS requirement onto the Quickstart door * EBP-2889: Harden verify.sh teardown against orphaned app processes - Make shutdown_long_running and the cleanup trap signal the process group (kill -- -PID) instead of the bare PID; every launch site backgrounds under set -m, so each recorded PID leads its own group - Close the macOS bash 3.2 leak where a compound START_CMD keeps the bash -c wrapper alive and a PID-directed kill stopped only the wrapper, orphaning the app (bash 4.4+ execs the final command) - Add a post-wait KILL sweep for any group member that outlives the leader - Document the SIGKILL residual: no trap can run on a hard kill, and set -m places children outside the script's group, so that orphan risk cannot be closed from inside the script - Swap shellcheck disable=SC1091 for the source=/dev/null idiom on the verify-hooks.sh source line - Addresses PR #4 review comments; verified on bash 3.2/4.4/5.0/5.1/5.2 (macOS, Ubuntu 20/22/24.04, Debian 12, openSUSE Leap 15.5): the trap runs on INT/TERM, rc semantics stay 130, and no process leaks * EBP-2889: Remove stale line numbers and wave-plan relics from comments - Remove all sample line-number references from implement-guaranteed-pubsub.md; the numbers drifted from the current samples and the text already names each construct - Remove the stale line-number reference from the ENDPOINT_DENIED comment in verify.sh - Drop the 'NEW (waves 2-4)' annotations and the 'Placeholder stages' header in verify.sh; the direct and request-reply stages are fully implemented - Addresses PR #4 inline review feedback on comment maintenance * EBP-2889: Bound the app-stage trigger and isolate verify-hooks.sh imports - Run TRIGGER_CMD backgrounded in its own process group, polled to PUBLISH_WAIT_S, then escalate SIGTERM/SIGKILL; the app stage's trigger was the script's only unbounded wait, so a wedged app could hang an unattended run forever - Keep the marker watch after a hung trigger; the markers render the verdict, and the publish may have landed even when the HTTP response never returned - Reuse PUB_PID for the trigger so the EXIT trap's group kill covers an orphaned trigger too - Source verify-hooks.sh in a child bash and import only the four contract values via declare -p; a stray cd, variable, or function in the generated file can no longer clobber the observer state or force a false PASS - Add --max-time to the template curl and document the four-assignment contract in implement-mode.md - Verified on macOS bash 3.2.57: a hostile hooks file (status=0, TIMEOUT_S override, cd, stray stdout) plus a 300s trigger exits 1 in 18s with no leaked processes; the pass path exits 0 - Addresses Claude-review findings 3 and 4 from PR #4 review feedback * EBP-2889: Note the documented reply-to shape in the guaranteed requestor sample - Add a comment at the setReplyTo site linking the canonical Queues page (well-known-queues section), which documents anonymous/temporary queues in the reply-to field for request/reply - The comments-follow-their-code contract carries the note into generated adaptations --- .../solace-application-development/SKILL.md | 9 +- .../references/jcsmp.md | 39 +- .../references/jcsmp/custom-mode.md | 6 +- .../references/jcsmp/design-mode.md | 17 +- .../jcsmp/evals/compile-fixture/pom.xml | 7 +- .../jcsmp/implement-direct-pubsub.md | 34 ++ .../jcsmp/implement-guaranteed-pubsub.md | 73 ++++ .../references/jcsmp/implement-mode.md | 354 ++++++------------ .../jcsmp/implement-request-reply.md | 78 ++++ .../jcsmp/jcsmp-direct-publisher-sample.java | 6 +- .../jcsmp/jcsmp-direct-replier-sample.java | 6 +- .../jcsmp/jcsmp-direct-requestor-sample.java | 6 +- .../jcsmp/jcsmp-direct-subscriber-sample.java | 6 +- .../jcsmp-guaranteed-publisher-sample.java | 6 +- .../jcsmp-guaranteed-replier-sample.java | 6 +- .../jcsmp-guaranteed-requestor-sample.java | 8 +- .../jcsmp-guaranteed-subscriber-sample.java | 6 +- .../jcsmp/jcsmp-solace-connection-config.java | 4 + .../references/jcsmp/prerequisites.md | 9 +- .../references/jcsmp/scripts/verify.sh | 256 +++++++++++-- .../references/jcsmp/solace-suggested-mode.md | 32 +- .../jcsmp/verification-checklist.md | 20 +- 22 files changed, 663 insertions(+), 325 deletions(-) create mode 100644 plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-direct-pubsub.md create mode 100644 plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-guaranteed-pubsub.md create mode 100644 plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-request-reply.md diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md b/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md index e8b100f..a291599 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md @@ -20,7 +20,7 @@ This skill is the front door for building Solace messaging applications grounded ## What is Solace -Ground the developer in the high-level model before discussing patterns or generating code. Solace is an event-driven messaging platform: an event broker carries messages between applications, brokers connect into an event mesh, publishers send to named topics, and consumers read either directly or from durable queues that subscribe to those topics (the publish and subscribe model). Solace Cloud is the primary broker this skill assumes for the work it generates; a self-hosted Software Broker or an existing Appliance are supported alternatives. Point the developer at the canonical [Solace Core Concepts](https://docs.solace.com/Get-Started/event-mesh-basics.md) page for the grounding, confirm they grasp the publish/subscribe and queue concepts, then continue. Do not paraphrase the page; link it. +Solace is an event-driven messaging platform: an event broker carries messages between applications, brokers connect into an event mesh, publishers send to named topics, and consumers read either directly or from durable queues that subscribe to those topics (the publish and subscribe model). Solace Cloud is the primary broker this skill assumes for the work it generates; a self-hosted Software Broker or an existing Appliance are supported alternatives. The canonical [Solace Core Concepts](https://docs.solace.com/Get-Started/event-mesh-basics.md) page is the doc grounding for this model. For topic-hierarchy and topic-architecture design questions, the co-installed solace-topic-best-practices skill reads the canonical Topic Architecture Best Practices page online and applies it to those decisions. @@ -39,8 +39,11 @@ If the API is unclear but the request is clearly about Solace messaging in Java, Non-negotiable rules that apply across every API. Apply all. 1. **Content sourcing**: Every named entity (API, parameter, configuration value) must be traceable to a canonical Solace source. Do not assert behavior, defaults, or best practices that are not in the grounding documentation. -2. **WebFetch-on-demand doc grounding**: This skill bundles no documentation. When a step needs documentation content to ground a generation or design decision, WebFetch the live canonical page on demand (its `docs.solace.com` `.md` URL, the same link the reference files carry), then quote or summarize the fetched page. Do not answer from memory and do not paraphrase guidance the page does not contain. Do not restate doc content in the skill; link the live `.md` URL. A short list of references stays live for the same reason it always has (Javadoc HTML, the Java API Release Notes, Maven Central, the tutorials, and the GitHub samples) because those have no `docs.solace.com` `.md` form. -3. **No hardcoded versions**: Never pin a library version anywhere in skill content. The generated build resolves the latest release of the relevant Solace coordinate at generation time. Each API's entry file names the exact coordinate to use. +2. **WebFetch-on-demand doc grounding**: This skill bundles no documentation. When a step needs documentation content to ground a generation or design decision, WebFetch the live canonical page on demand (its `docs.solace.com` `.md` URL, the same link the reference files carry), then quote or summarize the fetched page. Do not answer from memory and do not paraphrase guidance the page does not contain. Do not restate doc content in the skill; link the live `.md` URL. Never substitute another channel for that WebFetch: grounding taken from a docs chatbot, a search tool, or decompiling a jar does not satisfy this invariant, and a page never fetched in the session must never be cited as grounding. A short list of references stays live for the same reason it always has (Javadoc HTML, the JCSMP API Release Notes, Maven Central, the tutorials, and the GitHub samples) because those have no `docs.solace.com` `.md` form. +3. **No hardcoded versions**: Never pin a library version anywhere in skill content. The generated build resolves the latest release of every dependency at generation time from the AUTHORITATIVE Maven repository metadata; each API's entry file names the exact coordinate and carries the exact lookup command. Never read a version from the legacy `search.maven.org/solrsearch` index: it lags the repository metadata and reports stale versions. Security floors (a minimum version below which known CVEs live) are enforced as checks on the resolved result, never as pins. +4. **AI-assisted disclaimer header**: Every source file generated under this skill starts with this exact line: `AI-assisted code. Review before production use.` Directly below it: `See the verification checklist: solace-verification-checklist.md`. This applies to EVERY generated source file, including custom builds, web apps, and embedded shapes that adapt the reference samples outside the canonical generator. The pom and other non-source artifacts are exempt; the bundled reference samples themselves carry no header. +5. **Verification artifacts, always**: Any session that generates Solace code from this skill writes a tailored `solace-verification-checklist.md` beside the code as generation output, copies the bundled `verify.sh` (plus its generated `verify-hooks.sh`) into the project, and ends in a real verify run or the compile-only fallback with the exact handed-back commands. The `VERIFY:` markers are non-optional in generated Solace messaging code, whatever the app shape. An improvised check (for example curl against the app's own HTTP API) may TRIGGER traffic but never renders the verdict; the verdict comes from the markers and the verify exit code. +6. **Live-broker heads-up**: Immediately before starting ANY process that will connect to a live broker — verify.sh, `java -jar`, a `mvn` run, a run script, anything — print one line that names the broker host and states the actual broker-side effects of this run (the queues it provisions, the connections it opens, the messages it publishes), scaled to the real app. Environment discovery is never consent and never an answer: a running local broker, an existing config file, or found credentials are facts to report, not answers to consume; report them, then still ask which broker the developer wants to target. ## Reference Files (read on-demand only) diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md index a2784e9..d3ba17d 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md @@ -6,19 +6,37 @@ The cross-cutting invariants (content sourcing, WebFetch-on-demand doc grounding ## Mode Detection -Determine the user's intent and enter the appropriate mode: +Determine the user's intent and enter the appropriate mode. Routing is keyed on the design contract, not on the phrasing of the request: Implement mode never starts without a confirmed design (the design-contract gate below), so a "build me..." prompt is NOT an Implement signal by itself. | User intent | Mode | What to do | |---|---|---| | "Help me choose the Solace JCSMP messaging pattern for a new app" / "Topic or queue?" / "What delivery semantics should I use?" | **Design** | Read `jcsmp/design-mode.md` | -| "Build me a Solace JCSMP publisher/consumer app" / "Generate a Maven JCSMP project" / "Publish to a topic and consume from a queue" | **Implement** | Read `jcsmp/implement-mode.md` | +| Build or generate a NEW app or messaging component with NO valid design contract yet — "Build me a Solace JCSMP publisher/consumer app", "Generate a Maven JCSMP project", a web app / dashboard / service, Spring Boot, or any embedded shape where the JCSMP layer lives inside a larger application | **Design first** | Read `jcsmp/design-mode.md`. A prompt that already states the design resolves on its fully-specified path in ONE confirm; a bare prompt walks the tree. The confirmed summary then feeds Implement. | +| A build request WITH a valid design contract (see the design-contract gate below) | **Implement** | Read `jcsmp/implement-mode.md`; for an embedded shape generate the messaging layer per its leaf rules and verify with the `app` stage of `verify.sh` (its Step 5) | +| An edit to an EXISTING app that changes the messaging topology (the topology rule below) | **Design first** | Re-enter `jcsmp/design-mode.md` to re-confirm the affected summary fields, then Implement applies the change | +| A mechanical edit to an EXISTING app with no topology change — reconnect handling, payload format, logging, renames, ack tuning | **Implement** | Read `jcsmp/implement-mode.md`; no design pass | | "My JCSMP app is throwing on connect" / "Why is my consumer not binding the queue?" | **Debug** | Debug mode is not yet available in this release. Redirect the user to the canonical Solace JCSMP troubleshooting documentation: [JCSMP API Home](https://docs.solace.com/API/Messaging-APIs/JCSMP-API/jcsmp-api-home.md). Do not generate debugging guidance from memory. | If unclear, default to **Design**. Understand the messaging problem before generating code. +**The topology rule** decides between the two existing-app rows. An edit changes the messaging topology when it changes the structure the design summary records: a new app or messaging component (a new publisher, consumer, requestor, or replier, including a leaf change such as single-service growing into fan-out), a pattern change, a delivery-mode change, a consumption-endpoint change, or a queue access-type change. That list is the core; also treat an unlisted change as topology when your judgment says it alters the message flow between the apps and the broker. Everything else is a mechanical edit. + +Three gates hold on every path: + +- **The design-contract gate.** Implement mode never starts without a design contract, and never invents design values. Exactly three sources satisfy it: (1) a summary Design mode confirmed in this session — use it as-is, with no re-confirm; (2) an explicit summary the developer supplied in chat (the eight fields or an equivalent statement of them) — use it directly; (3) a saved `solace-design.md` in the project — restate it in one line and proceed on confirm. A prose build request that merely mentions pattern details is NOT a contract: route it through Design mode, whose fully-specified path derives the summary, echoes it, and takes one confirm. The one carve-out is the mechanical-edit row above. An experienced developer therefore never has to walk the design tree, but never skips the contract either. +- **Implement mode opens with a mandatory door question.** Ask Quickstart, Solace Suggested, or Custom — before you request or accept ANY broker details (`jcsmp/implement-mode.md` Step 0). Ask it even when a `config.json` with credentials already exists, even when a local broker is already running, and even when the requested app is bigger than the canonical generated shape; none of those answers the question. A confirmed design summary does not answer it either: the door question always follows the design contract, never merges into it. +- **Environment discovery never answers a question.** A running broker container, an existing config file, or found credentials are facts to report, not answers to consume. Report what you found, then still ask which broker the developer wants to target. + ## JCSMP coordinate -The single Solace dependency for this API is `com.solacesystems:sol-jcsmp`. Use the groupId `com.solacesystems` exactly; the shorter `com.solace` form does not resolve on Maven Central. Resolve the latest release at generation time (the no-hardcoded-versions invariant); never pin a `sol-jcsmp` version in skill content. +The single Solace dependency for this API is `com.solacesystems:sol-jcsmp`. Use the groupId `com.solacesystems` exactly; the shorter `com.solace` form does not resolve on Maven Central. Resolve the latest release at generation time (the no-hardcoded-versions invariant); never pin a `sol-jcsmp` version in skill content. Resolve it from the authoritative metadata with exactly this command, and read the `` element: + +```bash +curl -s https://repo1.maven.org/maven2/com/solacesystems/sol-jcsmp/maven-metadata.xml \ + | grep -oE '[^<]+' +``` + +Do NOT read the version from the legacy solrsearch index (`search.maven.org/solrsearch`): it lags behind the repository metadata and reports a stale `latestVersion`. ## Canonical doc links @@ -31,17 +49,20 @@ JCSMP-specific grounding. Each page below is a live `docs.solace.com` `.md` URL, Live exceptions (these have no `docs.solace.com` `.md` form; use the live URL directly): - [JCSMP Javadoc](https://docs.solace.com/API-Developer-Online-Ref-Documentation/java/index.html) -- [Java API Release Notes](https://products.solace.com/download/JAVA_API_RN) +- [JCSMP API Release Notes](https://products.solace.com/download/JAVA_API_RN) (the `JAVA_API_RN` download serves the JCSMP release notes; it is NOT the newer Solace Messaging API for Java) - [sol-jcsmp on Maven Central](https://central.sonatype.com/artifact/com.solacesystems/sol-jcsmp) ## Mode and reference files (read on-demand only) -- `jcsmp/prerequisites.md`: Solace core-concepts grounding and broker acquisition (route here first if the developer has no reachable broker). -- `jcsmp/design-mode.md`: choose the messaging pattern/topology before generating code. -- `jcsmp/implement-mode.md`: generate a runnable Maven JCSMP pub/sub project. +- `jcsmp/prerequisites.md`: broker acquisition (route here first if the developer has no reachable broker). +- `jcsmp/design-mode.md`: choose and confirm the messaging pattern/topology before any code is generated; every new build without a design contract routes here first. +- `jcsmp/implement-mode.md`: generate a runnable Maven JCSMP project from a confirmed design (any leaf, any app shape); its Step 4 dispatches onto the per-leaf wiring files below. +- `jcsmp/implement-guaranteed-pubsub.md`: the Guaranteed Pub/Sub leaf wiring (read from implement-mode Step 4). +- `jcsmp/implement-direct-pubsub.md`: the Direct Pub/Sub leaf wiring (read from implement-mode Step 4). +- `jcsmp/implement-request-reply.md`: both Request-Reply leaf wirings (read from implement-mode Step 4). - `jcsmp/solace-suggested-mode.md`: the leaf-agnostic Solace Suggested overlay (TLS secure session, DMQ on PERSISTENT, HA failover, separate Maven projects) reached from implement-mode Step 0 on the Solace Suggested path. - `jcsmp/custom-mode.md`: the thin a la carte Custom overlay reached from implement-mode Step 0 on the Custom path; a leaf-aware checklist of the hardening knobs (secure session, DMQ, HA, decoupled projects, admin-provisioned queue) that generates exactly the ticked subset, reusing the Solace Suggested overlay steps for the mapped knobs. -- `jcsmp/verification-checklist.md`: the master template of binary verification checks organized into three responsibility groups (delivered by this generation, the developer's responsibility, and verified by the round-trip); Implement mode Step 6 emits a tailored `solace-verification-checklist.md` into each generated project seeded for the chosen mode and reports the same in chat. -- `jcsmp/scripts/verify.sh`: the bundled stage-dispatched run-and-observe verify script (`verify.sh [pass]`) that Implement mode runs per generation stage against a reachable broker. +- `jcsmp/verification-checklist.md`: the master template of binary verification checks organized into responsibility groups (delivered by this generation, the developer's responsibility, verified by the round-trip, and generation conformance); Implement mode Step 4 writes a tailored `solace-verification-checklist.md` into each generated project as generation output, and Step 6 reports the same resolution in chat. +- `jcsmp/scripts/verify.sh`: the bundled stage-dispatched run-and-observe verify script (`verify.sh ...`) that Implement mode copies into every generated project and runs against a reachable broker; the `app` stage drives web and embedded shapes through a generated `verify-hooks.sh`. - `jcsmp/jcsmp-guaranteed-publisher-sample.java`: best-practices publisher sample (basic-auth connect, PERSISTENT publish to a topic, graceful shutdown). - `jcsmp/jcsmp-guaranteed-subscriber-sample.java`: best-practices consumer sample (provisions a durable queue + topic subscription, CLIENT-ack flow). diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/custom-mode.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/custom-mode.md index 300dc87..e80db0c 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/custom-mode.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/custom-mode.md @@ -2,7 +2,7 @@ The Custom overlay for Implement mode. This file is reached from `implement-mode.md` Step 0 when the developer chooses the Custom door. Custom is a la carte: the developer ticks which hardening knobs to include, and generation applies exactly the ticked subset and nothing more. It is a thin overlay. It does NOT re-describe how each knob works. For the four knobs that map to a `solace-suggested-mode.md` step, it points at that step and reuses it, so `solace-suggested-mode.md` stays the single source of truth for each hardening delta. -This overlay defers to `implement-mode.md` for all leaf and sample mechanics, exactly the way `solace-suggested-mode.md` does: the Step 2 design-summary input contract, the Step 4 leaf dispatch (which sample pair each leaf reads, and the near-verbatim adaptation discipline), the AI-assisted disclaimer header on every generated class, the explicit single-class imports invariant (keep every import explicit, per `implement-mode.md`; never collapse to a wildcard), and the four-file `VERIFY:` marker contract. Custom adds NO verify stage and changes NO `VERIFY:` marker string. +This overlay defers to `implement-mode.md` for all leaf and sample mechanics, exactly the way `solace-suggested-mode.md` does: the Step 2 design-summary input contract, the Step 4 leaf dispatch (which leaf wiring file and sample pair each leaf reads, and the near-verbatim adaptation discipline), the AI-assisted disclaimer header on every generated class, the explicit single-class imports invariant (keep every import explicit, per `implement-mode.md`; never collapse to a wildcard), and the fixed `VERIFY:` marker contract. Custom adds NO verify stage and changes NO `VERIFY:` marker string. Any live run on the Custom path prints the same one-line live-broker heads-up (SKILL.md Invariant 6) immediately before it starts: name the broker host and the run's actual broker-side effects. ## The checklist model: one up-front multi-select @@ -22,7 +22,7 @@ For knobs 1 through 4, point at the named overlay step and reuse it. Do NOT dupl ## The Quickstart-shaped floor -Zero knobs ticked yields a Quickstart-shaped base app: a single Maven project with two mains, a self-provisioned durable queue, plaintext `tcp://` transport, and the `config.json`-or-CLI connection convention. Each ticked knob layers its Suggested delta on top of that floor. There is NO minimum-one-knob guard: zero ticks is a valid result, and it is the Quickstart-shaped base app. One exception to the Quickstart shape: the floor KEEPS the reference samples' reconnect baseline (`reconnectRetries` 20, `connectRetriesPerHost` 3); the fail-fast default-channel posture is the Quickstart door's alone, and on Custom the HA knob decides between that baseline (unticked) and the four HA values (ticked). Custom therefore spans cleanly from about Quickstart (nothing ticked) to about Solace Suggested Secure (everything ticked). +Zero knobs ticked yields a Quickstart-shaped base app: a single Maven project with two mains, a self-provisioned durable queue, plaintext `tcp://` transport, and the `config.json`-or-CLI connection convention. Each ticked knob layers its Suggested delta on top of that floor. There is NO minimum-one-knob guard: zero ticks is a valid result, and it is the Quickstart-shaped base app. One exception to the Quickstart shape: the floor KEEPS the reference samples' reconnect baseline (`reconnectRetries` 20, `connectRetriesPerHost` 3); the fail-fast default-channel posture is the Quickstart door's alone, and on Custom the HA knob decides between that baseline (unticked) and the four HA values (ticked). Custom therefore spans cleanly from about Quickstart (nothing ticked) to about Solace Suggested (everything ticked). When the secure-session knob is unticked, the base stays plaintext `tcp://` (the same transport Quickstart uses) and carries an honest plaintext caveat: unencrypted transport, suitable for a dev or test broker with no TLS, not for real credentials or real data over untrusted networks. When the secure-session knob is ticked, the session upgrades to the `solace-suggested-mode.md` Step 1 Secure `tcps://` path with explicit certificate validation. @@ -62,7 +62,7 @@ Because the ticked app no longer provisions anything, an autonomous round-trip w ## Return to the implement-mode contracts -After capturing the ticked subset and applying the deltas above, return to `implement-mode.md` for the unchanged leaf mechanics: the Step 4 dispatch generates the chosen leaf's classes, and Step 6 resolves each `verification-checklist.md` item to its state for the Custom ticked subset, writes the tailored `solace-verification-checklist.md` into the project (one copy per project root when the decoupled-projects knob is ticked, otherwise one at the project root), invites the developer's additions, and reports the same in chat, the same way it does on Quickstart and Solace Suggested. This overlay selects which hardening deltas to layer; it does not replace those contracts. +After capturing the ticked subset and applying the deltas above, return to `implement-mode.md` for the unchanged leaf mechanics: the Step 4 dispatch generates the chosen leaf's classes and writes the tailored `solace-verification-checklist.md` into the project as generation output (one copy per project root when the decoupled-projects knob is ticked, otherwise one at the project root), and Step 6 resolves each item to its state for the Custom ticked subset, invites the developer's additions, and reports the same in chat, the same way it does on Quickstart and Solace Suggested. This overlay selects which hardening deltas to layer; it does not replace those contracts. ## Grounding references diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/design-mode.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/design-mode.md index 9ea6e22..92f9710 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/design-mode.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/design-mode.md @@ -6,12 +6,9 @@ The output of this mode is a filled-in design summary the developer can carry in ## Step 0: Confirm broker access and grounding -Before triaging anything, confirm two things (skip whichever the developer already has): +Before triaging anything, confirm the developer has access to a reachable broker. Confirm that they HAVE one; Design mode only records the broker TYPE (for example, Solace Cloud) in the summary, so do NOT ask them to paste connection details (host, message VPN, client username, password) into the chat. Those values are handled later in Implement mode, where Solace Suggested credentials go only into a gitignored `config.json` (Quickstart falls back to CLI args when it is absent). A broker discovered running in the environment (a local container, an existing config file) is a fact to report, never an answer: still confirm which broker the developer wants to target. -- The developer has access to a reachable broker. Confirm that they HAVE one; Design mode only records the broker TYPE (for example, Solace Cloud) in the summary, so do NOT ask them to paste connection details (host, message VPN, client username, password) into the chat. Those values are handled later in Implement mode, where Solace Suggested credentials go only into a gitignored `config.json` (Quickstart falls back to CLI args when it is absent). -- The developer understands the basic Solace model (broker, topics, queues, publish/subscribe). - -If either is missing, route through `prerequisites.md` first (Core Concepts grounding and broker acquisition, Solace Cloud recommended), then return here. +If the developer has no reachable broker, route through `prerequisites.md` first (broker acquisition, Solace Cloud recommended), then return here. ## Step 1: Read the prompt for stated intent before asking anything @@ -30,6 +27,8 @@ If a row matches, or if your own judgment maps an unmatched prompt cleanly to on **Recommend path (intent is clear).** Recommend the leaf in ONE line and confirm, for example "Sounds like Guaranteed Pub/Sub (fan-out), confirm?", then proceed on the developer's confirm. One line plus one confirm, not a rationale paragraph and not a silent zero-confirm jump. +**Fully-specified path (the prompt answers every applicable tree question).** When the prompt resolves the leaf AND answers every Step 2 question that applies to that leaf, skip the separate leaf confirm. Derive the full summary, present it (Step 4), and let the close-out question carry the single confirm: one confirm covers the leaf and the design. Do not ask a leaf confirm and then a close-out confirm; that is two questions where the prompt earned one. + **Partial-intent path (intent is partial).** When the prompt answers some branches but not all (for example it says "request a reply" but not direct vs guaranteed), lock the branches the prompt already answers and ask ONLY the remaining tree questions, in the Step 2 order, skipping every question the prompt already settled. Reach a leaf, then confirm it the same one-line way. **Unclear path (no stated intent).** When the prompt states no usable intent, walk all four tree questions in Step 2 order to reach a leaf. @@ -38,7 +37,7 @@ If a row matches, or if your own judgment maps an unmatched prompt cleanly to on ## Step 2: Walk the decision tree to a leaf -Ask only the questions the prompt left open, in this order. Read each grounding link only when that question comes up; link the canonical doc and state the decision in one line, never paraphrase the doc. +Ask only the questions the prompt left open, in this order. Before you answer each question you do walk, WebFetch its grounding page first — even when the prompt pre-answers the question — then link the canonical doc and state the decision in one line, quoting one line from the fetched page; never paraphrase guidance the page does not contain. A page you did not fetch in this session must not appear in the summary's Grounding docs field. 1. **Interaction shape: request-reply or publish/subscribe?** Grounded in [Message Exchange Patterns](https://docs.solace.com/Get-Started/message-exchange-patterns.md). Decision: a request that expects a correlated reply is Request-Reply; an event fanned out to one or many independent consumers is publish/subscribe. 2. **Delivery guarantee: direct or guaranteed?** This ONE question applies to both interaction shapes, so ask it next regardless of the answer to question 1. Grounded in [Message Delivery Modes](https://docs.solace.com/Get-Started/message-delivery-modes.md). Decision: direct is fire-and-forget with no broker-side persistence, for high-rate flows that tolerate occasional loss; guaranteed (PERSISTENT) is persisted and survives consumer downtime. @@ -84,7 +83,7 @@ Solace JCSMP Design Summary - Consumption endpoint: - Auth: Basic username/password - Broker: Solace Cloud -- Grounding docs: +- Grounding docs: ``` The eight fields are fixed: Pattern, Delivery, Access type, Topic, Consumption endpoint, Auth, Broker, Grounding docs. Formatting is at your discretion, but always present all eight, every run, with explicit `n/a` where a field does not apply. The eight field NAMES are fixed; the field VALUES record the design's actual choices, defaulting to the base-leaf shape and holding the variant topology when the design deviates (the variant path in Step 1). The `Pattern` value stays one of the six leaf strings verbatim even for a variant; the deviation lives in the topology fields (`Consumption endpoint`, and `Delivery` or `Access type` where relevant), never in a new leaf string. @@ -98,6 +97,6 @@ Field semantics: - **Topic.** The recommended topic string or hierarchy, derived by applying the solace-topic-best-practices skill to the use case (not chosen by interrogating the developer). Always publish to topics; never address a queue directly. - **Auth.** Basic username/password. - **Broker.** Default to `Solace Cloud` and record it without asking. Switch to another type (Software Broker or Appliance) ONLY if the developer explicitly insists; otherwise leave it as `Solace Cloud`. -- **Grounding docs.** The canonical pages this design referenced. +- **Grounding docs.** ONLY the pages this session actually WebFetched. Never list a page you did not fetch: a citation without a fetch is fabricated grounding. Write `none fetched` when no page was fetched; that is an honest value, an unfetched citation is not. -After presenting the summary in chat, close Design mode explicitly so the developer knows exactly what happens next; do NOT just display the summary and stop. If the design is a variant of its leaf, state the deviation and its rationale in one line here too, so the developer approves the actual topology (not just the leaf name) before it is built. Ask them directly, in one step, both whether they are happy with this design AND whether to save it to `solace-design.md` in their project (for example: "Happy with this design? If so, I can save it to `solace-design.md` and move into Implement mode to generate the runnable Maven project."). Write the file only on their OK; never write it unprompted. Once they confirm, state the next step plainly: the work moves into Implement mode, which generates the runnable Maven project from this summary. Implement mode treats this summary, whether it lives in the chat or in the saved `solace-design.md`, as its input contract. +After presenting the summary in chat, close Design mode explicitly so the developer knows exactly what happens next; do NOT just display the summary and stop. On the fully-specified path (Step 1) this close-out is the ONLY confirm of the run; it carries the leaf confirmation too. If the design is a variant of its leaf, state the deviation and its rationale in one line here too, so the developer approves the actual topology (not just the leaf name) before it is built. Ask them directly, in one step, both whether they are happy with this design AND whether to save it to `solace-design.md` in their project (for example: "Happy with this design? If so, I can save it to `solace-design.md` and move into Implement mode to generate the runnable Maven project."). Write the file only on their OK; never write it unprompted. Once they confirm, state the next step plainly: the work moves into Implement mode, which generates the runnable Maven project from this summary. Implement mode treats this summary, whether it lives in the chat or in the saved `solace-design.md`, as its input contract. Implement mode OPENS with its own Step 0 door question (Quickstart, Solace Suggested, or Custom), asked before any broker details are requested or accepted; an existing `config.json` or a running broker does not answer it, so ask it on the way in. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml index 4d71da3..eb9016d 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml @@ -13,8 +13,11 @@ The sol-jcsmp version is intentionally version-free here (Invariant 3): the property default below is a NON-NUMERIC placeholder token that compile.sh overrides at run time with the live from Maven Central metadata. - No sol-jcsmp version number is committed anywhere. log4j2 is the deliberate - pinned exception (2.26.0, at/above the 2.17.1 Log4Shell floor). + No sol-jcsmp version number is committed anywhere. The log4j2 version below + is FIXTURE-ONLY tooling (a concrete number so this CI fixture compiles + deterministically, at/above the 2.17.1 Log4Shell floor); it is NOT a skill + rule. Generated projects resolve their logging backend live per + implement-mode.md Step 3 and never pin it. --> com.solace.samples diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-direct-pubsub.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-direct-pubsub.md new file mode 100644 index 0000000..e2b68c8 --- /dev/null +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-direct-pubsub.md @@ -0,0 +1,34 @@ +# Implement: Direct Pub/Sub leaf + +The per-leaf wiring for the Direct Pub/Sub leaf. Read this file from `implement-mode.md` Step 4 when the `Pattern` field is `Direct Pub/Sub`. The shared Step 4 rules in `implement-mode.md` (the AI-assisted disclaimer header, the shared `SolaceConnectionConfig` helper, the variant discipline, the demo-harness rule, and the explicit-single-class-imports rule) apply here unchanged. + +Generation reads the direct pub/sub sample pair (`jcsmp-direct-publisher-sample.java` + `jcsmp-direct-subscriber-sample.java`) and adapts each into a generated class, the same near-verbatim discipline the guaranteed flow uses. Direct messaging is at-most-once: there is no broker ACK and no redelivery, so this leaf has NO queue, NO provisioning, NO CLIENT-ack, and NO publish-ACK callback. Read the two samples by their exact relative paths and adapt each into its own generated class (a Publisher and a Subscriber), each building its own `JCSMPSession`. The named-exec rule from the guaranteed flow applies here unchanged. + +- `jcsmp-direct-publisher-sample.java`: basic-auth connect with reconnect-configured channel properties, a continuous DIRECT publish loop with a rotating payload to a dynamic topic (mirroring the guaranteed publisher's send flow), a stats thread, and a graceful SIGINT shutdown. No publish-ACK callback and no correlation key (direct has no broker ACK). Generation STRIPS the unbounded loop, stats thread, and SIGINT hook down to a short bounded burst of DIRECT messages that self-exits (Publisher class steps below), the same way the guaranteed publisher's loop is stripped, so the generated Publisher runs in the foreground and ends on its own. +- `jcsmp-direct-subscriber-sample.java`: basic-auth connect, a topic subscription confirmed via `session.addSubscription(topic, true)` (the `true` waits for broker confirm; no queue, no provisioning), an async DIRECT consumer started with `consumer.start()`, `message.getDiscardIndication()` for at-most-once egress-discard evidence in `onReceive`, and the SIGINT graceful-shutdown hook (the Subscriber is the long-running SIGINT target). + +## Subscriber class (near-verbatim `DirectSubscriber`) + +The Subscriber subscribes directly to the topic (no queue), consumes asynchronously, and is the long-running process the developer runs FIRST. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire it in this sequence: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.REAPPLY_SUBSCRIPTIONS` to `true` on those properties BEFORE `createSession` (it defaults to false): a Direct topic subscription is a client-side session subscription, not held on a broker queue, so without this it is lost on an automatic reconnect and delivery silently stops. +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Acquire the async consumer with `session.getMessageConsumer(listener)`, add the topic subscription with `session.addSubscription(topic, true)` (the `true` waits for broker confirm, so the route is live before the subscriber reports ready, since direct is at-most-once), then `consumer.start()`. Immediately after `consumer.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. There is NO queue and NO `session.provision(...)` on the direct path. +4. In the listener's `onReceive`, count the message and call `message.getDiscardIndication()` for at-most-once egress-discard evidence; immediately emit the receipt marker: `trace("VERIFY: MESSAGE_RECEIVED");`. There is NO `ackMessage()` and NO CLIENT-ack on the direct path (direct is at-most-once, the broker holds no copy and expects no acknowledgement). +5. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the sample. The Subscriber is the SIGINT target: on SIGINT it calls `session.closeSession()` directly (closing the session also closes the consumer; direct has no acknowledgements to drain), printing `Shutdown signal received` so the verify script sees the shutdown-hook proof line. + +## Publisher class (near-verbatim `DirectPublisher`) + +The Publisher only connects and publishes a short bounded burst of DIRECT messages, then exits on its own; it does NOT provision anything and does NOT wait for a key press or a signal. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire it in this sequence: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Acquire the producer with a minimal mandatory streaming-publish event handler (`session.getMessageProducer(handler)`). The handler parameter is mandatory even for direct: a `null` handler is rejected at runtime with "Blocking publishing mode is not supported". Direct messaging is at-most-once with no broker ACK, so the handler's `responseReceivedEx` is not expected to fire and `handleErrorEx` only logs the error (rendering the broker subcode when present); there is no correlation key. Do NOT pass `null`, and do NOT wait on an ACK: direct has no broker ACK to act on (ACK/NACK handling and correlation keys are the guaranteed publisher's contract). +4. Set the delivery mode EXPLICITLY with `message.setDeliveryMode(DeliveryMode.DIRECT)` and publish a SHORT BOUNDED BURST to the design topic via `producer.send(message, topic)` in a small fixed-count loop, then self-exit (close the session and return from `main`). DIRECT is the API default, but stating it makes the at-most-once intent unambiguous. REMOVE the sample's UNBOUNDED ENTER/SIGINT publish loop, its rotating-payload/dynamic-topic illustration, its stats thread, and its SIGINT shutdown hook (the sample mirrors the guaranteed publisher's send flow; here it collapses to a bounded burst): the generated Publisher publishes its burst and terminates by itself, not on a key press or a signal, so the verify script can run it in the foreground and wait for it to exit. Direct has no outstanding ACKs to drain, so no pre-close sleep is needed. The SIGINT shutdown hook belongs to the long-running Subscriber, not the Publisher. See [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md). +5. The Publisher emits NO send-side marker. Direct has no broker ACK, so there is no `PUBLISH_ACKED` analog; a marker after `producer.send` would only prove the local call returned, not that the broker accepted the message, which is misleading for an at-most-once path. The real proof of the round-trip is `VERIFY: MESSAGE_RECEIVED` on the Subscriber side. + +## The marker contract and the named exec executions + +These three `VERIFY:` marker strings for the Direct Pub/Sub leaf (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: MESSAGE_RECEIVED`) are a fixed contract with `scripts/verify.sh` (the `direct` stage greps the Subscriber log for `VERIFY: SUBSCRIBED` as its readiness gate and for `VERIFY: MESSAGE_RECEIVED` as its pass condition). The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. + +**Expose both mains to the verify script via two named exec executions.** Just like the guaranteed flow, the generated pom MUST declare two `exec-maven-plugin` `` blocks for the direct pair, one with `publisher` and one with `subscriber`, each carrying its own `` set to that class's fully-qualified name. The `direct` stage addresses them as `mvn exec:java@subscriber` (started first, the readiness gate) and `mvn exec:java@publisher` (the foreground burst). Keep the `exec-maven-plugin` version resolved at generation time. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-guaranteed-pubsub.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-guaranteed-pubsub.md new file mode 100644 index 0000000..551ca2d --- /dev/null +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-guaranteed-pubsub.md @@ -0,0 +1,73 @@ +# Implement: Guaranteed Pub/Sub leaves + +The per-leaf wiring for the three Guaranteed Pub/Sub leaves. Read this file from `implement-mode.md` Step 4 when the `Pattern` field is `Guaranteed Pub/Sub (single service, exclusive)`, `Guaranteed Pub/Sub (single service, non-exclusive)`, or `Guaranteed Pub/Sub (fan-out)`. The shared Step 4 rules in `implement-mode.md` (the AI-assisted disclaimer header, the shared `SolaceConnectionConfig` helper, the variant discipline, the demo-harness rule, and the explicit-single-class-imports rule) apply here unchanged. + +## Guaranteed pub/sub flow + +The skill ships two best-practices reference samples for this flow. Read them by their exact relative paths and adapt each into its own generated class. There is no merge and no third skeleton: the Publisher class is a near-verbatim adaptation of the publisher sample, and the Subscriber class is a near-verbatim adaptation of the consumer sample. Generating two near-verbatim classes is more reliable than synthesizing a single program from both, because the source shape is copied, not invented. + +- `jcsmp-guaranteed-publisher-sample.java`: basic-auth connect, PERSISTENT BytesMessage publish to a topic, publish ACK/NACK handler, session-event and reconnect-event handling that pauses publishing while the transport is down, SIGINT graceful shutdown that finishes outstanding ACKs. +- `jcsmp-guaranteed-subscriber-sample.java`: basic-auth connect, in-process durable-queue provisioning plus topic subscription, CLIENT-ack flow that ACKs only after processing, session-event and flow-event handling. + +Each class builds its OWN `JCSMPSession` (two sessions, one per class, matching the reference samples; there is no shared-session plumbing). Preserve the session-event and reconnect-event and publish ACK/NACK handlers in the Publisher, and the session-event and flow-event handlers in the Subscriber; they are part of the best-practices contract and must not be stripped during adaptation. + +The three guaranteed pub/sub leaves all generate the same two-class Publisher and Subscriber described below. The exclusive versus non-exclusive choice is realized inside the flow as a `setAccessType` snippet (read the `Access type` summary field), not a separate sample; the fan-out leaf reuses the single-service generator, repeating the queue-and-subscription setup once per consuming service. + +## Subscriber class (near-verbatim `GuaranteedSubscriber`) + +The Subscriber provisions the durable queue and the topic subscription, and it is the long-running process the developer runs FIRST. The Publisher does NOT provision anything. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire the Subscriber in this canonical sequence, each step grounded in the docs linked in `implement-mode.md`'s Grounding references: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.IGNORE_DUPLICATE_SUBSCRIPTION_ERROR` to `true` BEFORE `createSession` (it defaults to false); otherwise `addSubscription` throws Subscription Already Exists on every run after the first. +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the consumer sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Best practice: check `session.isCapable(CapabilityType.ENDPOINT_MANAGEMENT)` before provisioning; exit cleanly if the broker disallows client-side endpoint management. +4. `JCSMPFactory.onlyInstance().createQueue(queueName)` for the queue from the design summary. +5. Build the `EndpointProperties` and set the queue access type from the `Access type` summary field (see the access-type snippet below), then `session.provision(queue, endpointProps, JCSMPSession.FLAG_IGNORE_ALREADY_EXISTS)` so a re-run is idempotent. See [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md). +6. `session.addSubscription(queue, topic, JCSMPSession.WAIT_FOR_CONFIRM)` to map the topic onto the queue. See [Adding a Topic Subscription](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Adding-Topic-Subscriptio.md). +7. Build a `ConsumerFlowProperties` flow with `setAckMode(JCSMPProperties.SUPPORTED_MESSAGE_ACK_CLIENT)`, create it with the consumer sample's `FlowEventHandler`, and `flow.start()`. Immediately after `flow.start()` returns, emit the queue-bound marker: `trace("VERIFY: QUEUE_BOUND");`. See [Acknowledging Messages](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Acknowledging-Messages.md). +8. In the flow's `onReceive`, call `msg.ackMessage()` only after processing is complete; immediately after `ackMessage()`, emit the round-trip marker: `trace("VERIFY: MESSAGE_RECEIVED");`. +9. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the consumer sample. The Subscriber is the SIGINT target: on SIGINT it stops the flow, finishes outstanding ACKs, and calls `session.closeSession()`. + +**Access type (exclusive vs non-exclusive): a `setAccessType` snippet, not a separate sample.** The `Access type` field in the design summary picks one line. Set the queue access type on the `EndpointProperties` before provisioning (item 5 above): + +```java +EndpointProperties endpointProps = new EndpointProperties(); +endpointProps.setAccessType(EndpointProperties.ACCESSTYPE_EXCLUSIVE); // one active consumer / ordered failover +// endpointProps.setAccessType(EndpointProperties.ACCESSTYPE_NONEXCLUSIVE); // competing consumers / round-robin load balance +endpointProps.setPermission(EndpointProperties.PERMISSION_CONSUME); +``` + +When the `Access type` field reads exclusive, keep the `ACCESSTYPE_EXCLUSIVE` line: only one active consumer receives at a time, and on its disconnect a standby flow takes over in order (ordered failover). When it reads non-exclusive, use the `ACCESSTYPE_NONEXCLUSIVE` line instead: bound flows compete and receive in round-robin, so the load is shared across workers. Read the `Access type` field and choose the one line; do not emit both active. This is grounded in [Receiving Guaranteed Messages](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Receiving-Guaranteed-Messages.md), which states an exclusive queue delivers in order with one active consumer and failover to the next flow, and a non-exclusive queue delivers round-robin across bound flows. + +## Publisher class (near-verbatim `GuaranteedPublisher`) + +The Publisher only connects and publishes; it does NOT provision the queue or the subscription (the Subscriber owns that, and starts first). It must exit on its own after the ack so the developer does not have to interrupt it. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire the Publisher in this canonical sequence: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the publisher sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Acquire the `XMLMessageProducer` with the publisher sample's publish ACK/NACK callback handler (`PublishCallbackHandler`), then register the publisher sample's `JCSMPReconnectEventHandler` through an empty never-started consumer (`session.getMessageConsumer(reconnectHandler, null)`). The docs ground this idiom for publish-only applications; keep it because it surfaces transport events that pause the send loop. In the ACK callback (`responseReceivedEx`), emit the publish-ACK marker: `trace("VERIFY: PUBLISH_ACKED");`. CRITICAL: the reference publisher sample logs the ACK at `logger.debug`, which is invisible in default output, so the verify script would never see it and would report a false code failure. The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. +4. Publish a PERSISTENT `BytesMessage` to the topic (`DeliveryMode.PERSISTENT`). See [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md). +5. EXIT on its own: after observing the ACK (the `VERIFY: PUBLISH_ACKED` marker), drain outstanding ACKs (the sample's `Thread.sleep(1500)` idiom), call `session.closeSession()`, and return from `main`. REMOVE the sample's ENTER/SIGINT loop: the Publisher must terminate by itself, not wait for a key press or a signal. The SIGINT shutdown hook belongs to the Subscriber, not the Publisher. + +## The marker contract and the named exec executions + +These four `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: PUBLISH_ACKED`, `VERIFY: QUEUE_BOUND`, `VERIFY: MESSAGE_RECEIVED`) are a fixed contract with `scripts/verify.sh`, which greps captured output PER PROCESS (the Subscriber log for `QUEUE_BOUND` and `MESSAGE_RECEIVED`, the Publisher log for `PUBLISH_ACKED`). Both classes connect, so both print `VERIFY: CONNECTED`; the script reads markers per-process for that reason. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. + +**Expose both mains to the verify script via two named exec executions.** The fixed `scripts/verify.sh` runs each main without knowing the per-generation package, so the generated pom MUST declare two `exec-maven-plugin` `` blocks, one with `publisher` and one with `subscriber`, each carrying its own `` set to that class's fully-qualified name. The script then addresses them as `mvn exec:java@publisher` and `mvn exec:java@subscriber`. The script references the fixed execution ids, never the generated FQCN, so the package and class names stay your discretion at generation time. Keep the `exec-maven-plugin` version resolved at generation time (no hardcoded version), the same way the `sol-jcsmp` coordinate is resolved. + +```xml + + org.codehaus.mojo + exec-maven-plugin + RESOLVED_AT_GENERATION_TIME + + + publisher + com.example.app.OrderPublisher + + + subscriber + com.example.app.OrderSubscriber + + + +``` diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md index 616e2a7..168a827 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md @@ -1,42 +1,63 @@ # Implement Mode -Generate a complete, runnable Maven JCSMP publish-subscribe project from a confirmed design summary. The generated output is TWO classes (a Publisher and a Subscriber) in ONE Maven project, the canonical solace-samples shape (one project, multiple mains). Each class is a near-verbatim adaptation of its reference sample. The Subscriber connects with basic auth, provisions its own durable queue, subscribes that queue to the topic, and consumes in a CLIENT-ack flow. The Publisher connects with basic auth and publishes a PERSISTENT message to the topic. Run the Subscriber first (it provisions the queue), then the Publisher, for a guaranteed pub/sub round-trip. Near-verbatim adaptation of two samples is more reliable than merging them into one program, and a real deployment lifts each class into its own service or project (see `verification-checklist.md`; Quickstart does not generate the decoupled two-project form). +Generate a complete, runnable Maven JCSMP project from a confirmed design summary. The canonical output is TWO classes (a Publisher and a Subscriber, or a Requestor and a Replier) in ONE Maven project, the canonical solace-samples shape (one project, multiple mains). Each class is a near-verbatim adaptation of its reference sample. Near-verbatim adaptation of two samples is more reliable than merging them into one program, and a real deployment lifts each class into its own service or project (see `verification-checklist.md`; Quickstart does not generate the decoupled two-project form). -Work through the steps in order. Read this file's grounding links only as each step needs them. Link the canonical doc and state the decision in one line, never paraphrase doc content. Resolve the `sol-jcsmp` version at generation time and pin no `sol-jcsmp` version in this skill (the Solace API coordinate stays fresh against Maven Central). log4j2 is the one deliberate exception: it is pinned to a known-good 2.x version, as Step 3 explains. +Implement mode is NOT only for the canonical shape. A web app, a dashboard, a service, or any embedded shape where the JCSMP layer lives inside a larger application follows the SAME steps: the same door question, the same leaf wiring for the messaging layer, the same generation outputs, and the same verification (the `app` stage of `verify.sh`). Building an off-catalog shape never waives a step in this file. + +Work through the steps in order. Read this file's grounding links only as each step needs them. Link the canonical doc and state the decision in one line, never paraphrase doc content. Resolve every dependency version at generation time; pin nothing (SKILL.md Invariant 3). Step 3 defines the logging backend's resolution and its security floor. + +## Generation outputs: the file contract + +Every Implement run writes ALL of these as generation output, on every door and every app shape. None of them is gated on consent, and none of them waits for the end of the session: + +1. The generated classes for the chosen leaf, plus the shared `SolaceConnectionConfig` helper. Every source file starts with the AI-assisted disclaimer header (Step 4). +2. `pom.xml` (or the leaf poms on a decoupled layout) with versions resolved per Step 3. +3. The logging config resource for the resolved backend (Step 3). +4. On Quickstart: `config.example.json` (placeholders) and a `.gitignore` that ignores `config.json` (Step 4). +5. A tailored `solace-verification-checklist.md` at each project root (tailoring rules in Step 6; the WRITE happens here at generation time, like the pom and the classes). +6. A copy of the bundled `scripts/verify.sh` at the project root, plus a generated `verify-hooks.sh` beside it (Step 5), so the verification commands resolve from inside the project. ## Step 0: Ask Quickstart, Solace Suggested, or Custom, before any broker details This is the FIRST question, and you ask it before you request or accept ANY broker connection details. Ask the developer plainly which of three doors they want: Quickstart, Solace Suggested, or Custom. Ask it as a direct question and wait for the answer. Frame the three doors honestly: Quickstart is a learning and try-it path to see a round-trip work, NOT a deployment target; Solace Suggested is the hardened baseline Solace suggests as a starting point, and Solace suggests it without guaranteeing it is ready for deployment; Custom lets the developer tick exactly which hardening knobs to include and generates that ticked subset and nothing more. Nothing has been generated yet and no credentials have been exchanged yet, so this is the up-front gate. The answer decides WHERE broker connection details belong, and it drives behavior for the rest of Implement mode. Every door generates the same classes (a Publisher and a Subscriber, plus the shared `SolaceConnectionConfig` helper that builds the connection `JCSMPProperties`). They differ in where the credential VALUES live: Solace Suggested puts them ONLY in a gitignored `config.json`, while Quickstart reads `config.json` when present and otherwise falls back to CLI args; Custom follows the Quickstart-shaped `config.json`-or-CLI convention by default, and `custom-mode.md` covers the decoupled-projects case where each project carries its own gitignored `config.json`. -- **Quickstart (default).** Each class builds its connection through the shared `SolaceConnectionConfig` helper, which reads host, message VPN, username, and password from a `config.json` in the working directory when present and otherwise falls back to CLI args (the upstream `patterns/` samples' CLI-args style). Simplest path to a running round-trip, framed honestly as a learning path rather than a deployment target. Generate the stages without per-stage preview gates. Quickstart output fails fast: generation omits the samples' reconnect-tuning block so connect and reconnect settings stay at the JCSMP defaults (the fail-fast rule in Step 4). -- **Solace Suggested.** On the Solace Suggested path, read `solace-suggested-mode.md` and follow the Solace Suggested overlay (TLS, DMQ on PERSISTENT, HA failover, separate projects, and the `config.json` convention); the leaf and sample mechanics, the Step 4 dispatch, and the four-file `VERIFY:` contract stay here in implement-mode.md. When the developer picks Solace Suggested, ask a second question right here in Step 0: do they want Secure or Non-Secure? Secure is the TLS `tcps://` path for a broker that already has TLS configured, and it validates the server certificate. Non-Secure applies the same Solace Suggested hardening over a plaintext `tcp://` session for a dev or test broker that has no TLS configured, so there is no certificate to validate; state plainly that Non-Secure uses unencrypted transport and fits a dev or test broker, not real credentials or real data over untrusted networks. `solace-suggested-mode.md` reads that Secure or Non-Secure flag and forks only at its session build; every other part of the overlay is identical on both sub-modes. The overlay is leaf-agnostic: it rides on whatever `Pattern` leaf the design summary chose and only adds the Solace Suggested deltas. It is what Solace suggests as a hardened baseline, not a guarantee that the output is ready for deployment. -- **Custom.** On the Custom path, read `custom-mode.md` and follow it. Custom opens a SINGLE up-front multi-select checklist of the applicable hardening knobs, captured once, not a sequence of per-knob questions; generation then applies exactly the ticked subset and nothing more. Nothing ticked lands a Quickstart-shaped base app, everything ticked lands about the Solace Suggested Secure shape, and each ticked knob layers one Suggested delta on top of that floor. `custom-mode.md` owns the leaf-aware checklist content and the knob application; the leaf and sample mechanics, the Step 4 dispatch, and the four-file `VERIFY:` contract stay here in implement-mode.md, exactly as they do on the Solace Suggested path. +Three things do NOT answer this question, and none of them waives it: + +- **An existing `config.json`**, even one that already holds real credentials. It tells you where credentials sit today; it does not tell you which door the developer wants. Ask anyway. +- **A running local broker discovered in the environment.** A discovered container or endpoint is a fact to report, never an answer to consume, neither for the door nor for WHICH broker to target. Report what you found, then ask both questions. +- **An app shape larger than the generated two-class project** (a web app, a dashboard, a service). The doors still govern the messaging layer and the credential placement. Ask anyway. + +- **Quickstart (default).** Each class builds its connection through the shared `SolaceConnectionConfig` helper, which reads host, message VPN, username, and password from a `config.json` in the working directory when present and otherwise falls back to CLI args (the upstream `patterns/` samples' CLI-args style). Simplest path to a running round-trip, framed honestly as a learning path rather than a deployment target. Quickstart adds no TLS hardening and has no TLS requirement: it connects with whatever scheme the developer's host carries, and a plaintext `tcp://` dev broker is a normal Quickstart target. The Solace Suggested door's TLS requirement never applies here; do not refuse a `tcp://` host on Quickstart. Generate the stages without per-stage preview gates. Quickstart output fails fast: generation omits the samples' reconnect-tuning block so connect and reconnect settings stay at the JCSMP defaults (the fail-fast rule in Step 4). +- **Solace Suggested.** On the Solace Suggested path, read `solace-suggested-mode.md` and follow the Solace Suggested overlay (TLS, DMQ on PERSISTENT, HA failover, separate projects, and the `config.json` convention); the leaf and sample mechanics, the Step 4 dispatch, and the fixed `VERIFY:` contract stay here in implement-mode.md and its leaf files. Solace Suggested is TLS-secure by definition: the session is `tcps://` with server-certificate validation, and it REQUIRES a broker that already has TLS configured (Solace Cloud has it out of the box). There is no non-secure variant of this door and no sub-question to ask; the door question stays a clean three-way choice. When the developer's target broker has no TLS configured (for example, a local dev container), say plainly that Solace Suggested requires a TLS-capable broker and offer the two honest alternatives: Custom with the equivalent knobs ticked minus the secure session (the same hardening, as the developer's explicit choice), or Quickstart for a learning run. Never generate a plaintext session under the Solace Suggested name. The overlay is leaf-agnostic: it rides on whatever `Pattern` leaf the design summary chose and only adds the Solace Suggested deltas. It is what Solace suggests as a hardened baseline, not a guarantee that the output is ready for deployment. +- **Custom.** On the Custom path, read `custom-mode.md` and follow it. Custom opens a SINGLE up-front multi-select checklist of the applicable hardening knobs, captured once, not a sequence of per-knob questions; generation then applies exactly the ticked subset and nothing more. Nothing ticked lands a Quickstart-shaped base app, everything ticked lands about the Solace Suggested shape, and each ticked knob layers one Suggested delta on top of that floor. `custom-mode.md` owns the leaf-aware checklist content and the knob application; the leaf and sample mechanics, the Step 4 dispatch, and the fixed `VERIFY:` contract stay here in implement-mode.md and its leaf files, exactly as they do on the Solace Suggested path. State plainly, grounded in the docs, that a real deployment lifts each class into its own service or project. Every door generates the same two classes in one project, unless the developer opts into decoupled projects on Solace Suggested or Custom; lifting each into its own project is the developer's next step beyond this generated project. See [JCSMP Best Practices](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Best-Practices.md). -Confirm which door the developer wants before continuing, and on the Solace Suggested door confirm the Secure or Non-Secure sub-choice too. Basic username/password is the only auth this journey generates; ground it in [Defining Client Authentication](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Defining-Client-Authentication.md). On the Solace Suggested and Custom paths, the overlay file carries the `config.json` convention; no door gates code generation behind a per-stage confirmation. +Confirm which door the developer wants before continuing. Basic username/password is the only auth this journey generates; ground it in [Defining Client Authentication](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Defining-Client-Authentication.md). On the Solace Suggested and Custom paths, the overlay file carries the `config.json` convention; no door gates code generation behind a per-stage confirmation. ## Step 1: Confirm broker access and grounding -With the door chosen, confirm two things (skip whichever the developer already has): +With the door chosen, confirm the developer has access to a reachable broker, and confirm WHICH broker they want to target. Confirm that they HAVE one; do NOT ask them to paste connection details into the chat. When Design mode already confirmed broker access in this session (its Step 0), do not re-ask whether a broker exists; ask only WHICH broker to target. A broker discovered in the environment (a running container, an existing config file) is a fact to report while asking, never an answer. Where the credential VALUES live depends on the Step 0 door: for Solace Suggested they go ONLY into the gitignored `config.json`; for Quickstart they reach the app through a `config.json` when present, otherwise as CLI args at run time. The skill itself never needs the values typed into the chat (see the honesty rule in Step 5: if the values are placeholders or withheld, compile only and hand the developer the exact commands to run against their own broker). -- The developer has access to a reachable broker. Confirm that they HAVE one; do NOT ask them to paste connection details into the chat. Where the credential VALUES live depends on the Step 0 door: for Solace Suggested they go ONLY into the gitignored `config.json`; for Quickstart they reach the app through a `config.json` when present, otherwise as CLI args at run time. The skill itself never needs the values typed into the chat (see the honesty rule in Step 5: if the values are placeholders or withheld, compile only and hand the developer the exact commands to run against their own broker). -- The developer understands the basic Solace model (broker, topics, queues, publish/subscribe). +If the developer has no reachable broker, route through `prerequisites.md` first (broker acquisition, Solace Cloud recommended), then return here. -If either is missing, route through `prerequisites.md` first (Core Concepts grounding and broker acquisition, Solace Cloud recommended), then return here. +Grounding catch-up: if the design summary names a page in its Grounding docs field that this session has not WebFetched, fetch it now before generating against it (SKILL.md Invariant 2; an unfetched citation is not grounding). ## Step 2: Establish the design as the input contract -Implement mode works from the unified design summary of eight fields: Pattern, Delivery, Access type, Topic, Consumption endpoint, Auth, Broker, Grounding docs. The `Pattern` field is the discriminated leaf string (one of the six Design-mode leaves), `Consumption endpoint` carries what used to be the fixed `Queue` field (for the guaranteed pub/sub journey this resolves to a durable queue plus topic subscription), and `Access type` is surfaced explicitly. The summary can come from Design mode, but it does not have to: an experienced developer who already has a design can supply the fields directly. +Implement mode works from the unified design summary of eight fields: Pattern, Delivery, Access type, Topic, Consumption endpoint, Auth, Broker, Grounding docs. The `Pattern` field is the discriminated leaf string (one of the six Design-mode leaves), `Consumption endpoint` carries what used to be the fixed `Queue` field (for the guaranteed pub/sub journey this resolves to a durable queue plus topic subscription), and `Access type` is surfaced explicitly. + +The summary is a HARD precondition (the design-contract gate in `jcsmp.md`): Implement mode never proceeds past this step without one, and never invents design values. Exactly three sources satisfy it: + +- A summary Design mode confirmed in this session: use it as-is, with no re-confirm. +- An explicit summary the developer supplied in chat (the eight fields or an equivalent statement of them): use it directly. If it omits a derivable field, derive it rather than asking (the topic via the solace-topic-best-practices skill, the queue name via the `q.`-prefixed convention), the same way Design mode does; only the Pattern and payload genuinely need the developer's input. An experienced developer is not required to walk the design tree; supplying the summary satisfies the contract. +- A saved `solace-design.md` in the developer's project: read it, restate it in one line, and proceed on their confirm. -- If a summary is already in the chat, use it as-is. -- If the developer saved one, read `solace-design.md` from their project. -- If no summary exists but the developer already knows their design, take the values directly from them and proceed (at minimum: the Pattern, the topic string, the `Consumption endpoint` the pattern requires (the queue name for the guaranteed pub/sub journey, `n/a` for patterns with no queue), and the message payload; Delivery and Auth default to PERSISTENT guaranteed and basic username/password for the guaranteed pub/sub journey). An experienced developer is NOT required to run Design mode or produce a `solace-design.md` first. If the developer does not supply a topic or a queue name, derive them rather than asking (the topic via the solace-topic-best-practices skill, the queue name via the `q.`-prefixed convention), the same way Design mode does; only the Pattern and payload genuinely need the developer's input. -- Only route the developer to Design mode (`design-mode.md`) if they want help choosing the pattern/topology, or if the essential inputs above are missing and they cannot supply them. Do not invent design decisions; ask for the missing values or offer Design mode. +If none of the three exists, do NOT collect the fields piecemeal here and do NOT proceed: route through Design mode (`design-mode.md`) and return with its confirmed summary. This holds even when the request sounds like a build. A prose build request that mentions pattern details is not a contract; Design mode's fully-specified path resolves it in ONE confirm (derive the summary, echo it, confirm). The only carve-out is the topology rule in `jcsmp.md`: a mechanical edit to an existing app (no topology change) proceeds without a summary, while a topology-changing edit re-enters Design mode to re-confirm the affected fields first. -Treat the confirmed values (Pattern, topic string, `Consumption endpoint`, payload) as the requirements for everything that follows. The discriminated `Pattern` field carries the developer's chosen leaf, and Step 4 dispatches generation onto that leaf: it reads only the sample pair the chosen leaf needs. Read the design as the input contract and do not reject a summary because its `Pattern` is not pub/sub. +Treat the confirmed values (Pattern, topic string, `Consumption endpoint`, payload) as the requirements for everything that follows. The discriminated `Pattern` field carries the developer's chosen leaf, and Step 4 dispatches generation onto that leaf: it reads only the leaf file and sample pair the chosen leaf needs. Read the design as the input contract and do not reject a summary because its `Pattern` is not pub/sub. -## Step 3: Resolve the version, then write the pom and Maven layout +## Step 3: Resolve the versions, then write the pom and Maven layout ### Resolve the latest sol-jcsmp version at generation time @@ -57,10 +78,14 @@ Write `pom.xml` and the standard `src/main/java//` layout yourself, gro - **Single Solace dependency: `com.solacesystems:sol-jcsmp`.** Use the groupId `com.solacesystems` exactly. The Maven doc shows the groupId as `com.solace` (without the `systems` suffix), which is WRONG and does not resolve on Maven Central; always use `com.solacesystems`. Confirm the artifact at [Maven Central](https://central.sonatype.com/artifact/com.solacesystems/sol-jcsmp). - **No `solsuite` aggregate pom.** It was dropped at 10.29; depend on `sol-jcsmp` directly. -- **Logging: log4j2 `2.x`, two artifacts: `org.apache.logging.log4j:log4j-api` and `org.apache.logging.log4j:log4j-core`.** Both reference samples import log4j2 (`LogManager`, `Logger`), so neither generated class compiles without these artifacts. Also generate a `log4j2.xml` configuration resource at `src/main/resources/log4j2.xml` so it lands on the classpath (both samples carry the reminder "remember to add log4j2.xml to your classpath"). Pin both `log4j-api` and `log4j-core` to the same specific known-good 2.x version: `2.26.0`. Unlike `sol-jcsmp`, log4j2 is NOT resolved live from Maven Central metadata; the pin gives the developer a reproducible, vetted logging dependency. This matters for security: log4j2 `2.x` versions below `2.17.1` carry the critical Log4Shell vulnerabilities (CVE-2021-44228 family), so the pinned version MUST be at or above `2.17.1` (`2.26.0` satisfies that floor). Do not pull the log4j2 `3.x` beta; the live `` element currently reports a 3.x beta, which is exactly why a pin is safer here than live resolution. Updating the pin to a newer 2.x later is a deliberate maintenance action, not a silent live resolution on every generation. +- **Logging: one working backend, resolved like every other dependency; never pinned; never silenced.** The reference samples log through log4j2 (`LogManager`, `Logger`), so the DEFAULT backend is log4j2 (`org.apache.logging.log4j:log4j-api` plus `log4j-core`), and neither generated class compiles without a backend wired. Four outcome rules, in force on every door and every app shape: + 1. **Resolve, do not pin.** Resolve the backend's version at generation time from the authoritative Maven metadata, the same `maven-metadata.xml` mechanism `sol-jcsmp` uses. One caution: the log4j metadata's `` element can report a pre-release of the next major (a 3.x alpha or beta). The samples target the log4j2 2.x API, so resolve the HIGHEST STABLE 2.x from the metadata's `` list instead, excluding any alpha, beta, or RC: `curl -s https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/maven-metadata.xml | grep -oE '2[^<]*' | grep -viE 'alpha|beta|rc' | tail -1`. + 2. **Security floor, enforced as a check.** Any `log4j-core` on the classpath, including one dragged in transitively, MUST be at or above `2.17.1`: versions below it carry the critical Log4Shell vulnerabilities (CVE-2021-44228 family). The `verify.sh` preflight and the verification checklist test this floor; it is a check on the resolved result, never a pin. + 3. **A framework's own backend wins.** When the app is built on a framework that ships its own SLF4J backend (for example Spring Boot ships Logback), use the framework's backend instead of adding log4j2. NEVER put two logging backends on one classpath. Adapt the samples' log4j2 (`LogManager`/`Logger`) usage to the SLF4J API in that case, and hold the same floor discipline for that backend's own CVE floors. + 4. **Config file, visible Solace loggers.** Whatever the backend, generate its configuration resource on the classpath (`src/main/resources/log4j2.xml` for log4j2; the framework's equivalent otherwise) with a console appender, so diagnostics reach stdout. NEVER silence the Solace API's own logging: the `com.solacesystems` loggers stay at INFO or lower-threshold, because they carry the connect and reconnect diagnostics the developer needs most when something breaks. - **Java target:** Java 11 by default. It is the minimum the samples support, and they compile on 11 and up. Set `maven-compiler-plugin` to `release` 11; use a higher release only if the developer's environment requires it. groupId, artifactId, and package naming are your discretion. -Keep this skill `sol-jcsmp`-version-free. In any illustrative pom snippet, write the `sol-jcsmp` dependency version as a placeholder, never a concrete number (log4j2 is the deliberate exception, pinned to `2.26.0` as described above): +Keep this skill version-free. In any illustrative pom snippet, write every dependency version as a placeholder, never a concrete number: ```xml @@ -72,20 +97,24 @@ Keep this skill `sol-jcsmp`-version-free. In any illustrative pom snippet, write ## Step 4: Dispatch on the Pattern leaf, then generate the classes -Generation is keyed on the `Pattern` leaf read in Step 2. Each leaf reads its own sample pair plus the shared `jcsmp-solace-connection-config.java` helper; do not read samples for a leaf the design did not choose. The six discriminated leaf strings (spelled exactly as Design mode emits them) dispatch as follows: +Generation is keyed on the `Pattern` leaf read in Step 2. Each leaf has its own wiring file, which names its sample pair; read ONLY the leaf file the design chose, and do not read samples for a leaf the design did not choose. The six discriminated leaf strings (spelled exactly as Design mode emits them) dispatch as follows: -| `Pattern` leaf (verbatim) | Sample pair to read | Status | -|---------------------------|---------------------|--------| -| `Guaranteed Pub/Sub (single service, exclusive)` | `jcsmp-guaranteed-publisher-sample.java` + `jcsmp-guaranteed-subscriber-sample.java` | Generated here (the guaranteed pub/sub flow below) | -| `Guaranteed Pub/Sub (single service, non-exclusive)` | `jcsmp-guaranteed-publisher-sample.java` + `jcsmp-guaranteed-subscriber-sample.java` | Generated here (the guaranteed pub/sub flow below) | -| `Guaranteed Pub/Sub (fan-out)` | `jcsmp-guaranteed-publisher-sample.java` + `jcsmp-guaranteed-subscriber-sample.java` | Generated here (the guaranteed pub/sub flow below) | -| `Direct Pub/Sub` | `jcsmp-direct-publisher-sample.java` + `jcsmp-direct-subscriber-sample.java` | Generated here (the Direct Pub/Sub leaf below) | -| `Request-Reply (Direct)` | `jcsmp-direct-requestor-sample.java` + `jcsmp-direct-replier-sample.java` | Generated here (the Request-Reply (Direct) leaf below) | -| `Request-Reply (Guaranteed)` | `jcsmp-guaranteed-requestor-sample.java` + `jcsmp-guaranteed-replier-sample.java` | Generated here (the Request-Reply (Guaranteed) leaf below) | +| `Pattern` leaf (verbatim) | Leaf wiring file | Sample pair it reads | +|---------------------------|------------------|----------------------| +| `Guaranteed Pub/Sub (single service, exclusive)` | `implement-guaranteed-pubsub.md` | `jcsmp-guaranteed-publisher-sample.java` + `jcsmp-guaranteed-subscriber-sample.java` | +| `Guaranteed Pub/Sub (single service, non-exclusive)` | `implement-guaranteed-pubsub.md` | `jcsmp-guaranteed-publisher-sample.java` + `jcsmp-guaranteed-subscriber-sample.java` | +| `Guaranteed Pub/Sub (fan-out)` | `implement-guaranteed-pubsub.md` | `jcsmp-guaranteed-publisher-sample.java` + `jcsmp-guaranteed-subscriber-sample.java` | +| `Direct Pub/Sub` | `implement-direct-pubsub.md` | `jcsmp-direct-publisher-sample.java` + `jcsmp-direct-subscriber-sample.java` | +| `Request-Reply (Direct)` | `implement-request-reply.md` | `jcsmp-direct-requestor-sample.java` + `jcsmp-direct-replier-sample.java` | +| `Request-Reply (Guaranteed)` | `implement-request-reply.md` | `jcsmp-guaranteed-requestor-sample.java` + `jcsmp-guaranteed-replier-sample.java` | **Every leaf also generates the shared connection helper.** In addition to its pattern pair, each leaf reads `jcsmp-solace-connection-config.java` and generates it as a `SolaceConnectionConfig` class in the project. Each pattern class builds its basic-auth connection `JCSMPProperties` by calling `SolaceConnectionConfig.load(args, APP_NAME).toSessionProperties()` rather than setting `host`/`vpn_name`/`username`/`password` inline. The helper passes every flat string key in `config.json` through to `JCSMPProperties.setProperty` (host, vpn_name, and username are required; password is optional) and otherwise falls back to the CLI args, so it adds NO JSON-library dependency and keeps the single `sol-jcsmp` dependency intact. Like every generated class it carries the AI-assisted disclaimer header and keeps its imports explicit. -The three guaranteed pub/sub leaves all generate the same two-class Publisher and Subscriber described in the guaranteed pub/sub flow below. The exclusive versus non-exclusive choice is realized inside that flow as a `setAccessType` snippet (read the `Access type` summary field), not a separate sample; the fan-out leaf reuses the single-service generator, repeating the queue-and-subscription setup once per consuming service. +**Disclaimer header on EVERY generated source file.** Stamp the top of each generated source file (every class, in every shape; the pom is exempt as it is not source) with this exact line: `AI-assisted code. Review before production use.` Directly below it, add a pointer to the tailored verification checklist this generation writes into the project: `See the verification checklist: solace-verification-checklist.md`. That tailored `solace-verification-checklist.md` ships inside the generated project (this Step writes it as generation output), so the disclaimer points each generated file at the local checklist beside it rather than at a remote copy. Stamp the header AT WRITE TIME, as the first content of each new file, not as a later pass. The reference samples themselves carry NO disclaimer; that rule is the inverse. This rule is SKILL.md Invariant 4 and applies to every leaf's generated classes AND to custom or embedded builds that adapt the samples outside the canonical two-class shape. + +**Embedded and web-app shapes.** When the JCSMP layer sits inside a larger application (a web dashboard, a REST service, a framework app), the chosen leaf's wiring still governs the messaging layer: the same connection helper, the same session/flow/reconnect/ACK handlers, the same `VERIFY:` markers emitted through a stdout-reaching `trace(...)`, and the disclaimer header on every generated file, messaging or not. The `VERIFY:` markers are NOT demo harness; they stay in the messaging layer whatever the shape, because the `app` verify stage (Step 5) greps for them in the app's captured output. + +**Write the verification artifacts in this same step.** Alongside the classes and the pom, write: the tailored `solace-verification-checklist.md` at each project root (tailoring rules in Step 6; the write is generation output with NO consent gate), a copy of the bundled `scripts/verify.sh` at the project root, and the generated `verify-hooks.sh` (Step 5 defines its contract). A session that ends early still leaves the checklist and the verify entry points in the project. ### Adapting a sample to a variant of its leaf @@ -93,268 +122,131 @@ The samples are grounded REFERENCE for the API idioms, not templates that must b Hold every invariant while adapting: keep the shared connection helper, the session, flow, and reconnect handlers, the publish ACK/NACK handler, explicit single-class imports, the disclaimer header, and the leaf's `VERIFY:` markers. Any NEW named entity the variant introduces (an async reply listener, a durable reply endpoint, a redelivery setting) must be traceable to a canonical Solace source per Invariant 1: WebFetch the grounding page for it, the same way the base leaf grounds its idioms. A verified variant is fully first-class: it carries only the standard AI-assisted disclaimer, with no extra "hand-built" or "off-spec" caveat that would read as risky freelancing. -**Guardrail: variant versus off-catalog.** Adapt freely WITHIN the chosen pattern family when the change is a topology or reliability knob that the canonical docs ground and that still compiles, emits the leaf's `VERIFY:` markers, and is verifiable by the matching `verify.sh` stage (durable versus temporary reply queue, an async versus blocking reply listener, exclusive versus non-exclusive access, redelivery handling). If a requirement leaves the JCSMP topic/queue publish and subscribe model, or needs behavior the canonical docs cannot ground, STOP and tell the developer it is off-catalog rather than inventing it. - -**Preserve the marker contract.** A variant adapts the SHAPE, never the marker strings. It MUST still emit its leaf's `VERIFY:` markers, character for character, so the fixed `verify.sh` stage drives it unchanged (this is exactly why a durable-reply-queue requestor still prints `VERIFY: REPLY_RECEIVED`). Never rename or drop a marker to fit a variant. - -**Demo harness is not application logic.** The samples carry demo-harness elements that exist only to make a standalone run observable: the ENTER-to-quit `System.in` loop, the once-per-second stats-printing thread, the pacing `Thread.sleep` between sends, and the rotating example payload. When the developer's request describes a real application domain, do NOT carry these into the generated classes: replace the example payload and send cadence with the application's real messages and triggers, and keep only the lifecycle pieces the leaf's steps call for (the SIGINT hook on the long-running role, the self-exit on the foreground role). When the request IS a demo or a try-it run, keeping the harness is fine. Two things are NOT demo harness: the `trace(...)` narration method (keep it; applications replace its single body to route narration to their logging or reporting system) and the `VERIFY:` markers, which are emitted through `trace(...)` like every other status line. Because `verify.sh` greps captured stdout for the markers and for the shutdown hook's `Shutdown signal received` proof line (both flow through `trace(...)`), the generated `trace(...)` body MUST stay `System.out.println` through the Step 5 verification stages; rerouting that body to a logger or reporting system is the developer's step AFTER the verify stages pass, never part of generation. - -### Quickstart fail-fast channel defaults - -On the Quickstart door ONLY, do not carry the reference samples' reconnect-tuning block into the generated classes. The samples configure a `JCSMPChannelProperties` reconnect budget (`reconnectRetries` 20, `connectRetriesPerHost` 3); Quickstart generation OMITS that whole block (the `JCSMPChannelProperties` object and the `CLIENT_CHANNEL_PROPERTIES` setProperty call), leaving connect and reconnect retries and their timeouts at the JCSMP defaults, so a wrong host, port, or credential in this learning setup fails immediately instead of retrying for minutes. Drop ONE short comment at the session-properties build site in each generated class stating that posture: connect and reconnect settings are at the JCSMP defaults so failures surface immediately; see the [JCSMP Best Practices](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Best-Practices.md) page for reconnect tuning beyond Quickstart. The Solace Suggested door is unchanged: it keeps the samples' baseline or upgrades it per `solace-suggested-mode.md` Step 2. Custom follows its floor exception in `custom-mode.md`. - -### Guaranteed pub/sub flow (the three Guaranteed Pub/Sub leaves) - -The skill ships two best-practices reference samples for this flow. Read them by their exact relative paths and adapt each into its own generated class. There is no merge and no third skeleton: the Publisher class is a near-verbatim adaptation of the publisher sample, and the Subscriber class is a near-verbatim adaptation of the consumer sample. Generating two near-verbatim classes is more reliable than synthesizing a single program from both, because the source shape is copied, not invented. - -- `jcsmp-guaranteed-publisher-sample.java`: basic-auth connect, PERSISTENT BytesMessage publish to a topic, publish ACK/NACK handler, session-event and reconnect-event handling that pauses publishing while the transport is down, SIGINT graceful shutdown that finishes outstanding ACKs. -- `jcsmp-guaranteed-subscriber-sample.java`: basic-auth connect, in-process durable-queue provisioning plus topic subscription, CLIENT-ack flow that ACKs only after processing, session-event and flow-event handling. - -Each class builds its OWN `JCSMPSession` (two sessions, one per class, matching the reference samples; there is no shared-session plumbing). Preserve the session-event and reconnect-event and publish ACK/NACK handlers in the Publisher, and the session-event and flow-event handlers in the Subscriber; they are part of the best-practices contract and must not be stripped during adaptation. - -### Subscriber class (near-verbatim `GuaranteedSubscriber`) - -The Subscriber provisions the durable queue and the topic subscription, and it is the long-running process the developer runs FIRST. The Publisher does NOT provision anything. Wire the Subscriber in this canonical sequence, each step grounded in the doc linked at the bottom of this file: - -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.IGNORE_DUPLICATE_SUBSCRIPTION_ERROR` to `true` BEFORE `createSession` (it defaults to false); otherwise `addSubscription` throws Subscription Already Exists on every run after the first (sample line 87). -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the consumer sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Best practice: check `session.isCapable(CapabilityType.ENDPOINT_MANAGEMENT)` before provisioning; exit cleanly if the broker disallows client-side endpoint management (sample lines 120-125). -4. `JCSMPFactory.onlyInstance().createQueue(queueName)` for the queue from the design summary. -5. Build the `EndpointProperties` and set the queue access type from the `Access type` summary field (see the access-type snippet below), then `session.provision(queue, endpointProps, JCSMPSession.FLAG_IGNORE_ALREADY_EXISTS)` so a re-run is idempotent. See [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md). -6. `session.addSubscription(queue, topic, JCSMPSession.WAIT_FOR_CONFIRM)` to map the topic onto the queue. See [Adding a Topic Subscription](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Adding-Topic-Subscriptio.md). -7. Build a `ConsumerFlowProperties` flow with `setAckMode(JCSMPProperties.SUPPORTED_MESSAGE_ACK_CLIENT)`, create it with the consumer sample's `FlowEventHandler`, and `flow.start()`. Immediately after `flow.start()` returns, emit the queue-bound marker: `trace("VERIFY: QUEUE_BOUND");`. See [Acknowledging Messages](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Acknowledging-Messages.md). -8. In the flow's `onReceive`, call `msg.ackMessage()` only after processing is complete; immediately after `ackMessage()`, emit the round-trip marker: `trace("VERIFY: MESSAGE_RECEIVED");`. -9. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the consumer sample (sample lines 179-207). The Subscriber is the SIGINT target: on SIGINT it stops the flow, finishes outstanding ACKs, and calls `session.closeSession()`. - -**Access type (exclusive vs non-exclusive): a `setAccessType` snippet, not a separate sample.** The `Access type` field in the design summary picks one line. Set the queue access type on the `EndpointProperties` before provisioning (item 5 above): - -```java -EndpointProperties endpointProps = new EndpointProperties(); -endpointProps.setAccessType(EndpointProperties.ACCESSTYPE_EXCLUSIVE); // one active consumer / ordered failover -// endpointProps.setAccessType(EndpointProperties.ACCESSTYPE_NONEXCLUSIVE); // competing consumers / round-robin load balance -endpointProps.setPermission(EndpointProperties.PERMISSION_CONSUME); -``` - -When the `Access type` field reads exclusive, keep the `ACCESSTYPE_EXCLUSIVE` line: only one active consumer receives at a time, and on its disconnect a standby flow takes over in order (ordered failover). When it reads non-exclusive, use the `ACCESSTYPE_NONEXCLUSIVE` line instead: bound flows compete and receive in round-robin, so the load is shared across workers. Read the `Access type` field and choose the one line; do not emit both active. This is grounded in [Receiving Guaranteed Messages](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Receiving-Guaranteed-Messages.md), which states an exclusive queue delivers in order with one active consumer and failover to the next flow, and a non-exclusive queue delivers round-robin across bound flows. - -### Publisher class (near-verbatim `GuaranteedPublisher`) - -The Publisher only connects and publishes; it does NOT provision the queue or the subscription (the Subscriber owns that, and starts first). It must exit on its own after the ack so the developer does not have to interrupt it. Wire the Publisher in this canonical sequence: - -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the publisher sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Acquire the `XMLMessageProducer` with the publisher sample's publish ACK/NACK callback handler (`PublishCallbackHandler`), then register the publisher sample's `JCSMPReconnectEventHandler` through an empty never-started consumer (`session.getMessageConsumer(reconnectHandler, null)`). The docs ground this idiom for publish-only applications; keep it because it surfaces transport events that pause the send loop. In the ACK callback (`responseReceivedEx`), emit the publish-ACK marker: `trace("VERIFY: PUBLISH_ACKED");`. CRITICAL: the reference publisher sample logs the ACK at `logger.debug` (sample line 229), which is invisible in default output, so the verify script would never see it and would report a false code failure. The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. -4. Publish a PERSISTENT `BytesMessage` to the topic (`DeliveryMode.PERSISTENT`). See [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md). -5. EXIT on its own: after observing the ACK (the `VERIFY: PUBLISH_ACKED` marker), drain outstanding ACKs (the sample's `Thread.sleep(1500)` idiom at sample line 216), call `session.closeSession()`, and return from `main`. REMOVE the sample's ENTER/SIGINT loop (sample line 168): the Publisher must terminate by itself, not wait for a key press or a signal. The SIGINT shutdown hook belongs to the Subscriber, not the Publisher. - -These four `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: PUBLISH_ACKED`, `VERIFY: QUEUE_BOUND`, `VERIFY: MESSAGE_RECEIVED`) are a fixed contract with `scripts/verify.sh`, which greps captured output PER PROCESS (the Subscriber log for `QUEUE_BOUND` and `MESSAGE_RECEIVED`, the Publisher log for `PUBLISH_ACKED`). Both classes connect, so both print `VERIFY: CONNECTED`; the script reads markers per-process for that reason. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. - -**Expose both mains to the verify script via two named exec executions.** The fixed `scripts/verify.sh` runs each main without knowing the per-generation package, so the generated pom MUST declare two `exec-maven-plugin` `` blocks, one with `publisher` and one with `subscriber`, each carrying its own `` set to that class's fully-qualified name. The script then addresses them as `mvn exec:java@publisher` and `mvn exec:java@subscriber`. The script references the fixed execution ids, never the generated FQCN, so the package and class names stay your discretion at generation time. Keep the `exec-maven-plugin` version resolved at generation time (no hardcoded version), the same way the `sol-jcsmp` coordinate is resolved. - -```xml - - org.codehaus.mojo - exec-maven-plugin - RESOLVED_AT_GENERATION_TIME - - - publisher - com.example.app.OrderPublisher - - - subscriber - com.example.app.OrderSubscriber - - - -``` - -**Disclaimer header on BOTH generated source files.** Stamp the top of each generated source file (the Publisher and the Subscriber; the pom is exempt as it is not source) with this exact line: `AI-assisted code. Review before production use.` Directly below it, add a pointer to the tailored verification checklist this generation writes into the project: `See the verification checklist: solace-verification-checklist.md`. That tailored `solace-verification-checklist.md` ships inside the generated project, since Step 6 writes it into the project root on every run, so the disclaimer points each generated file at the local checklist beside it rather than at a remote copy. The reference samples themselves carry NO disclaimer; that rule is the inverse. The disclaimer-header and explicit-single-class-imports rules apply to every leaf's generated classes, not just the guaranteed pub/sub pair. - -### Direct Pub/Sub leaf +**Guardrail: variant versus off-catalog.** Adapt freely WITHIN the chosen pattern family when the change is a topology or reliability knob that the canonical docs ground and that still compiles, emits the leaf's `VERIFY:` markers, and is verifiable by a `verify.sh` stage (durable versus temporary reply queue, an async versus blocking reply listener, exclusive versus non-exclusive access, redelivery handling). If a requirement leaves the JCSMP topic/queue publish and subscribe model, or needs behavior the canonical docs cannot ground, STOP and tell the developer it is off-catalog rather than inventing it. -When the `Pattern` field is `Direct Pub/Sub`, generation reads the direct pub/sub sample pair (`jcsmp-direct-publisher-sample.java` + `jcsmp-direct-subscriber-sample.java`) and adapts each into a generated class, the same near-verbatim discipline the guaranteed flow uses. Direct messaging is at-most-once: there is no broker ACK and no redelivery, so this leaf has NO queue, NO provisioning, NO CLIENT-ack, and NO publish-ACK callback. Read the two samples by their exact relative paths and adapt each into its own generated class (a Publisher and a Subscriber), each building its own `JCSMPSession`. The disclaimer-header, explicit-single-class-imports, and named-exec rules from the guaranteed flow apply here unchanged. +**Preserve the marker contract.** A variant adapts the SHAPE, never the marker strings. It MUST still emit its leaf's `VERIFY:` markers, character for character, so the fixed `verify.sh` stages drive it unchanged (this is exactly why a durable-reply-queue requestor still prints `VERIFY: REPLY_RECEIVED`). Never rename or drop a marker to fit a variant. -- `jcsmp-direct-publisher-sample.java`: basic-auth connect with reconnect-configured channel properties, a continuous DIRECT publish loop with a rotating payload to a dynamic topic (mirroring the guaranteed publisher's send flow), a stats thread, and a graceful SIGINT shutdown. No publish-ACK callback and no correlation key (direct has no broker ACK). Generation STRIPS the unbounded loop, stats thread, and SIGINT hook down to a short bounded burst of DIRECT messages that self-exits (Publisher class steps below), the same way the guaranteed publisher's loop is stripped, so the generated Publisher runs in the foreground and ends on its own. -- `jcsmp-direct-subscriber-sample.java`: basic-auth connect, a topic subscription confirmed via `session.addSubscription(topic, true)` (the `true` waits for broker confirm; no queue, no provisioning), an async DIRECT consumer started with `consumer.start()`, `message.getDiscardIndication()` for at-most-once egress-discard evidence in `onReceive`, and the SIGINT graceful-shutdown hook (the Subscriber is the long-running SIGINT target). +**Demo harness is not application logic.** The samples carry demo-harness elements that exist only to make a standalone run observable: the ENTER-to-quit `System.in` loop, the once-per-second stats-printing thread, the pacing `Thread.sleep` between sends, and the rotating example payload. When the developer's request describes a real application domain, do NOT carry these into the generated classes: replace the example payload and send cadence with the application's real messages and triggers, and keep only the lifecycle pieces the leaf's steps call for (the SIGINT hook on the long-running role, the self-exit on the foreground role). When the request IS a demo or a try-it run, keeping the harness is fine. Two things are NOT demo harness: the `trace(...)` narration method (keep it; applications replace its single body to route narration to their logging or reporting system) and the `VERIFY:` markers, which are emitted through `trace(...)` like every other status line. Because `verify.sh` greps captured stdout for the markers and for the shutdown hook's `Shutdown signal received` proof line (both flow through `trace(...)`), the generated `trace(...)` body MUST stay `System.out.println` through the Step 5 verification stages; rerouting that body to a logger or reporting system is the developer's step AFTER the verify stages pass, never part of generation. Never treat the logging backend (Step 3) as demo harness: the backend and its config file ship in every generated app. -#### Subscriber class (near-verbatim `DirectSubscriber`) +**Comments follow their code.** The samples' comments are part of the reference, not decoration: they carry the documented best practices (the session-independent `createMessage`, the binary payload and the legacy `XML` naming, `getData()` versus `getBytes()`, registering a session event handler at session creation, the ACK/NACK handling options, the correlation-key rule) into the code a developer reads long after this session. Generation carries them under ONE contract, on every door and every app shape: -The Subscriber subscribes directly to the topic (no queue), consumes asynchronously, and is the long-running process the developer runs FIRST. Wire it in this sequence: +- A generated class that carries a construct from a sample (near-verbatim or adapted) carries that construct's comment too, with only the names adapted to the application. Never strip a comment from a construct you kept. +- A dropped construct (the demo harness above, or a part the leaf does not use) takes its comment with it. Never orphan a comment onto unrelated code. +- Fresh messaging code with no sample twin (an embedded shape's own wiring) that applies a documented practice gets a SHORT comment naming the practice, in the samples' comment style. +- The one-line comments this skill mandates elsewhere (the Quickstart fail-fast posture below, the `setCorrelationKey` distinction in `implement-request-reply.md`) always apply on their paths; this contract adds to them, it does not replace them. -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.REAPPLY_SUBSCRIPTIONS` to `true` on those properties BEFORE `createSession` (it defaults to false): a Direct topic subscription is a client-side session subscription, not held on a broker queue, so without this it is lost on an automatic reconnect and delivery silently stops. -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Acquire the async consumer with `session.getMessageConsumer(listener)`, add the topic subscription with `session.addSubscription(topic, true)` (the `true` waits for broker confirm, so the route is live before the subscriber reports ready, since direct is at-most-once), then `consumer.start()`. Immediately after `consumer.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. There is NO queue and NO `session.provision(...)` on the direct path. -4. In the listener's `onReceive`, count the message and call `message.getDiscardIndication()` for at-most-once egress-discard evidence; immediately emit the receipt marker: `trace("VERIFY: MESSAGE_RECEIVED");`. There is NO `ackMessage()` and NO CLIENT-ack on the direct path (direct is at-most-once, the broker holds no copy and expects no acknowledgement). -5. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the sample. The Subscriber is the SIGINT target: on SIGINT it calls `session.closeSession()` directly (closing the session also closes the consumer; direct has no acknowledgements to drain), printing `Shutdown signal received` so the verify script sees the shutdown-hook proof line. +The tailored checklist's Generation conformance group carries a binary item for this contract, so a run that strips the comments fails its own checklist. -#### Publisher class (near-verbatim `DirectPublisher`) - -The Publisher only connects and publishes a short bounded burst of DIRECT messages, then exits on its own; it does NOT provision anything and does NOT wait for a key press or a signal. Wire it in this sequence: - -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Acquire the producer with a minimal mandatory streaming-publish event handler (`session.getMessageProducer(handler)`). The handler parameter is mandatory even for direct: a `null` handler is rejected at runtime with "Blocking publishing mode is not supported". Direct messaging is at-most-once with no broker ACK, so the handler's `responseReceivedEx` is not expected to fire and `handleErrorEx` only logs the error (rendering the broker subcode when present); there is no correlation key. Do NOT pass `null`, and do NOT wait on an ACK: direct has no broker ACK to act on (ACK/NACK handling and correlation keys are the guaranteed publisher's contract). -4. Set the delivery mode EXPLICITLY with `message.setDeliveryMode(DeliveryMode.DIRECT)` and publish a SHORT BOUNDED BURST to the design topic via `producer.send(message, topic)` in a small fixed-count loop, then self-exit (close the session and return from `main`). DIRECT is the API default, but stating it makes the at-most-once intent unambiguous. REMOVE the sample's UNBOUNDED ENTER/SIGINT publish loop, its rotating-payload/dynamic-topic illustration, its stats thread, and its SIGINT shutdown hook (the sample mirrors the guaranteed publisher's send flow; here it collapses to a bounded burst): the generated Publisher publishes its burst and terminates by itself, not on a key press or a signal, so the verify script can run it in the foreground and wait for it to exit. Direct has no outstanding ACKs to drain, so no pre-close sleep is needed. The SIGINT shutdown hook belongs to the long-running Subscriber, not the Publisher. See [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md). -5. The Publisher emits NO send-side marker. Direct has no broker ACK, so there is no `PUBLISH_ACKED` analog; a marker after `producer.send` would only prove the local call returned, not that the broker accepted the message, which is misleading for an at-most-once path. The real proof of the round-trip is `VERIFY: MESSAGE_RECEIVED` on the Subscriber side. - -These three `VERIFY:` marker strings for the Direct Pub/Sub leaf (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: MESSAGE_RECEIVED`) are a fixed contract with `scripts/verify.sh` (the `direct` stage greps the Subscriber log for `VERIFY: SUBSCRIBED` as its readiness gate and for `VERIFY: MESSAGE_RECEIVED` as its pass condition). The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. - -**Expose both mains to the verify script via two named exec executions.** Just like the guaranteed flow, the generated pom MUST declare two `exec-maven-plugin` `` blocks for the direct pair, one with `publisher` and one with `subscriber`, each carrying its own `` set to that class's fully-qualified name. The `direct` stage addresses them as `mvn exec:java@subscriber` (started first, the readiness gate) and `mvn exec:java@publisher` (the foreground burst). Keep the `exec-maven-plugin` version resolved at generation time. - -### Request-Reply (Direct) leaf - -When the `Pattern` field is `Request-Reply (Direct)`, generation reads the direct request-reply sample pair (`jcsmp-direct-requestor-sample.java` + `jcsmp-direct-replier-sample.java`) and adapts each into a generated class, the same near-verbatim discipline the guaranteed and direct pub/sub flows use. Direct request-reply is at-most-once on both legs: there is no broker ACK and no redelivery, so this leaf has NO queue, NO provisioning, and NO CLIENT-ack. The requestor self-exits after the reply (its blocking request is the synchronization point); the replier is the long-running SIGINT target. Read the two samples by their exact relative paths and adapt each into its own generated class (a Requestor and a Replier), each building its own `JCSMPSession`. The disclaimer-header and explicit-single-class-imports rules from the guaranteed flow apply here unchanged. - -- `jcsmp-direct-requestor-sample.java`: basic-auth connect, a started consumer (required before any request), a blocking `session.createRequestor()` plus `requestor.request(requestMsg, REQUEST_TIMEOUT_MS, topic)` with a POSITIVE `REQUEST_TIMEOUT_MS` (3000), a `JCSMPRequestTimeoutException` catch (optionally retrying once to absorb a cold-start race), then a self-exit. No SIGINT hook (the Requestor is a foreground process that ends on its own). -- `jcsmp-direct-replier-sample.java`: basic-auth connect, a plain topic subscription on the request topic via `session.addSubscription(requestTopic)` (no queue, no provisioning), an async consumer started with `consumer.start()`, an `onReceive` that guards on `requestMsg.getReplyTo() != null` and answers with the DIRECT reply convenience `producer.sendReply(requestMsg, replyMsg)`, and the SIGINT graceful-shutdown hook (the Replier is the long-running SIGINT target). - -#### Replier class (near-verbatim `DirectReplier`) - -The Replier subscribes directly to the request topic (no queue), answers each request, and is the long-running process the developer runs FIRST. Wire it in this sequence: - -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.REAPPLY_SUBSCRIPTIONS` to `true` on those properties BEFORE `createSession` (it defaults to false): the Direct request-topic subscription is a client-side session subscription, not held on a broker queue, so without this it is lost on an automatic reconnect and the replier silently stops answering requests. -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Acquire the reply producer with a minimal mandatory streaming-publish event handler (`session.getMessageProducer(handler)`). The handler parameter is mandatory even for direct: a `null` handler is rejected at runtime with "Blocking publishing mode is not supported". Direct messaging is at-most-once with no broker ACK, so the handler's `responseReceivedEx` is not expected to fire and `handleErrorEx` only logs. Do NOT pass `null`. -4. Acquire the async consumer with `session.getMessageConsumer(listener)`, add the plain request-topic subscription with `session.addSubscription(requestTopic)`, then `consumer.start()`. Immediately after `consumer.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. There is NO queue and NO `session.provision(...)` on the direct path. -5. In the listener's `onReceive`, guard on `requestMsg.getReplyTo() != null` (a message with no reply-to is not a request, so ignore it); immediately emit the request marker: `trace("VERIFY: REQUEST_RECEIVED");`. Then build a reply `BytesMessage` and answer with `producer.sendReply(requestMsg, replyMsg)`, the DIRECT reply convenience (it auto-routes to the reply-to and copies the correlation). Use `sendReply`, NOT a manual `producer.send(reply, getReplyTo())`: the manual form is the guaranteed replier's contract, and crossing them loses the explicit PERSISTENT reply control the guaranteed pattern documents. -6. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the sample. The Replier is the SIGINT target: on SIGINT it calls `session.closeSession()` directly (closing the session also closes the consumer; direct has no acknowledgements to drain), printing `Shutdown signal received` so the verify script sees the shutdown-hook proof line. - -#### Requestor class (near-verbatim `DirectRequestor`) - -The Requestor connects, issues one blocking request, and exits on its own; it does NOT provision anything and does NOT wait for a key press or a signal. Wire it in this sequence: - -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Acquire BOTH a producer and a started consumer BEFORE issuing the request. The blocking Requestor sends the request through the session's message producer AND receives the correlated reply through the session's message consumer, so both must be set up first. Acquire the producer with a minimal mandatory streaming-publish event handler (`session.getMessageProducer(handler)`): the API throws "No producer to perform operation" if no producer exists, and a `null` handler is rejected with "Blocking publishing mode is not supported", so pass a no-op handler (direct is at-most-once, the handler never acts on an ACK). Then start the consumer with `session.getMessageConsumer((XMLMessageListener) null)` and `consumer.start()`; a null listener is fine because the blocking `requestor.request(...)` returns the reply directly rather than via `onReceive`. -4. Define a POSITIVE `REQUEST_TIMEOUT_MS` (3000), create the blocking requestor with `session.createRequestor()`, and issue `requestor.request(requestMsg, REQUEST_TIMEOUT_MS, requestTopic)`. The positive timeout is the direct reliability knob (NOT a burst): the call blocks up to that long for the correlated reply before throwing `JCSMPRequestTimeoutException`. Catch that exception and optionally retry once to absorb a cold-start race (the replier may not have propagated its subscription on the very first attempt). -5. When the reply is non-null, emit the reply marker: `trace("VERIFY: REPLY_RECEIVED");`. Then self-exit: close the session and return from `main`. REMOVE any ENTER/SIGINT loop: the Requestor must terminate by itself, not wait for a key press or a signal. The SIGINT shutdown hook belongs to the Replier, not the Requestor. - -These four `VERIFY:` marker strings for the Request-Reply (Direct) leaf (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: REQUEST_RECEIVED`, `VERIFY: REPLY_RECEIVED`) are a fixed contract with `scripts/verify.sh` (the `direct-request-reply` stage greps the Replier log for `VERIFY: SUBSCRIBED` as its readiness gate and for `VERIFY: REQUEST_RECEIVED` as richer evidence, and the Requestor log for `VERIFY: REPLY_RECEIVED` as its pass condition). The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. - -**Expose both mains to the verify script via two named exec executions.** Just like the other flows, the generated pom MUST declare two `exec-maven-plugin` `` blocks for the request-reply pair, but with NEW ids `requestor` and `replier` (NOT `publisher`/`subscriber`), each carrying its own `` set to that class's fully-qualified name. The `direct-request-reply` stage addresses them as `mvn exec:java@replier` (started first, the readiness gate) and `mvn exec:java@requestor` (the foreground request). Keep the `exec-maven-plugin` version resolved at generation time. - -### Request-Reply (Guaranteed) leaf - -When the `Pattern` field is `Request-Reply (Guaranteed)`, generation reads the guaranteed request-reply sample pair (`jcsmp-guaranteed-requestor-sample.java` + `jcsmp-guaranteed-replier-sample.java`) and adapts each into a generated class, the same near-verbatim discipline the other leaves use. Guaranteed request-reply is PERSISTENT on both legs: the request and the reply are both delivered with `DeliveryMode.PERSISTENT` and broker-ACKed. Whatever the reply topology, this leaf answers with a MANUAL `producer.send(reply, request.getReplyTo())` and is explicitly NOT the direct-only blocking `Requestor` convenience API and NOT `producer.sendReply(...)` (those are the direct request-reply path, and crossing them onto the guaranteed path loses the explicit PERSISTENT reply control this pattern documents); that rule holds for the base shape and every variant. - -The BASE shape uses a per-request TEMPORARY reply queue plus a `FlowReceiver` (one outstanding request per temp queue, the reply read by a blocking `flow.receive(timeoutMs)`). The base also sets an application-managed CorrelationID on the request and matches it on receipt (the replier echoes it back): this is the doc-recommended request/reply matching mechanism, and it sits ALONGSIDE the dedicated temporary reply queue, which on its own already isolates the single outstanding reply. Generate the base shape when the design summary's `Consumption endpoint` reads `temporary reply queue + FlowReceiver` (the default). - -When the summary records a VARIANT instead (for example `durable reply queue + FlowReceiver`, chosen because the reply must survive the requestor restarting or because many requests are in flight at once), adapt the base shape to it per "Adapting a sample to a variant of its leaf" above: the requestor provisions a DURABLE reply queue (`createQueue` plus `session.provision(..., FLAG_IGNORE_ALREADY_EXISTS)`) instead of `createTemporaryQueue`, and binds it with a `FlowReceiver` on an ASYNC listener that matches each reply to its pending request by the CorrelationID (ground the durable-endpoint APIs against the canonical docs). The application-managed CorrelationID matching carries over from the base unchanged; what the variant adds is the durable reply endpoint and the async reception model (in place of the base's blocking `flow.receive`), needed because a shared durable reply queue can hold many in-flight replies at once and the CorrelationID is what sorts each reply to its request. Both legs stay PERSISTENT, the reply is still the manual `producer.send(reply, request.getReplyTo())`, CLIENT-ack discipline is unchanged, and the four `VERIFY:` markers are emitted exactly as below. A durable reply queue survives the requestor restarting because the broker retains the reply until it is consumed, which a temporary reply queue (deleted with its session) cannot do. - -The requestor self-exits after the reply (its blocking receive, or its correlated match, is the synchronization point); the replier is the long-running SIGINT target. Read the two samples by their exact relative paths and adapt each into its own generated class (a Requestor and a Replier), each building its own `JCSMPSession`. The disclaimer-header and explicit-single-class-imports rules from the guaranteed pub/sub flow apply here unchanged. +### Quickstart fail-fast channel defaults -- `jcsmp-guaranteed-requestor-sample.java`: basic-auth connect, a producer with a mandatory streaming-publish event handler, a per-request temporary reply queue (`session.createTemporaryQueue()`) bound by a `FlowReceiver` with a NULL listener (the blocking model), a PERSISTENT request whose reply-to is set to that temporary queue, an application-managed CorrelationID set on the request and matched on the reply (the request/reply matching mechanism), `setCorrelationKey` used for PUBLISHER ACK correlation ONLY (a local key, not request/reply matching), a `producer.send(request, requestTopic)`, then a BLOCKING `flow.receive(timeoutMs)` on the temporary reply queue, then a self-exit. No SIGINT hook (the Requestor is a foreground process that ends on its own). -- `jcsmp-guaranteed-replier-sample.java`: basic-auth connect, idempotent in-process provisioning of a durable request queue (behind a `session.isCapable(CapabilityType.ENDPOINT_MANAGEMENT)` gate) plus a topic subscription mapped onto it, a `FlowReceiver` whose handler answers each request with a MANUAL PERSISTENT `producer.send(reply, request.getReplyTo())`, and the SIGINT graceful-shutdown hook (the Replier is the long-running SIGINT target). +On the Quickstart door ONLY, do not carry the reference samples' reconnect-tuning block into the generated classes. The samples configure a `JCSMPChannelProperties` reconnect budget (`reconnectRetries` 20, `connectRetriesPerHost` 3); Quickstart generation OMITS that whole block (the `JCSMPChannelProperties` object and the `CLIENT_CHANNEL_PROPERTIES` setProperty call), leaving connect and reconnect retries and their timeouts at the JCSMP defaults, so a wrong host, port, or credential in this learning setup fails immediately instead of retrying for minutes. Drop ONE short comment at the session-properties build site in each generated class stating that posture: connect and reconnect settings are at the JCSMP defaults so failures surface immediately; see the [JCSMP Best Practices](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Best-Practices.md) page for reconnect tuning beyond Quickstart. The Solace Suggested door is unchanged: it keeps the samples' baseline or upgrades it per `solace-suggested-mode.md` Step 2. Custom follows its floor exception in `custom-mode.md`. -#### Replier class (near-verbatim `GuaranteedReplier`) +### Scaffold the opt-in autonomous-run config (Quickstart only) -The Replier provisions a durable request queue and its topic subscription, answers each request with a manual PERSISTENT reply, and is the long-running process the developer runs FIRST. Wire it in this sequence: +On the Quickstart path only, alongside the pom and the two classes, also generate two small project files so the developer can opt into having the agent run verify.sh for them (Step 5): -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.IGNORE_DUPLICATE_SUBSCRIPTION_ERROR` to `true` BEFORE `createSession` (it defaults to false); otherwise `addSubscription` on the request queue throws Subscription Already Exists on every run after the first. -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Acquire the reply producer with a mandatory streaming-publish event handler (`session.getMessageProducer(handler)`). Guaranteed replies are broker-ACKed, so the handler's `responseReceivedEx` fires per reply; the handler parameter is mandatory (a `null` handler is rejected at runtime with "Blocking publishing mode is not supported"). Do NOT pass `null`. -4. Best practice: check `session.isCapable(CapabilityType.ENDPOINT_MANAGEMENT)` before provisioning; exit cleanly if the broker disallows client-side endpoint management (the sample prints the `does not allow client-side endpoint management` line, which the verify script classifies as an environment failure). Then `JCSMPFactory.onlyInstance().createQueue(requestQueueName)`, build the `EndpointProperties`, and `session.provision(requestQueue, endpointProps, JCSMPSession.FLAG_IGNORE_ALREADY_EXISTS)` so a re-run is idempotent. See [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md). -5. `session.addSubscription(requestQueue, requestTopic, JCSMPSession.WAIT_FOR_CONFIRM)` to map the request topic onto the durable request queue. See [Adding a Topic Subscription](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Adding-Topic-Subscriptio.md). Build a `ConsumerFlowProperties` flow on the request queue with the sample's `FlowEventHandler` and `flow.start()`. Immediately after `flow.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. -6. In the flow handler's `onReceive`, guard on `requestMsg.getReplyTo() != null` (a message with no reply-to is not a request, so ACK it and ignore it); immediately emit the request marker: `trace("VERIFY: REQUEST_RECEIVED");`. Then build a reply `BytesMessage`, set it `DeliveryMode.PERSISTENT`, mark it a reply with `replyMsg.setAsReplyMessage(true)` (the requestor reads the reply field back with `isReplyMessage()` as one of the agreed reply-acceptance conditions), echo the request's CorrelationID onto it verbatim, and answer with a MANUAL `producer.send(replyMsg, requestMsg.getReplyTo())`, sending the PERSISTENT reply to the requestor's temporary reply queue. Use the manual `producer.send(reply, getReplyTo())`, NOT `producer.sendReply(...)`: `sendReply` is the direct replier's at-most-once convenience, and crossing it onto the guaranteed path loses the explicit PERSISTENT reply control. ACK the request (`requestMsg.ackMessage()`) only after the reply has been sent. -7. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the sample. The Replier is the SIGINT target: on SIGINT it stops the flow, finishes outstanding ACKs, and calls `session.closeSession()`, printing `Shutdown signal received` so the verify script sees the shutdown-hook proof line. See [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md). +- A committed `config.example.json` carrying placeholder values for the same four keys the Solace Suggested path already defines (`host`, `vpn_name`, `username`, `password`, matching the `JCSMPProperties` property-name strings, the constants' values, exactly). The placeholder values must be obviously fake so a committed example can never be mistaken for live credentials: `host` is `tcp://HOST:55555`, `vpn_name` is `YOUR_VPN`, `username` is `YOUR_USERNAME`, `password` is `YOUR_PASSWORD`. Tell the developer they opt in by copying `config.example.json` to `config.json` and filling in their real broker values. +- A `.gitignore` at the generated project root that ignores `config.json`, so the real credential-bearing file is never committed; only the placeholder `config.example.json` is. -#### Requestor class (near-verbatim `GuaranteedRequestor`) +This reuses the ONE config.json convention the skill already defines on the Solace Suggested path (Step 0): the same file name `config.json` and the same four keys (`host`, `vpn_name`, `username`, `password`), never a parallel file and never a divergent key set. Both doors read `config.json` through the shared `SolaceConnectionConfig` helper: Solace Suggested reads it directly (no CLI args), and Quickstart reads it when present and otherwise falls back to CLI args. On the Quickstart path a present `config.json` is therefore both the app's connection source and the agent's input for the Step 5 autonomous run. -The Requestor connects, issues one PERSISTENT request, blocks for the correlated reply on a temporary queue, and exits on its own; it does NOT provision the durable request queue (the Replier owns that, and starts first) and does NOT wait for a key press or a signal. Wire it in this sequence: +## Step 5: Verify with verify.sh — on every shape -1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). -2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. -3. Acquire the producer with a mandatory streaming-publish event handler (`session.getMessageProducer(handler)`): the PERSISTENT request is broker-ACKed, and a `null` handler is rejected at runtime with "Blocking publishing mode is not supported", so pass a real handler. Create the per-request temporary reply queue with `session.createTemporaryQueue()`, build a `ConsumerFlowProperties` with `setEndpoint(replyQueue)`, create the flow with a NULL listener via `session.createFlow(null, flowProps)` (the blocking model returns the reply directly from `flow.receive(...)` rather than via an async `onReceive`), and `flow.start()`. Immediately after `flow.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. Do NOT use `session.createRequestor()`: that blocking-Requestor convenience is the direct path; the guaranteed path uses the temp-queue plus blocking `flow.receive`. VARIANT: when the summary records a durable reply queue, replace `session.createTemporaryQueue()` with `createQueue(replyQueueName)` plus `session.provision(replyQueue, endpointProps, JCSMPSession.FLAG_IGNORE_ALREADY_EXISTS)`, and bind it with a `FlowReceiver` on an async listener that matches each reply to its pending request by the CorrelationID (the same CorrelationID the base already sets, now read on the async listener instead of the blocking receive); keep everything else (PERSISTENT request, the marker, no direct convenience) unchanged. -4. Build the request `BytesMessage`, set it `DeliveryMode.PERSISTENT`, and set its reply-to to the temporary reply queue with `requestMsg.setReplyTo(replyQueue)`. Set an application-managed CorrelationID on the request (`requestMsg.setCorrelationId(...)`, a unique value such as a `UUID` carrying the application's agreed notable prefix, per the reference sample's `CORRELATION_ID_PREFIX`); the replier echoes it onto the reply and the requestor matches it on receipt (step 5). This is the request/reply matching mechanism the Solace docs recommend, and it sits alongside the dedicated temporary reply queue. Also set `requestMsg.setCorrelationKey(requestMsg)`, and state in a code comment that `setCorrelationKey` is for PUBLISHER ACK correlation ONLY (a local key that identifies the request in the producer's ACK callback, never sent on the wire), distinct from the on-the-wire CorrelationID used for request/reply matching. Then `producer.send(requestMsg, requestTopic)`. -5. BLOCK on `flow.receive(timeoutMs)` on the temporary reply queue (a positive timeout; it returns null on timeout). A single PERSISTENT request plus a temp-queue reply is reliable, so issue exactly ONE request with NO burst and NO retry. When the reply meets the agreed reply-acceptance contract (its reply field is set, read with `isReplyMessage()`, and its CorrelationID carries the agreed prefix and equals the one set on the request; the reference sample's `isExpectedReply` method is the contract's single home), emit the reply marker: `trace("VERIFY: REPLY_RECEIVED");`. Then self-exit: stop the flow, close the session (which auto-deletes the temporary reply queue), and return from `main`. REMOVE any ENTER/SIGINT loop: the Requestor must terminate by itself. The SIGINT shutdown hook belongs to the Replier, not the Requestor. VARIANT: with a durable reply queue, the reply arrives on the async `FlowReceiver` listener and is matched by correlation id rather than a blocking `flow.receive`; emit `VERIFY: REPLY_RECEIVED` on the matched reply, and on self-exit close the session WITHOUT relying on temp-queue auto-delete (the durable reply queue is provisioned and persists by design, which is what delivers the restart-survival guarantee). +Verification is a `verify.sh` run, on every door and every app shape. Step 4 copied `scripts/verify.sh` into the project root and generated `verify-hooks.sh` beside it, so every command below resolves from inside the project. An improvised check (curl against the app's own HTTP API, a manual browser click) may TRIGGER traffic, but it NEVER renders the verdict: the verdict is the `VERIFY:` markers plus the script's exit code, which is what separates "your code is broken" (exit 1) from "your broker is unreachable" (exit 2). -These four `VERIFY:` marker strings for the Request-Reply (Guaranteed) leaf (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: REQUEST_RECEIVED`, `VERIFY: REPLY_RECEIVED`) are a fixed contract with `scripts/verify.sh` (the `guaranteed-request-reply` stage greps the Replier log for `VERIFY: SUBSCRIBED` as its readiness gate and for `VERIFY: REQUEST_RECEIVED` as richer evidence, and the Requestor log for `VERIFY: REPLY_RECEIVED` as its pass condition). The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. +**Live-broker heads-up (every run, every launcher).** Immediately before starting ANY process that will connect to a live broker — a verify.sh stage, `java -jar`, a `mvn` run, a run script — print ONE line that names the broker host and states the actual broker-side effects of this run: the queues it provisions (how many, durable or not), the connections it opens, and the messages it publishes, scaled to the real app (SKILL.md Invariant 6). -**Expose both mains to the verify script via two named exec executions.** Just like the direct request-reply flow, the generated pom MUST declare two `exec-maven-plugin` `` blocks for the guaranteed request-reply pair, with the ids `requestor` and `replier` (NOT `publisher`/`subscriber`), each carrying its own `` set to that class's fully-qualified name. The `guaranteed-request-reply` stage addresses them as `mvn exec:java@replier` (started first, the readiness gate) and `mvn exec:java@requestor` (the foreground request). Keep the `exec-maven-plugin` version resolved at generation time. +### The canonical two-class shape: the six fixed stages -### Scaffold the opt-in autonomous-run config (Quickstart only) +Generate the two classes rather than emitting everything at once, and verify each against the developer's reachable broker. The run story (run the Subscriber first, then the Publisher): the durable queue makes a sequential publish-then-consume safe once the Subscriber has provisioned it. The script compiles the project, starts the Subscriber first via `mvn exec:java@subscriber` (long-running), runs the Publisher via `mvn exec:java@publisher` (which exits on its own after the ack), watches each process's captured output for that stage's `VERIFY:` marker, confirms receipt on the Subscriber, sends SIGINT to the Subscriber to exercise its graceful-shutdown hook, and returns an exit code that classifies the outcome. Each stage waits for a different marker in a different process. -On the Quickstart path only, alongside the pom and the two classes, also generate two small project files so the developer can opt into having the agent run verify.sh for them (Step 5): +Invoke the matching stage after generating each class, passing the connection params through as positional args (the same ` [password]` contract the apps use): -- A committed `config.example.json` carrying placeholder values for the same four keys the Solace Suggested path already defines (`host`, `vpn_name`, `username`, `password`, matching the `JCSMPProperties` property-name strings, the constants' values, exactly). The placeholder values must be obviously fake so a committed example can never be mistaken for live credentials: `host` is `tcp://HOST:55555`, `vpn_name` is `YOUR_VPN`, `username` is `YOUR_USERNAME`, `password` is `YOUR_PASSWORD`. Tell the developer they opt in by copying `config.example.json` to `config.json` and filling in their real broker values. -- A `.gitignore` at the generated project root that ignores `config.json`, so the real credential-bearing file is never committed; only the placeholder `config.example.json` is. +1. After generating the Subscriber: `./verify.sh consumer `. It waits for `VERIFY: QUEUE_BOUND` in the Subscriber output (queue provisioned, subscribed, flow started). +2. After generating the Publisher: `./verify.sh publisher `. NOTE: this stage starts the Subscriber first to bind the queue, so it requires BOTH classes; it waits for `VERIFY: PUBLISH_ACKED` in the Publisher's own output (connect, then publish ACK). +3. At the end: `./verify.sh roundtrip `. It starts the Subscriber, runs the Publisher, and waits for `VERIFY: MESSAGE_RECEIVED` in the Subscriber output (a published PERSISTENT message lands on the queue and is consumed and ACKed). -This reuses the ONE config.json convention the skill already defines on the Solace Suggested path (Step 0): the same file name `config.json` and the same four keys (`host`, `vpn_name`, `username`, `password`), never a parallel file and never a divergent key set. Both doors read `config.json` through the shared `SolaceConnectionConfig` helper: Solace Suggested reads it directly (no CLI args), and Quickstart reads it when present and otherwise falls back to CLI args. On the Quickstart path a present `config.json` is therefore both the app's connection source and the agent's input for the Step 5 autonomous run. +The stages above (`consumer`, `publisher`, `roundtrip`) verify the guaranteed pub/sub leaves. `verify.sh` also accepts three further stage names for the other leaves: `direct` (the Direct Pub/Sub leaf), `direct-request-reply` (the Request-Reply (Direct) leaf), and `guaranteed-request-reply` (the Request-Reply (Guaranteed) leaf). All are runnable. The two request-reply stages are replier-first single round-trips: each starts the Replier first (the readiness gate on `VERIFY: SUBSCRIBED`), then runs the Requestor in the foreground and passes on `VERIFY: REPLY_RECEIVED` in the Requestor log (with `VERIFY: REQUEST_RECEIVED` in the Replier log as richer evidence). The `guaranteed-request-reply` stage differs only in that the Replier provisions a durable request queue (an endpoint-management denial classifies as an environment failure, exit 2) and the Requestor uses a temporary reply queue plus a blocking `flow.receive` rather than the direct `Requestor` convenience. -## Step 5: Verify each stage with scripts/verify.sh +### Every other shape: the `app` stage and `verify-hooks.sh` -Generate the two classes rather than emitting everything at once, and verify each against the developer's reachable broker with the bundled `scripts/verify.sh`. The run story (run the Subscriber first, then the Publisher): the durable queue makes a sequential publish-then-consume safe once the Subscriber has provisioned it. The script compiles the project, starts the Subscriber first via `mvn exec:java@subscriber` (long-running), runs the Publisher via `mvn exec:java@publisher` (which exits on its own after the ack), watches each process's captured output for that stage's `VERIFY:` marker, confirms receipt on the Subscriber, sends SIGINT to the Subscriber to exercise its graceful-shutdown hook, and returns an exit code that classifies the outcome. Each stage waits for a different marker in a different process. +A web app, an embedded service, or any single-process shape that the six fixed stages cannot drive is verified with the `app` stage: `./verify.sh app`. The stage sources the generated `verify-hooks.sh`, which carries the ONLY app-specific facts — how to start the app and how to cause one publish — while the script keeps the universal logic: marker watching, timeouts, environment-signature classification, and the 0/1/2 exit contract. Generation writes `verify-hooks.sh` because the generator just wrote the app and knows both commands: -Invoke the matching stage after generating each class, passing the connection params through as positional args (the same ` [password]` contract the apps use): +```bash +# verify-hooks.sh — generated with the project; verify.sh's app stage sources it. +START_CMD='java -jar target/the-app.jar' # starts the app (long-running) +TRIGGER_CMD='curl -s --max-time 10 -X POST localhost:8081/api/orders -d "{\"demo\":1}"' # causes exactly one publish +READY_MARKER='VERIFY: QUEUE_BOUND' # marker proving the consuming side is live +PASS_MARKER='VERIFY: MESSAGE_RECEIVED' # marker proving the round trip +``` -1. After generating the Subscriber: `scripts/verify.sh consumer `. It waits for `VERIFY: QUEUE_BOUND` in the Subscriber output (queue provisioned, subscribed, flow started). -2. After generating the Publisher: `scripts/verify.sh publisher `. NOTE: this stage starts the Subscriber first to bind the queue, so it requires BOTH classes; it waits for `VERIFY: PUBLISH_ACKED` in the Publisher's own output (connect, then publish ACK). -3. At the end: `scripts/verify.sh roundtrip `. It starts the Subscriber, runs the Publisher, and waits for `VERIFY: MESSAGE_RECEIVED` in the Subscriber output (a published PERSISTENT message lands on the queue and is consumed and ACKed). +Generate `verify-hooks.sh` with ONLY these four assignments. The app stage imports exactly these four names from a child shell and ignores everything else in the file, so an extra variable, a `cd`, or a function never reaches the script's own state; keep the file to the four lines all the same. Give a network trigger its own timeout (the `--max-time` above): the stage also bounds the trigger externally and force-kills one that never returns, but a self-timing trigger leaves cleaner evidence. -The stages above (`consumer`, `publisher`, `roundtrip`) verify the guaranteed pub/sub leaves. `scripts/verify.sh` also accepts three further stage names for the other leaves: `direct` (the Direct Pub/Sub leaf), `direct-request-reply` (the Request-Reply (Direct) leaf), and `guaranteed-request-reply` (the Request-Reply (Guaranteed) leaf). All three are runnable. The two request-reply stages are replier-first single round-trips: each starts the Replier first (the readiness gate on `VERIFY: SUBSCRIBED`), then runs the Requestor in the foreground and passes on `VERIFY: REPLY_RECEIVED` in the Requestor log (with `VERIFY: REQUEST_RECEIVED` in the Replier log as richer evidence). The `guaranteed-request-reply` stage differs only in that the Replier provisions a durable request queue (an endpoint-management denial classifies as an environment failure, exit 2) and the Requestor uses a temporary reply queue plus a blocking `flow.receive` rather than the direct `Requestor` convenience. +The `app` stage starts `START_CMD` in the background and captures its output, waits for `VERIFY: CONNECTED` and then `READY_MARKER` (an environment signature or a timeout before that classifies as exit 2 or exit 1), runs `TRIGGER_CMD` under a bounded deadline (a trigger that never returns is force-killed, and the marker watch still decides the verdict), waits for `PASS_MARKER` in the app's captured output, then stops the app with SIGINT and classifies. Set `READY_MARKER`/`PASS_MARKER` to the chosen leaf's own markers. A curl inside `TRIGGER_CMD` is exactly the right use of curl: it triggers, and the markers judge. On the canonical two-class shape, generate `verify-hooks.sh` too, carrying the classic commands (`START_CMD='mvn -q exec:java@subscriber'`, `TRIGGER_CMD='mvn -q exec:java@publisher'`) so the file documents the same contract everywhere; the fixed stages do not read it. -**Interpret the exit code:** +### Interpret the exit code -- **Exit 0 (stage passes).** The stage marker appeared, the graceful-shutdown hook ran (the `Shutdown signal received...` line is in the captured Subscriber output, proving `kill -INT` reached the app), and the JVM exited cleanly. A clean JVM exit is exit code `0` OR `130`: a JVM terminated by SIGINT *after* its shutdown hooks run reports `130` (128 + SIGINT), which is the expected graceful outcome, so the script accepts both. Advance to the next stage. -- **Exit 1 (code failure).** Compile failed, the marker never appeared, the shutdown hook did not run, or the run hung after SIGINT and had to be force-killed. Enter the bounded fix loop below. +- **Exit 0 (stage passes).** The stage milestone appeared and the run tore down cleanly (for the fixed stages that includes the graceful-shutdown proof line and a clean JVM exit, 0 or 130). Advance to the next stage. +- **Exit 1 (code failure).** Compile failed, a marker never appeared, a process died before its milestone, or a run hung and had to be force-killed. Enter the bounded fix loop below. - **Exit 2 (environment failure).** A doc-traceable JCSMP connection/auth signature appeared (the broker is unreachable or the credentials are wrong). STOP. Do NOT enter the fix loop. The code is not the problem. **Bounded fix loop (exit 1 only).** Diagnose the captured output, fix the code, and re-run the same stage, up to 3 automatic attempts per stage. After the third failure, stop fixing: summarize exactly what you tried across the attempts and what the captured output shows, then hand the decision to the developer. Report the evidence, not just "it failed". -**Fixes apply without a confirmation gate (both doors).** During the fix loop, apply the fix and re-run the same stage; do not preview the changed lines and ask the developer to confirm before writing them. This holds on the Solace Suggested path too, consistent with Step 0 generating the classes without a per-stage gate. +**Fixes apply without a confirmation gate (all doors).** During the fix loop, apply the fix and re-run the same stage; do not preview the changed lines and ask the developer to confirm before writing them. This holds on the Solace Suggested path too, consistent with Step 0 generating the classes without a per-stage gate. **Environment-failure path (exit 2).** Report the environment problem with the matched signature line from the captured output as evidence, and do NOT modify the code (a working app against a bad broker or wrong credentials must not be "fixed"). The fix is the developer's broker or credentials. -**Autonomous run from config.json (Quickstart only).** This applies to Quickstart ONLY. On the Quickstart path, run scripts/verify.sh for real instead of handing the commands back ONLY when BOTH conditions hold: a `config.json` is present at the generated project root, AND its four values are real, meaning they are NOT the `config.example.json` placeholders (`tcp://HOST:55555`, `YOUR_VPN`, `YOUR_USERNAME`, `YOUR_PASSWORD`). The opt-in is filling in real values, not merely copying the example: a `config.json` that still carries any placeholder value is treated as not-yet-opted-in, so fall through to the honesty-rule path below (compile only, then hand back the three commands). This keeps a half-finished `cp config.example.json config.json` from triggering a real run against placeholder credentials. When the values are real, read the four (`host`, `vpn_name`, `username`, `password`) from `config.json` and pass them to verify.sh as POSITIONAL CLI args, using the existing `verify.sh [password]` contract: config `host` becomes the `` arg, `vpn_name` the `` arg, `username` the `` arg, `password` the `[password]` arg (the stage name is the first positional, the four values follow). Run the three stages in this order: consumer first (it provisions the queue), then publisher, then roundtrip; this is the operational run order, distinct from the reference command list in the honesty rule below. verify.sh itself is unchanged; the agent reads `config.json` and passes the four values as positional args. Because a real `config.json` sits at the project root during this run, the generated apps read it directly through the `SolaceConnectionConfig` helper (the positional args are a harmless same-value fallback), so verify.sh drives the run unchanged. Interpret each stage's exit code exactly as the "Interpret the exit code" block below defines it: an exit 1 enters the bounded fix loop, an exit 2 stops without a fix, an exit 0 advances. Immediately before the live run, print ONE heads-up line that names the host from `config.json` and states the side effects (it provisions a durable queue and publishes one message), then run immediately with NO confirmation prompt: a config.json with real values is the consent, so there is no gate. This is the config.json-present-with-real-values branch of the honesty rule below; when `config.json` is absent or still holds placeholders, Quickstart keeps the existing behavior (compile only, then hand back the three commands). On that compile-only fall-through, also point the developer at the autonomous option so the loop is discoverable at the moment it matters: tell them you generated a `config.example.json`, and that they can have you run all three stages for them by copying it to `config.json`, filling in their real broker values, and asking you to verify (rather than running the handed-back commands by hand). This specific autonomous-run mechanism (reading the `config.json` VALUES and passing them to verify.sh as POSITIONAL args) applies to Quickstart ONLY. The Solace Suggested path ALSO runs verify.sh under the same real-values consent, but in its two-project mode (the per-role `--subscriber-dir`/`--publisher-dir` flags, each app reading its own `config.json` directly), as `solace-suggested-mode.md` Step 4 defines; the agent does not extract config values into positional args on the Solace Suggested path. +**Autonomous run from config.json (Quickstart only).** This applies to Quickstart ONLY. On the Quickstart path, run verify.sh for real instead of handing the commands back ONLY when BOTH conditions hold: a `config.json` is present at the generated project root, AND its four values are real, meaning they are NOT the `config.example.json` placeholders (`tcp://HOST:55555`, `YOUR_VPN`, `YOUR_USERNAME`, `YOUR_PASSWORD`). The opt-in is filling in real values, not merely copying the example: a `config.json` that still carries any placeholder value is treated as not-yet-opted-in, so fall through to the honesty-rule path below (compile only, then hand back the stage commands). This keeps a half-finished `cp config.example.json config.json` from triggering a real run against placeholder credentials. When the values are real, read the four (`host`, `vpn_name`, `username`, `password`) from `config.json` and pass them to verify.sh as POSITIONAL CLI args, using the existing `verify.sh [password]` contract. Run the three stages in this order: consumer first (it provisions the queue), then publisher, then roundtrip. Interpret each stage's exit code exactly as defined above. Immediately before the live run, print the ONE heads-up line this Step requires — name the host from `config.json` and state the ACTUAL broker-side effects of this run (the queues it provisions, the connections it opens, the messages it publishes) — then run immediately with NO confirmation prompt: a config.json with real values is the consent, so there is no gate. When `config.json` is absent or still holds placeholders, Quickstart keeps the existing behavior (compile only, then hand back the commands), and on that fall-through also point the developer at the autonomous option: tell them you generated a `config.example.json`, and that they can have you run all stages for them by copying it to `config.json`, filling in their real broker values, and asking you to verify. The Solace Suggested path ALSO runs verify.sh under the same real-values consent, but in its two-project mode (the per-role `--subscriber-dir`/`--publisher-dir` flags, each app reading its own `config.json` directly), as `solace-suggested-mode.md` Step 4 defines; the agent does not extract config values into positional args on the Solace Suggested path. -**Honesty rule (no fabricated runs).** If the developer's credentials are placeholders or not provided, do NOT fabricate a successful run. Compile only (for example `mvn -q compile`), report the compile result, and list the exact commands the developer must run against their broker: +**Honesty rule (no fabricated runs).** If the developer's credentials are placeholders or not provided, do NOT fabricate a successful run. Compile only (for example `mvn -q compile`), report the compile result, and list the exact commands the developer must run against their broker (from the project root, where Step 4 placed the script): ``` -scripts/verify.sh publisher -scripts/verify.sh consumer -scripts/verify.sh roundtrip +./verify.sh publisher +./verify.sh consumer +./verify.sh roundtrip ``` +For an `app`-stage shape, the handed-back command is `./verify.sh app` after the developer fills in `config.json`. + **Close every run with the next steps, stated plainly.** After generating the project, do NOT leave the developer guessing what to do next; end with a short, explicit next-steps message. -- **Quickstart.** Tell the developer, in order: (1) copy `config.example.json` to `config.json` and fill in their real broker values (`host`, `vpn_name`, `username`, `password`); (2) then just ask you to verify, and you will run `scripts/verify.sh` for them against their broker (it provisions the durable queue and publishes one message). If they would rather run it themselves, point them at the three `verify.sh` stage commands above. Make the fill-in-`config.json`-then-ask-me path the recommended one; it is the frictionless path to a verified round-trip. Step 6 writes the tailored `solace-verification-checklist.md` into the project root; when you deliver this close-of-run message, invite the developer to add their own items in its Your additional items section and to record any skill-undelivered concerns under Developer-owned items (not delivered by this skill). -- **Solace Suggested.** Same shape as Quickstart, via `verify.sh` two-project mode: fill in each project's gitignored `config.json` with real values, then just ask you to verify and you will run `scripts/verify.sh` against the broker with the per-role directory flags (`--subscriber-dir`/`--publisher-dir`, or `--replier-dir`/`--requestor-dir` for request-reply); it reads each `config.json` (no CLI-arg credentials) and needs a TLS-capable broker. If any `config.json` is still placeholder, compile each project and hand back the two-project `verify.sh` command instead. See `solace-suggested-mode.md` Step 4. Then emit and report the verification checklist per Step 6, which writes the tailored `solace-verification-checklist.md` into each project root; when you deliver this close-of-run message, invite the developer to add their own items in its Your additional items section and to record any skill-undelivered concerns under Developer-owned items (not delivered by this skill). +- **Quickstart.** Tell the developer, in order: (1) copy `config.example.json` to `config.json` and fill in their real broker values (`host`, `vpn_name`, `username`, `password`); (2) then just ask you to verify, and you will run `verify.sh` for them against their broker (naming the host and side effects first). If they would rather run it themselves, point them at the stage commands above. Make the fill-in-`config.json`-then-ask-me path the recommended one; it is the frictionless path to a verified round-trip. Step 4 already wrote the tailored `solace-verification-checklist.md` into the project root; when you deliver this close-of-run message, invite the developer to add their own items in its Your additional items section and to record any skill-undelivered concerns under Developer-owned items (not delivered by this skill). +- **Solace Suggested.** Same shape as Quickstart, via `verify.sh` two-project mode: fill in each project's gitignored `config.json` with real values, then just ask you to verify and you will run `verify.sh` against the broker with the per-role directory flags (`--subscriber-dir`/`--publisher-dir`, or `--replier-dir`/`--requestor-dir` for request-reply); it reads each `config.json` (no CLI-arg credentials) and needs a TLS-capable broker. If any `config.json` is still placeholder, compile each project and hand back the two-project `verify.sh` command instead. See `solace-suggested-mode.md` Step 4. Then report the verification checklist per Step 6; when you deliver this close-of-run message, invite the developer to add their own items in its Your additional items section and to record any skill-undelivered concerns under Developer-owned items (not delivered by this skill). ### Traps to avoid +- **Improvised verdicts.** Curl against the app's own API, or a browser click, proves the web layer answered; it does not prove the broker round trip, and it loses the exit-code triad. Trigger with whatever fits; judge ONLY by the markers and the verify.sh exit code. - **Publishing direct-to-queue instead of pub/sub.** Publish to a topic; let the durable queue carry a topic subscription. See [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md). - **XML or text payload instead of binary.** Default to a binary `BytesMessage` payload. The `XML` in JCSMP type names like `XMLMessage` and `BytesXMLMessage` is legacy API naming, not an XML payload format. Do not conflate them. - **DIRECT delivery where Guaranteed is required.** Use PERSISTENT delivery for the guaranteed pub/sub journey. See [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md). -- **Hardcoded credentials.** Never inline host, VPN, username, or password. Both doors build the connection `JCSMPProperties` through the shared `SolaceConnectionConfig` helper: Solace Suggested reads a gitignored `config.json`; Quickstart reads `config.json` when present and otherwise falls back to CLI args. -- **Hardcoded `sol-jcsmp` version.** Resolve the `sol-jcsmp` version at generation time (Step 3); never pin a `sol-jcsmp` number in the skill. log4j2 is the intentional exception: it is pinned to a known-good 2.x version (at or above `2.17.1`) per Step 3, and updating that pin is a deliberate maintenance action. -- **Wildcard imports.** When adapting the two samples, keep every import explicit in BOTH generated classes as the reference samples do; never collapse them to a wildcard like `import com.solacesystems.jcsmp.*;`. Wildcard imports are an anti-pattern: they hide which API types the code actually depends on and risk silent collisions as packages evolve. +- **Hardcoded credentials.** Never inline host, VPN, username, or password. All doors build the connection `JCSMPProperties` through the shared `SolaceConnectionConfig` helper: Solace Suggested reads a gitignored `config.json`; Quickstart reads `config.json` when present and otherwise falls back to CLI args. +- **Hardcoded versions.** Resolve every dependency version at generation time (Step 3); never pin a number in the skill or the generated pom rationale. The log4j2 `2.17.1` Log4Shell floor is a CHECK on the resolved result, not a pin. +- **Wildcard imports.** When adapting the samples, keep every import explicit in ALL generated classes as the reference samples do; never collapse them to a wildcard like `import com.solacesystems.jcsmp.*;`. Wildcard imports are an anti-pattern: they hide which API types the code actually depends on and risk silent collisions as packages evolve. - **Wrong groupId.** The Maven doc shows `com.solace`; the correct groupId is `com.solacesystems`. The shorter form does not resolve on Maven Central. -- **Running the Publisher before the Subscriber has provisioned the queue.** If the Publisher runs first, the PERSISTENT message is published to a topic the durable queue is not yet subscribed to, so it is silently dropped and never received. The default is NOT to add a Publisher fail-fast that probes for the queue (real producers do not check consumer queues, and it would dilute the near-verbatim shape). Instead, run the Subscriber first: `scripts/verify.sh` enforces that ordering, and the `VERIFY: MESSAGE_RECEIVED` marker is the real proof the round-trip worked. +- **Running the Publisher before the Subscriber has provisioned the queue.** If the Publisher runs first, the PERSISTENT message is published to a topic the durable queue is not yet subscribed to, so it is silently dropped and never received. The default is NOT to add a Publisher fail-fast that probes for the queue (real producers do not check consumer queues, and it would dilute the near-verbatim shape). Instead, run the Subscriber first: `verify.sh` enforces that ordering, and the `VERIFY: MESSAGE_RECEIVED` marker is the real proof the round-trip worked. - **Treating a sample as a spec that overrides the approved design.** The samples are grounded reference, not fixed templates. When the design summary records a variant of its leaf (a topology or reliability deviation from the base sample), adapt the sample to the design; do NOT snap the variant back to the sample's default shape, and do NOT report the deviation as a "conflict" with the skill. See "Adapting a sample to a variant of its leaf" above; hold the invariants and the marker contract while adapting, and treat a verified variant as fully first-class. -## Step 6: Emit the tailored verification checklist and report +## Step 6: Tailor the checklist (written at Step 4) and report -At the end of EVERY Implement run, on every door (Quickstart, Solace Suggested Secure or Non-Secure, and Custom), do two things: write a tailored `solace-verification-checklist.md` into the generated project, and report the same resolution in chat. The checklist is generation output, not an in-chat-only walk. +Step 4 writes the tailored `solace-verification-checklist.md` into the generated project as generation output. This step defines the tailoring and adds the end-of-run report in chat. At the end of EVERY Implement run, on every door (Quickstart, Solace Suggested, and Custom) and every app shape, report the checklist resolution in chat; the emitted file and the report say the same thing. -**1. Resolve each item to its state for the chosen mode.** Read `verification-checklist.md` (the master template) and resolve each item to its actual state for the door the developer chose and the code that was generated, using the master template's per-mode branch notes. Sort each resolved item into one of the three responsibility groups the master template defines: +**1. Resolve each item to its state for the chosen mode.** Read `verification-checklist.md` (the master template) and resolve each item to its actual state for the door the developer chose and the code that was generated, using the master template's per-mode branch notes. Sort each resolved item into one of the responsibility groups the master template defines: - **Delivered by this generation**: the chosen mode generated this into the code. - **Your responsibility (not delivered here)**: the chosen mode did not generate this, so it stays the developer's job. - **Verified by the round-trip**: the conformance checks the publisher to consumer round-trip already exercises. +- **Generation conformance**: the conformance checks on the generated output itself: the mechanical ones the `verify.sh` preflight reports (the disclaimer header, the logging floor, the version freshness, the recorded verify stage and exit code), plus the comments-follow-their-code item, which is checked by reading the generated classes rather than by the preflight. -An item the chosen mode generated lands under Delivered by this generation; an item that mode did not generate lands under Your responsibility (not delivered here). For example the TLS secure-session item is Delivered by this generation on Solace Suggested Secure and on Custom with the secure-session knob ticked, and it is Your responsibility on Quickstart, on Solace Suggested Non-Secure, and on Custom with that knob unticked. Resolve the admin-provisioned-queue item explicitly, not just the session-and-project items: it is Delivered by this generation ONLY on Custom with the admin-provisioned-queue knob ticked (the generated app binds a pre-existing admin-owned queue per `custom-mode.md`), and it is Your responsibility on Quickstart, on Solace Suggested (both sub-modes, which give admin-provisioned-queue guidance only and generate no binding code per `solace-suggested-mode.md`), and on Custom with that knob unticked. - -**2. Write the tailored file into the generated project, always.** Write the resolved result into the generated project as a file named exactly `solace-verification-checklist.md` at the project root, parallel to the `solace-design.md` that Design mode writes, so the two per-project Solace files read as a set. This write is ALWAYS done as generation output, like the pom and the classes, with NO consent gate. It differs from `solace-design.md`, which `design-mode.md` Step 4 writes only on the developer's OK: the checklist is always written, never gated. On the decoupled two-project layout (Solace Suggested, both sub-modes, and Custom with the decoupled-projects knob ticked) write ONE copy into EACH project root, consistent with each project already carrying its own `config.json` and `.gitignore`; single-project modes (Quickstart, and Custom without the decoupled-projects knob) get one copy at the project root. +An item the chosen mode generated lands under Delivered by this generation; an item that mode did not generate lands under Your responsibility (not delivered here). For example the TLS secure-session item is Delivered by this generation on Solace Suggested and on Custom with the secure-session knob ticked, and it is Your responsibility on Quickstart and on Custom with that knob unticked. Resolve the admin-provisioned-queue item explicitly, not just the session-and-project items: it is Delivered by this generation ONLY on Custom with the admin-provisioned-queue knob ticked (the generated app binds a pre-existing admin-owned queue per `custom-mode.md`), and it is Your responsibility on Quickstart, on Solace Suggested (which gives admin-provisioned-queue guidance only and generates no binding code per `solace-suggested-mode.md`), and on Custom with that knob unticked. -**3. Give the emitted file its tailored shape.** The emitted file carries the three responsibility groups above with each item pre-resolved to its state for this mode (not the master template's branch notes, which exist only for tailoring), then two more sections carried over from the master template: +**2. The emitted file's tailored shape.** The emitted file carries the responsibility groups above with each item pre-resolved to its state for this mode (not the master template's branch notes, which exist only for tailoring), then two more sections carried over from the master template: -- **Developer-owned items (not delivered by this skill)**: any item the skill does not deliver (non-Solace, out-of-scope, or otherwise), recorded as developer-owned rather than skill-satisfied, so at a glance a reader sees which items the skill stands behind and which the developer is tracking on their own. +- **Developer-owned items (not delivered by this skill)**: any item the skill does not deliver (non-Solace, out-of-scope, or otherwise), recorded as developer-owned rather than skill-satisfied, so at a glance a reader sees which items the skill stands behind and which the developer is tracking on their own. A consented deviation (for example dropping the decoupled-projects shape on a single-deliverable ask) is recorded here. - **Your additional items**: open space for the developer to keep growing the list after the run. -The emitted checklist is an artifact, not source, so by default it carries NO AI-assisted disclaimer header; that header is for generated source files, per Step 4. +On the decoupled two-project layout (Solace Suggested, and Custom with the decoupled-projects knob ticked) write ONE copy into EACH project root, consistent with each project already carrying its own `config.json` and `.gitignore`; single-project modes get one copy at the project root, parallel to the `solace-design.md` that Design mode writes, so the per-project Solace files read as a set. The emitted checklist is an artifact, not source, so it carries NO AI-assisted disclaimer header; that header is for generated source files, per Step 4. -**4. Report the same resolution in chat.** Tell the developer what landed under each responsibility group and that the tailored `solace-verification-checklist.md` was written into the project (into each project root on the decoupled layout). The in-chat report is retained: the emitted file and the report say the same thing. +**3. Report the same resolution in chat.** Tell the developer what landed under each responsibility group and that the tailored `solace-verification-checklist.md` was written into the project (into each project root on the decoupled layout). Record the verify.sh stage that ran and its exit code in the checklist's Generation conformance group; on the compile-only fallback, record that the run was handed back instead. ## Grounding references @@ -371,5 +263,5 @@ Live `docs.solace.com` `.md` pages (WebFetch on demand): Public URLs (not docs.solace.com doc pages; use the live URL directly): - [JCSMP Javadoc](https://docs.solace.com/API-Developer-Online-Ref-Documentation/java/index.html) -- [Java API Release Notes](https://products.solace.com/download/JAVA_API_RN) +- [JCSMP API Release Notes](https://products.solace.com/download/JAVA_API_RN) - [sol-jcsmp on Maven Central](https://central.sonatype.com/artifact/com.solacesystems/sol-jcsmp) diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-request-reply.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-request-reply.md new file mode 100644 index 0000000..a252e44 --- /dev/null +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-request-reply.md @@ -0,0 +1,78 @@ +# Implement: Request-Reply leaves + +The per-leaf wiring for both request-reply leaves. Read this file from `implement-mode.md` Step 4 when the `Pattern` field is `Request-Reply (Direct)` or `Request-Reply (Guaranteed)`. The shared Step 4 rules in `implement-mode.md` (the AI-assisted disclaimer header, the shared `SolaceConnectionConfig` helper, the variant discipline, the demo-harness rule, and the explicit-single-class-imports rule) apply here unchanged. + +## Request-Reply (Direct) leaf + +When the `Pattern` field is `Request-Reply (Direct)`, generation reads the direct request-reply sample pair (`jcsmp-direct-requestor-sample.java` + `jcsmp-direct-replier-sample.java`) and adapts each into a generated class, the same near-verbatim discipline the guaranteed and direct pub/sub flows use. Direct request-reply is at-most-once on both legs: there is no broker ACK and no redelivery, so this leaf has NO queue, NO provisioning, and NO CLIENT-ack. The requestor self-exits after the reply (its blocking request is the synchronization point); the replier is the long-running SIGINT target. Read the two samples by their exact relative paths and adapt each into its own generated class (a Requestor and a Replier), each building its own `JCSMPSession`. + +- `jcsmp-direct-requestor-sample.java`: basic-auth connect, a started consumer (required before any request), a blocking `session.createRequestor()` plus `requestor.request(requestMsg, REQUEST_TIMEOUT_MS, topic)` with a POSITIVE `REQUEST_TIMEOUT_MS` (3000), a `JCSMPRequestTimeoutException` catch (optionally retrying once to absorb a cold-start race), then a self-exit. No SIGINT hook (the Requestor is a foreground process that ends on its own). +- `jcsmp-direct-replier-sample.java`: basic-auth connect, a plain topic subscription on the request topic via `session.addSubscription(requestTopic)` (no queue, no provisioning), an async consumer started with `consumer.start()`, an `onReceive` that guards on `requestMsg.getReplyTo() != null` and answers with the DIRECT reply convenience `producer.sendReply(requestMsg, replyMsg)`, and the SIGINT graceful-shutdown hook (the Replier is the long-running SIGINT target). + +### Replier class (near-verbatim `DirectReplier`) + +The Replier subscribes directly to the request topic (no queue), answers each request, and is the long-running process the developer runs FIRST. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire it in this sequence: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.REAPPLY_SUBSCRIPTIONS` to `true` on those properties BEFORE `createSession` (it defaults to false): the Direct request-topic subscription is a client-side session subscription, not held on a broker queue, so without this it is lost on an automatic reconnect and the replier silently stops answering requests. +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Acquire the reply producer with a minimal mandatory streaming-publish event handler (`session.getMessageProducer(handler)`). The handler parameter is mandatory even for direct: a `null` handler is rejected at runtime with "Blocking publishing mode is not supported". Direct messaging is at-most-once with no broker ACK, so the handler's `responseReceivedEx` is not expected to fire and `handleErrorEx` only logs. Do NOT pass `null`. +4. Acquire the async consumer with `session.getMessageConsumer(listener)`, add the plain request-topic subscription with `session.addSubscription(requestTopic)`, then `consumer.start()`. Immediately after `consumer.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. There is NO queue and NO `session.provision(...)` on the direct path. +5. In the listener's `onReceive`, guard on `requestMsg.getReplyTo() != null` (a message with no reply-to is not a request, so ignore it); immediately emit the request marker: `trace("VERIFY: REQUEST_RECEIVED");`. Then build a reply `BytesMessage` and answer with `producer.sendReply(requestMsg, replyMsg)`, the DIRECT reply convenience (it auto-routes to the reply-to and copies the correlation). Use `sendReply`, NOT a manual `producer.send(reply, getReplyTo())`: the manual form is the guaranteed replier's contract, and crossing them loses the explicit PERSISTENT reply control the guaranteed pattern documents. +6. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the sample. The Replier is the SIGINT target: on SIGINT it calls `session.closeSession()` directly (closing the session also closes the consumer; direct has no acknowledgements to drain), printing `Shutdown signal received` so the verify script sees the shutdown-hook proof line. + +### Requestor class (near-verbatim `DirectRequestor`) + +The Requestor connects, issues one blocking request, and exits on its own; it does NOT provision anything and does NOT wait for a key press or a signal. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire it in this sequence: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Acquire BOTH a producer and a started consumer BEFORE issuing the request. The blocking Requestor sends the request through the session's message producer AND receives the correlated reply through the session's message consumer, so both must be set up first. Acquire the producer with a minimal mandatory streaming-publish event handler (`session.getMessageProducer(handler)`): the API throws "No producer to perform operation" if no producer exists, and a `null` handler is rejected with "Blocking publishing mode is not supported", so pass a no-op handler (direct is at-most-once, the handler never acts on an ACK). Then start the consumer with `session.getMessageConsumer((XMLMessageListener) null)` and `consumer.start()`; a null listener is fine because the blocking `requestor.request(...)` returns the reply directly rather than via `onReceive`. +4. Define a POSITIVE `REQUEST_TIMEOUT_MS` (3000), create the blocking requestor with `session.createRequestor()`, and issue `requestor.request(requestMsg, REQUEST_TIMEOUT_MS, requestTopic)`. The positive timeout is the direct reliability knob (NOT a burst): the call blocks up to that long for the correlated reply before throwing `JCSMPRequestTimeoutException`. Catch that exception and optionally retry once to absorb a cold-start race (the replier may not have propagated its subscription on the very first attempt). +5. When the reply is non-null, emit the reply marker: `trace("VERIFY: REPLY_RECEIVED");`. Then self-exit: close the session and return from `main`. REMOVE any ENTER/SIGINT loop: the Requestor must terminate by itself, not wait for a key press or a signal. The SIGINT shutdown hook belongs to the Replier, not the Requestor. + +### Marker contract and exec executions (direct) + +These four `VERIFY:` marker strings for the Request-Reply (Direct) leaf (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: REQUEST_RECEIVED`, `VERIFY: REPLY_RECEIVED`) are a fixed contract with `scripts/verify.sh` (the `direct-request-reply` stage greps the Replier log for `VERIFY: SUBSCRIBED` as its readiness gate and for `VERIFY: REQUEST_RECEIVED` as richer evidence, and the Requestor log for `VERIFY: REPLY_RECEIVED` as its pass condition). The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. + +**Expose both mains to the verify script via two named exec executions.** Just like the other flows, the generated pom MUST declare two `exec-maven-plugin` `` blocks for the request-reply pair, but with NEW ids `requestor` and `replier` (NOT `publisher`/`subscriber`), each carrying its own `` set to that class's fully-qualified name. The `direct-request-reply` stage addresses them as `mvn exec:java@replier` (started first, the readiness gate) and `mvn exec:java@requestor` (the foreground request). Keep the `exec-maven-plugin` version resolved at generation time. + +## Request-Reply (Guaranteed) leaf + +When the `Pattern` field is `Request-Reply (Guaranteed)`, generation reads the guaranteed request-reply sample pair (`jcsmp-guaranteed-requestor-sample.java` + `jcsmp-guaranteed-replier-sample.java`) and adapts each into a generated class, the same near-verbatim discipline the other leaves use. Guaranteed request-reply is PERSISTENT on both legs: the request and the reply are both delivered with `DeliveryMode.PERSISTENT` and broker-ACKed. Whatever the reply topology, this leaf answers with a MANUAL `producer.send(reply, request.getReplyTo())` and is explicitly NOT the direct-only blocking `Requestor` convenience API and NOT `producer.sendReply(...)` (those are the direct request-reply path, and crossing them onto the guaranteed path loses the explicit PERSISTENT reply control this pattern documents); that rule holds for the base shape and every variant. + +The BASE shape uses a per-request TEMPORARY reply queue plus a `FlowReceiver` (one outstanding request per temp queue, the reply read by a blocking `flow.receive(timeoutMs)`). The base also sets an application-managed CorrelationID on the request and matches it on receipt (the replier echoes it back): this is the doc-recommended request/reply matching mechanism, and it sits ALONGSIDE the dedicated temporary reply queue, which on its own already isolates the single outstanding reply. Generate the base shape when the design summary's `Consumption endpoint` reads `temporary reply queue + FlowReceiver` (the default). + +When the summary records a VARIANT instead (for example `durable reply queue + FlowReceiver`, chosen because the reply must survive the requestor restarting or because many requests are in flight at once), adapt the base shape to it per "Adapting a sample to a variant of its leaf" in `implement-mode.md`: the requestor provisions a DURABLE reply queue (`createQueue` plus `session.provision(..., FLAG_IGNORE_ALREADY_EXISTS)`) instead of `createTemporaryQueue`, and binds it with a `FlowReceiver` on an ASYNC listener that matches each reply to its pending request by the CorrelationID (ground the durable-endpoint APIs against the canonical docs). The application-managed CorrelationID matching carries over from the base unchanged; what the variant adds is the durable reply endpoint and the async reception model (in place of the base's blocking `flow.receive`), needed because a shared durable reply queue can hold many in-flight replies at once and the CorrelationID is what sorts each reply to its request. Both legs stay PERSISTENT, the reply is still the manual `producer.send(reply, request.getReplyTo())`, CLIENT-ack discipline is unchanged, and the four `VERIFY:` markers are emitted exactly as below. A durable reply queue survives the requestor restarting because the broker retains the reply until it is consumed, which a temporary reply queue (deleted with its session) cannot do. + +The requestor self-exits after the reply (its blocking receive, or its correlated match, is the synchronization point); the replier is the long-running SIGINT target. Read the two samples by their exact relative paths and adapt each into its own generated class (a Requestor and a Replier), each building its own `JCSMPSession`. + +- `jcsmp-guaranteed-requestor-sample.java`: basic-auth connect, a producer with a mandatory streaming-publish event handler, a per-request temporary reply queue (`session.createTemporaryQueue()`) bound by a `FlowReceiver` with a NULL listener (the blocking model), a PERSISTENT request whose reply-to is set to that temporary queue, an application-managed CorrelationID set on the request and matched on the reply (the request/reply matching mechanism), `setCorrelationKey` used for PUBLISHER ACK correlation ONLY (a local key, not request/reply matching), a `producer.send(request, requestTopic)`, then a BLOCKING `flow.receive(timeoutMs)` on the temporary reply queue, then a self-exit. No SIGINT hook (the Requestor is a foreground process that ends on its own). +- `jcsmp-guaranteed-replier-sample.java`: basic-auth connect, idempotent in-process provisioning of a durable request queue (behind a `session.isCapable(CapabilityType.ENDPOINT_MANAGEMENT)` gate) plus a topic subscription mapped onto it, a `FlowReceiver` whose handler answers each request with a MANUAL PERSISTENT `producer.send(reply, request.getReplyTo())`, and the SIGINT graceful-shutdown hook (the Replier is the long-running SIGINT target). + +### Replier class (near-verbatim `GuaranteedReplier`) + +The Replier provisions a durable request queue and its topic subscription, answers each request with a manual PERSISTENT reply, and is the long-running process the developer runs FIRST. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire it in this sequence: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). Set `JCSMPProperties.IGNORE_DUPLICATE_SUBSCRIPTION_ERROR` to `true` BEFORE `createSession` (it defaults to false); otherwise `addSubscription` on the request queue throws Subscription Already Exists on every run after the first. +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Acquire the reply producer with a mandatory streaming-publish event handler (`session.getMessageProducer(handler)`). Guaranteed replies are broker-ACKed, so the handler's `responseReceivedEx` fires per reply; the handler parameter is mandatory (a `null` handler is rejected at runtime with "Blocking publishing mode is not supported"). Do NOT pass `null`. +4. Best practice: check `session.isCapable(CapabilityType.ENDPOINT_MANAGEMENT)` before provisioning; exit cleanly if the broker disallows client-side endpoint management (the sample prints the `does not allow client-side endpoint management` line, which the verify script classifies as an environment failure). Then `JCSMPFactory.onlyInstance().createQueue(requestQueueName)`, build the `EndpointProperties`, and `session.provision(requestQueue, endpointProps, JCSMPSession.FLAG_IGNORE_ALREADY_EXISTS)` so a re-run is idempotent. See [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md). +5. `session.addSubscription(requestQueue, requestTopic, JCSMPSession.WAIT_FOR_CONFIRM)` to map the request topic onto the durable request queue. See [Adding a Topic Subscription](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Adding-Topic-Subscriptio.md). Build a `ConsumerFlowProperties` flow on the request queue with the sample's `FlowEventHandler` and `flow.start()`. Immediately after `flow.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. +6. In the flow handler's `onReceive`, guard on `requestMsg.getReplyTo() != null` (a message with no reply-to is not a request, so ACK it and ignore it); immediately emit the request marker: `trace("VERIFY: REQUEST_RECEIVED");`. Then build a reply `BytesMessage`, set it `DeliveryMode.PERSISTENT`, mark it a reply with `replyMsg.setAsReplyMessage(true)` (the requestor reads the reply field back with `isReplyMessage()` as one of the agreed reply-acceptance conditions), echo the request's CorrelationID onto it verbatim, and answer with a MANUAL `producer.send(replyMsg, requestMsg.getReplyTo())`, sending the PERSISTENT reply to the requestor's temporary reply queue. Use the manual `producer.send(reply, getReplyTo())`, NOT `producer.sendReply(...)`: `sendReply` is the direct replier's at-most-once convenience, and crossing it onto the guaranteed path loses the explicit PERSISTENT reply control. ACK the request (`requestMsg.ackMessage()`) only after the reply has been sent. +7. KEEP the long-running loop and the SIGINT graceful-shutdown hook from the sample. The Replier is the SIGINT target: on SIGINT it stops the flow, finishes outstanding ACKs, and calls `session.closeSession()`, printing `Shutdown signal received` so the verify script sees the shutdown-hook proof line. See [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md). + +### Requestor class (near-verbatim `GuaranteedRequestor`) + +The Requestor connects, issues one PERSISTENT request, blocks for the correlated reply on a temporary queue, and exits on its own; it does NOT provision the durable request queue (the Replier owns that, and starts first) and does NOT wait for a key press or a signal. Start the file with the disclaimer header (the exact two lines `implement-mode.md` Step 4 defines), then wire it in this sequence: + +1. Build basic-auth `JCSMPProperties` (`host`, `vpn_name`, `username`, `password`) via the shared `SolaceConnectionConfig` helper (`config.json` if present, else CLI args). +2. `JCSMPFactory.onlyInstance().createSession(...)`, passing the sample's `SessionEventHandler`, then `session.connect()`. Immediately after `connect()` returns, emit the connect marker: `trace("VERIFY: CONNECTED");`. +3. Acquire the producer with a mandatory streaming-publish event handler (`session.getMessageProducer(handler)`): the PERSISTENT request is broker-ACKed, and a `null` handler is rejected at runtime with "Blocking publishing mode is not supported", so pass a real handler. Create the per-request temporary reply queue with `session.createTemporaryQueue()`, build a `ConsumerFlowProperties` with `setEndpoint(replyQueue)`, create the flow with a NULL listener via `session.createFlow(null, flowProps)` (the blocking model returns the reply directly from `flow.receive(...)` rather than via an async `onReceive`), and `flow.start()`. Immediately after `flow.start()` returns, emit the subscribed marker: `trace("VERIFY: SUBSCRIBED");`. Do NOT use `session.createRequestor()`: that blocking-Requestor convenience is the direct path; the guaranteed path uses the temp-queue plus blocking `flow.receive`. VARIANT: when the summary records a durable reply queue, replace `session.createTemporaryQueue()` with `createQueue(replyQueueName)` plus `session.provision(replyQueue, endpointProps, JCSMPSession.FLAG_IGNORE_ALREADY_EXISTS)`, and bind it with a `FlowReceiver` on an async listener that matches each reply to its pending request by the CorrelationID (the same CorrelationID the base already sets, now read on the async listener instead of the blocking receive); keep everything else (PERSISTENT request, the marker, no direct convenience) unchanged. +4. Build the request `BytesMessage`, set it `DeliveryMode.PERSISTENT`, and set its reply-to to the temporary reply queue with `requestMsg.setReplyTo(replyQueue)`. Set an application-managed CorrelationID on the request (`requestMsg.setCorrelationId(...)`, a unique value such as a `UUID` carrying the application's agreed notable prefix, per the reference sample's `CORRELATION_ID_PREFIX`); the replier echoes it onto the reply and the requestor matches it on receipt (step 5). This is the request/reply matching mechanism the Solace docs recommend, and it sits alongside the dedicated temporary reply queue. Also set `requestMsg.setCorrelationKey(requestMsg)`, and state in a code comment that `setCorrelationKey` is for PUBLISHER ACK correlation ONLY (a local key that identifies the request in the producer's ACK callback, never sent on the wire), distinct from the on-the-wire CorrelationID used for request/reply matching. Then `producer.send(requestMsg, requestTopic)`. +5. BLOCK on `flow.receive(timeoutMs)` on the temporary reply queue (a positive timeout; it returns null on timeout). A single PERSISTENT request plus a temp-queue reply is reliable, so issue exactly ONE request with NO burst and NO retry. When the reply meets the agreed reply-acceptance contract (its reply field is set, read with `isReplyMessage()`, and its CorrelationID carries the agreed prefix and equals the one set on the request; the reference sample's `isExpectedReply` method is the contract's single home), emit the reply marker: `trace("VERIFY: REPLY_RECEIVED");`. Then self-exit: stop the flow, close the session (which auto-deletes the temporary reply queue), and return from `main`. REMOVE any ENTER/SIGINT loop: the Requestor must terminate by itself. The SIGINT shutdown hook belongs to the Replier, not the Requestor. VARIANT: with a durable reply queue, the reply arrives on the async `FlowReceiver` listener and is matched by correlation id rather than a blocking `flow.receive`; emit `VERIFY: REPLY_RECEIVED` on the matched reply, and on self-exit close the session WITHOUT relying on temp-queue auto-delete (the durable reply queue is provisioned and persists by design, which is what delivers the restart-survival guarantee). + +### Marker contract and exec executions (guaranteed) + +These four `VERIFY:` marker strings for the Request-Reply (Guaranteed) leaf (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: REQUEST_RECEIVED`, `VERIFY: REPLY_RECEIVED`) are a fixed contract with `scripts/verify.sh` (the `guaranteed-request-reply` stage greps the Replier log for `VERIFY: SUBSCRIBED` as its readiness gate and for `VERIFY: REQUEST_RECEIVED` as richer evidence, and the Requestor log for `VERIFY: REPLY_RECEIVED` as its pass condition). The marker MUST reach stdout: emit it via `trace(...)`, whose generated body prints with `System.out.println`, NOT via `logger.debug`. This is the single highest silent-failure risk in the whole verify flow. The strings here and the strings the script greps for MUST stay in sync character-for-character; if you change one marker string, change it in `scripts/verify.sh` in the same effort. + +**Expose both mains to the verify script via two named exec executions.** Just like the direct request-reply flow, the generated pom MUST declare two `exec-maven-plugin` `` blocks for the guaranteed request-reply pair, with the ids `requestor` and `replier` (NOT `publisher`/`subscriber`), each carrying its own `` set to that class's fully-qualified name. The `guaranteed-request-reply` stage addresses them as `mvn exec:java@replier` (started first, the readiness gate) and `mvn exec:java@requestor` (the foreground request). Keep the `exec-maven-plugin` version resolved at generation time. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-publisher-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-publisher-sample.java index fcfe43f..93c618f 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-publisher-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-publisher-sample.java @@ -18,6 +18,10 @@ * teardown run from main's finally on every exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -57,7 +61,7 @@ public class DirectPublisher { private static final int PAYLOAD_SIZE = 512; // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 private static volatile int msgSentCounter = 0; // num messages sent private static volatile boolean isShutdown = false; diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-replier-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-replier-sample.java index 021efa8..27fe303 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-replier-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-replier-sample.java @@ -20,6 +20,10 @@ * teardown run from main's finally on every exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -59,7 +63,7 @@ public class DirectReplier { private static XMLMessageProducer producer; // shared producer used to send replies // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 /** This is the main app. Use this type of app to answer Direct (at-most-once) requests on a topic. */ public static void main(String... args) throws JCSMPException, InterruptedException, IOException { diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-requestor-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-requestor-sample.java index 29897d5..3bfe9a2 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-requestor-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-requestor-sample.java @@ -20,6 +20,10 @@ * main's finally on every exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -63,7 +67,7 @@ public class DirectRequestor { private static JCSMPSession session; // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 /** Main. */ public static void main(String... args) throws JCSMPException, IOException, InterruptedException { diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-subscriber-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-subscriber-sample.java index 7e4913b..d4b6607 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-subscriber-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-direct-subscriber-sample.java @@ -17,6 +17,10 @@ * main's finally on every exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -54,7 +58,7 @@ public class DirectSubscriber { private static JCSMPSession session; // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 /** This is the main app. Use this type of app for receiving Direct (at-most-once) messages from a topic. */ public static void main(String... args) throws JCSMPException, InterruptedException, IOException { diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-publisher-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-publisher-sample.java index 9974cd2..a4d0ef1 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-publisher-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-publisher-sample.java @@ -15,6 +15,10 @@ * teardownSolace(), with teardown run from main's finally on every exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -58,7 +62,7 @@ public class GuaranteedPublisher { private static final int PAYLOAD_SIZE = 512; // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 private static volatile int msgSentCounter = 0; // num messages sent private static volatile boolean isShutdown = false; diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-replier-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-replier-sample.java index b15d82b..471a19c 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-replier-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-replier-sample.java @@ -15,6 +15,10 @@ * teardownSolace(), with teardown run from main's finally on every exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -66,7 +70,7 @@ public class GuaranteedReplier { private static JCSMPSession session; // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 /** This is the main app. Use this type of app to answer Guaranteed (PERSISTENT) requests off a durable queue. */ public static void main(String... args) throws JCSMPException, InterruptedException, IOException { diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-requestor-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-requestor-sample.java index 8c9f8ed..7dfdb2c 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-requestor-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-requestor-sample.java @@ -15,6 +15,10 @@ * exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -66,7 +70,7 @@ public class GuaranteedRequestor { private static FlowReceiver replyFlow; // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 /** Main. */ public static void main(String... args) throws JCSMPException, IOException, InterruptedException { @@ -153,6 +157,8 @@ private static void sendRequest() throws JCSMPException { BytesMessage requestMsg = JCSMPFactory.onlyInstance().createMessage(BytesMessage.class); requestMsg.setData(("Request from " + APP_NAME).getBytes(StandardCharsets.UTF_8)); requestMsg.setDeliveryMode(DeliveryMode.PERSISTENT); // required for Guaranteed + // a temporary (anonymous) queue in the reply-to field is the documented shape for + // request/reply: https://docs.solace.com/Messaging/Guaranteed-Msg/Queues.htm#well-known-queues requestMsg.setReplyTo(replyQueue); // reply-to = the temporary reply queue // set a unique CorrelationID on the request (the doc-recommended request/reply // matching mechanism). The replier echoes it onto the reply; the requestor is diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-subscriber-sample.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-subscriber-sample.java index 720ff6e..6f13cbd 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-subscriber-sample.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-guaranteed-subscriber-sample.java @@ -17,6 +17,10 @@ * with teardown run from main's finally on every exit path. * * Only practices documented in canonical Solace sources are encoded here. + * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) */ package com.solace.samples.jcsmp; @@ -62,7 +66,7 @@ public class GuaranteedSubscriber { private static JCSMPSession session; // remember to add log4j2.xml to your classpath - private static final Logger logger = LogManager.getLogger(); // log4j2, but could also use SLF4J, JCL, etc. + private static final Logger logger = LogManager.getLogger(); // log4j2 by default; any backend swap follows the logging rule in implement-mode.md Step 3 /** This is the main app. Use this type of app for receiving Guaranteed messages (e.g. via a queue endpoint). */ public static void main(String... args) throws JCSMPException, InterruptedException, IOException { diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-solace-connection-config.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-solace-connection-config.java index 04b43b6..2a2476f 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-solace-connection-config.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/jcsmp-solace-connection-config.java @@ -1,6 +1,10 @@ /* * Shared connection-config helper for the JCSMP reference samples. * + * Any generated adaptation of this sample MUST begin with the exact line: + * AI-assisted code. Review before production use. + * (This reference sample itself carries no such header by design.) + * * Loads connection properties and turns them into a JCSMPProperties. Source * precedence: * 1. a config.json in the working directory (the project root), when present. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/prerequisites.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/prerequisites.md index 6b94832..f817f62 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/prerequisites.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/prerequisites.md @@ -1,24 +1,19 @@ # Prerequisites -Read this when a developer is new to Solace or does not yet have a broker to build against. Both Design mode and Implement mode route here first if the developer lacks basic Solace grounding or a reachable broker. +Read this when a developer does not yet have a broker to build against. Both Design mode and Implement mode route here first if the developer lacks a reachable broker. ## Table of contents -- What is Solace - Obtain a broker - Learn the JCSMP basics -## What is Solace - -Ground the developer in the high-level model (event broker, event mesh, topics, queues) before discussing patterns or generating code: [Solace Core Concepts](https://docs.solace.com/Get-Started/event-mesh-basics.md). Do not paraphrase the page. Point the developer at it, confirm they grasp the publish/subscribe and queue concepts, then continue. - ## Obtain a broker You need one reachable broker. The primary, recommended choice is [Solace Cloud](https://docs.solace.com/Get-Started/Getting-Started-Try-Broker.md): it is the simplest path to a running broker for a greenfield app and for the run-and-observe round-trip. Default to it unless the developer has a reason not to. As brief alternatives, the same page also documents a self-hosted Software Broker (run locally via container or VM) and an Appliance (existing hardware the developer already operates) for developers who cannot use Solace Cloud. -The skill does not provision or configure the broker. It assumes the broker is reachable and that the developer has connection details (host, message VPN, client username, password). +The skill does not provision or configure the broker. It assumes the broker is reachable and that the developer has connection details (host, message VPN, client username, password). A broker discovered running in the environment (for example a local container) is a fact to report, never an answer: still ask which broker the developer wants to target. ## Learn the JCSMP basics diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh index 64fb433..3e915fc 100755 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh @@ -7,6 +7,13 @@ # Solace Suggested two-project (each app reads its own gitignored config.json): # verify.sh --subscriber-dir --publisher-dir # verify.sh --replier-dir --requestor-dir +# Any other app shape (web app / embedded service; the app reads its own config): +# verify.sh app +# Sources ./verify-hooks.sh (generated with the project) for the ONLY +# app-specific facts: START_CMD (starts the app), TRIGGER_CMD (causes one +# publish; curl is fine HERE, as the trigger), READY_MARKER, PASS_MARKER. +# The observer logic (marker watching, timeouts, env classification, the +# 0/1/2 exit contract) stays in this script, identical for every shape. # # What it does (guaranteed pub/sub stage; the direct and request-reply stages are # documented at their own functions below): compiles the generated Maven project, then runs the subscriber @@ -86,6 +93,9 @@ usage: # Solace Suggested two-project (each app reads its own gitignored config.json): verify.sh --subscriber-dir --publisher-dir verify.sh --replier-dir --requestor-dir + + # Any other app shape (web app, embedded service; reads ./verify-hooks.sh): + verify.sh app EOF exit 1 } @@ -134,12 +144,65 @@ if [ "$TWO_PROJECT" -eq 1 ]; then # project dir (its config.json is read relative to that dir) regardless of cwd. SUB_DIR="$(cd "$SUB_DIR" && pwd)" PUB_DIR="$(cd "$PUB_DIR" && pwd)" -else +elif [ "$STAGE" != "app" ]; then [ -z "$HOST" ] && usage [ -z "$VPN" ] && usage [ -z "$USER_NAME" ] && usage fi +# ── Preflight conformance warnings (warn-only; never changes the exit contract) ── +# Reports the generation-conformance misses that the verification checklist's +# "Generation conformance" group records: a missing tailored checklist, source +# files without the AI-assisted disclaimer header, a log4j-core below the 2.17.1 +# Log4Shell floor, and a sol-jcsmp version that drifted from the authoritative +# repo1.maven.org metadata. Warnings only: the 0/1/2 contract is untouched. +preflight() { + local d="$1" + if [ ! -f "$d/solace-verification-checklist.md" ]; then + echo "PREFLIGHT WARN: $d/solace-verification-checklist.md is missing (every generation must emit it; implement-mode.md Step 4)" >&2 + fi + if [ -d "$d/src/main/java" ]; then + local missing + missing=$(grep -rL --include='*.java' 'AI-assisted code. Review before production use.' "$d/src/main/java" 2>/dev/null || true) + if [ -n "$missing" ]; then + echo "PREFLIGHT WARN: generated source files missing the AI-assisted disclaimer header:" >&2 + echo "$missing" >&2 + fi + fi + if [ -f "$d/pom.xml" ]; then + local l4j lo pomv relv + l4j=$(grep -A2 'log4j-core' "$d/pom.xml" 2>/dev/null | grep -oE '[^<]+' | head -1 | sed 's///') || true + case "${l4j:-}" in + ''|*'${'*) : ;; # absent or a Maven property; nothing to compare + *) + lo=$(printf '%s\n' "$l4j" '2.17.1' | sort -V 2>/dev/null | head -1) || true + if [ -n "${lo:-}" ] && [ "$lo" != "2.17.1" ]; then + echo "PREFLIGHT WARN: log4j-core $l4j in $d/pom.xml is below the 2.17.1 Log4Shell floor (CVE-2021-44228 family)" >&2 + fi + ;; + esac + pomv=$(grep -A2 'sol-jcsmp' "$d/pom.xml" 2>/dev/null | grep -oE '[^<]+' | head -1 | sed 's///') || true + case "${pomv:-}" in + ''|*'${'*) : ;; # absent or a Maven property; nothing to compare + *) + relv=$(curl -s --max-time 5 https://repo1.maven.org/maven2/com/solacesystems/sol-jcsmp/maven-metadata.xml 2>/dev/null | grep -oE '[^<]+' | sed -E 's/<\/?release>//g') || true + if [ -n "${relv:-}" ] && [ "$pomv" != "$relv" ]; then + echo "PREFLIGHT WARN: pom sol-jcsmp $pomv in $d differs from the authoritative latest GA $relv (resolve from repo1.maven.org metadata, never solrsearch)" >&2 + fi + ;; + esac + fi +} +if [ "$TWO_PROJECT" -eq 1 ]; then + preflight "$SUB_DIR" + preflight "$PUB_DIR" +else + preflight "." +fi + +# ── Live-run heads-up (SKILL.md Invariant 6): name the target and the side effects ─ +echo "── Live-run heads-up: stage '$STAGE' starts processes that connect to broker ${HOST:-}; broker-side effects can include provisioned durable queues, opened connections, and published messages ──" + # ── Stage → marker mapping ────────────────────────────────────────────────────── # The generated app emits four milestone markers across its two classes # (implement-mode.md Step 4): @@ -161,9 +224,10 @@ case "$STAGE" in publisher) ;; # waits for VERIFY: PUBLISH_ACKED in PUB_LOG consumer) ;; # waits for VERIFY: QUEUE_BOUND in SUB_LOG roundtrip) ;; # waits for VERIFY: MESSAGE_RECEIVED in SUB_LOG - direct) ;; # NEW (waves 2-4): direct pub/sub stage - direct-request-reply) ;; # NEW (waves 2-4): direct request-reply stage - guaranteed-request-reply) ;; # NEW (waves 2-4): guaranteed request-reply stage + direct) ;; # direct pub/sub stage + direct-request-reply) ;; # direct request-reply stage + guaranteed-request-reply) ;; # guaranteed request-reply stage + app) ;; # generic single-app stage driven by ./verify-hooks.sh *) echo "unknown stage: $STAGE" >&2; usage ;; esac @@ -202,7 +266,7 @@ SHUTDOWN_LINE='Shutdown signal received' # The baked consumer sample prints this line and exits CLEANLY when the broker # disallows client-side endpoint management (session.isCapable(ENDPOINT_MANAGEMENT) -# is false, consumer sample lines 119-125; implement-mode.md Step 4 Subscriber +# is false; implement-mode.md Step 4 Subscriber # item 3 keeps that behavior in the generated class). On that path the queue can # never bind, but the code is NOT the problem: a working app against a # restrictively configured broker must classify as an environment failure @@ -215,15 +279,25 @@ PUBLISH_WAIT_S=30 # foreground-publisher deadline before escalating to a code SHUTDOWN_WAIT_S=15 # bounded post-SIGINT wait before escalating to SIGTERM/SIGKILL # ── Bounded long-running shutdown (shared by the env-failure teardown and STAGE D) ─ -# Send SIGINT to the PID passed as $1, poll up to SHUTDOWN_WAIT_S for the JVM to -# exit on its own, then escalate SIGTERM then SIGKILL. A JVM that hangs after -# SIGINT is an anticipated failure mode, so NO teardown path may use a bare -# unbounded `wait`. Sets the global `rc` to the process's real exit code, or to -# the synthetic 124 when the run hung and had to be force-killed (124 is never a -# clean exit). The PID is a parameter so every long-running stage (the guaranteed -# subscriber, and the request/reply repliers added in later plans) can reuse it. +# Send SIGINT to the process GROUP of the PID passed as $1, poll up to +# SHUTDOWN_WAIT_S for the JVM to exit on its own, then escalate SIGTERM then +# SIGKILL (also group-directed). A JVM that hangs after SIGINT is an anticipated +# failure mode, so NO teardown path may use a bare unbounded `wait`. Sets the +# global `rc` to the process's real exit code, or to the synthetic 124 when the +# run hung and had to be force-killed (124 is never a clean exit). The PID is a +# parameter so every long-running stage (the guaranteed subscriber, the +# request/reply repliers, and the app stage) can reuse it. +# +# Why the kills target "-$1" (the group), not "$1": every launch site +# backgrounds its process under `set -m`, so each recorded PID leads its own +# process group. For the mvn stages the group is just the JVM, so this is +# equivalent to a PID kill. For the app stage the target may be a `bash -c` +# wrapper: macOS's system bash 3.2 keeps the wrapper alive for a compound +# START_CMD (bash 4.4+ execs the final command), and a PID-directed kill would +# then stop only the wrapper and orphan the app underneath it. Group-directed +# signals reach both. shutdown_long_running() { - kill -INT "$1" 2>/dev/null || true + kill -INT -- "-$1" 2>/dev/null || true rc=0 local hung=0 @@ -239,14 +313,17 @@ shutdown_long_running() { if [ "$hung" -eq 1 ]; then # The process did not honour SIGINT within the deadline; escalate and fail. echo "── Process did not exit ${SHUTDOWN_WAIT_S}s after SIGINT; escalating (SIGTERM, then SIGKILL) ──" >&2 - kill -TERM "$1" 2>/dev/null || true + kill -TERM -- "-$1" 2>/dev/null || true sleep 2 - kill -KILL "$1" 2>/dev/null || true + kill -KILL -- "-$1" 2>/dev/null || true wait "$1" 2>/dev/null || true rc=124 # synthetic "timed out" code; never a clean exit else # The process exited within the deadline; capture its real exit code. wait "$1" 2>/dev/null || rc=$? + # Sweep any group member that outlived the leader (a wrapper's app child); + # once the whole group is gone this is a no-op. + kill -KILL -- "-$1" 2>/dev/null || true fi } @@ -261,6 +338,8 @@ if [ "$TWO_PROJECT" -eq 1 ]; then exit 1 fi done +elif [ "$STAGE" = "app" ] && [ ! -f pom.xml ]; then + echo "── No pom.xml in the working directory; skipping the compile step (the app stage builds through its own START_CMD) ──" else echo "── Compiling (mvn -q compile) ─────────────────────────────────────" if ! mvn -q compile; then @@ -281,10 +360,18 @@ PUB_LOG="$(mktemp)" # a set -e exit), the backgrounded subscriber JVM would otherwise keep running # detached. It holds the EXCLUSIVE queue flow, so an orphan breaks every later # verify.sh run until manually killed, and both mktemp logs would leak. The -# EXIT trap makes teardown unconditional; for processes already waited on and -# logs already removed it is a harmless no-op. +# EXIT trap covers every graceful exit path: bash runs it on normal exit, on +# set -e exits, and on untrapped SIGINT/SIGTERM (verified on bash 3.2 through +# 5.2). SIGKILL is the one residual: no trap can run, and `set -m` has put each +# child in its own process group, so a hard kill of this script's group cannot +# reach them. That orphan risk is inherent to the set -m SIGINT-delivery +# mechanism (header note) and cannot be closed from inside the script. The +# kills are group-directed for the same reason shutdown_long_running's are (see +# its note). For processes already waited on and logs already removed the trap +# is a harmless no-op. cleanup() { - kill -TERM "${SUB_PID:-}" "${PUB_PID:-}" 2>/dev/null || true + if [ -n "${SUB_PID:-}" ]; then kill -TERM -- "-$SUB_PID" 2>/dev/null || true; fi + if [ -n "${PUB_PID:-}" ]; then kill -TERM -- "-$PUB_PID" 2>/dev/null || true; fi rm -f "${SUB_LOG:-}" "${PUB_LOG:-}" } trap cleanup EXIT @@ -456,11 +543,6 @@ else fi } # end run_guaranteed_stage -# ── Placeholder stages (NOT implemented in this plan; filled by waves 2-4) ─────── -# The dispatch wiring is in place so a later per-pattern increment drops a real -# body here without touching the dispatch front or the byte-stable guaranteed -# flow. Until then each stub echoes that it is not implemented and exits non-zero -# so no new pattern behavior is claimed. # ── run_direct_stage: the direct (at-most-once) pub/sub flow (D-04/D-05) ────────── # Direct messaging is at-most-once: no broker ACK, no redelivery, so a message # published to a not-yet-propagated subscription is simply dropped. This stage drives @@ -779,16 +861,128 @@ run_request_reply_stage() { fi } +# ── run_app_stage: the generic single-app flow (web app / embedded service) ────── +# The shape-agnostic observer. The generated ./verify-hooks.sh carries the ONLY +# app-specific facts (how to start the app, how to cause one publish, which leaf +# markers gate readiness and prove the pass); this function keeps the universal +# logic: marker watching in the app's own captured output, bounded waits, the +# ENV_SIGNATURES classification, and the shared 0/1/2 exit contract. The hooks +# still run as generated shell by design (they are generation output, exactly +# like the app they drive), but in a CHILD bash: only the four declared values +# are imported, so a stray extra line in the generated file cannot clobber the +# observer's own state. TRIGGER_CMD may be a curl against the +# app's own API — the curl is the TRIGGER; the markers render the VERDICT. The +# app stage does not require the samples' shutdown-hook proof line: an embedded +# app may manage shutdown its own way, so a pass is READY_MARKER + PASS_MARKER +# observed and a bounded teardown. +run_app_stage() { + if [ ! -f ./verify-hooks.sh ]; then + echo "app stage needs ./verify-hooks.sh (generated with the project; defines START_CMD, TRIGGER_CMD, READY_MARKER, PASS_MARKER)" >&2 + exit 1 + fi + # Source the hooks in a CHILD bash and import ONLY the four contract values. + # A hooks file that cds, sets observer state (status, TIMEOUT_S, SUB_LOG, + # ...), or redefines a function affects only the child, never this scope. + # The child routes the sourcing's stdout to stderr so stray output stays + # visible but is never eval'd; only declare -p's safely-quoted assignments + # come back. A missing value is caught by the loop below. + eval "$(bash -c '. ./verify-hooks.sh >&2; declare -p START_CMD TRIGGER_CMD READY_MARKER PASS_MARKER 2>/dev/null' || true)" + local v + for v in START_CMD TRIGGER_CMD READY_MARKER PASS_MARKER; do + [ -n "${!v:-}" ] || { echo "verify-hooks.sh must set $v" >&2; exit 1; } + done + + # Start the app (long-running, the SIGINT target); job control per the header note. + echo "── Starting app (START_CMD from verify-hooks.sh), watching for: $READY_MARKER ──" + set -m + bash -c "$START_CMD" >"$SUB_LOG" 2>&1 & + SUB_PID=$! + set +m + + local ready=0 + local deadline=$(( SECONDS + TIMEOUT_S )) + while [ "$SECONDS" -lt "$deadline" ]; do + if grep -qE "$ENV_SIGNATURES" "$SUB_LOG"; then + status=2 # environment failure; bypass the fix loop + break + fi + if grep -qF "$ENDPOINT_DENIED" "$SUB_LOG"; then + status=2 # broker disallows client-side endpoint management; environment, not code + break + fi + if grep -qF "$READY_MARKER" "$SUB_LOG"; then + ready=1 + break + fi + if ! kill -0 "$SUB_PID" 2>/dev/null; then + break # app exited before readiness (code failure, status stays 1) + fi + sleep 1 + done + + if [ "$status" -eq 2 ]; then + shutdown_long_running "$SUB_PID" + elif [ "$ready" -eq 1 ]; then + # Settle window, then the trigger. Bounded like STAGE B's foreground + # publisher: a trigger that never returns (a wedged app that accepts the + # request and never responds) must not hang the unattended script, so it + # runs backgrounded in its own group, polled to PUBLISH_WAIT_S, then + # escalated SIGTERM/SIGKILL. The marker watch below still runs after a + # hung trigger: the trigger's exit code was never part of the verdict + # (the markers are), and the publish may have landed even if the HTTP + # response never came back. Reusing PUB_PID (not a new name) keeps the + # EXIT trap's group-kill coverage over an orphaned trigger. + sleep 2 + echo "── Running trigger (TRIGGER_CMD from verify-hooks.sh), watching for: $PASS_MARKER ──" + set -m + bash -c "$TRIGGER_CMD" >"$PUB_LOG" 2>&1 & + PUB_PID=$! + set +m + local trig_deadline=$(( SECONDS + PUBLISH_WAIT_S )) + while kill -0 "$PUB_PID" 2>/dev/null; do + if [ "$SECONDS" -ge "$trig_deadline" ]; then + echo "── Trigger did not exit within ${PUBLISH_WAIT_S}s; escalating (SIGTERM, then SIGKILL) ──" >&2 + kill -TERM -- "-$PUB_PID" 2>/dev/null || true + sleep 2 + kill -KILL -- "-$PUB_PID" 2>/dev/null || true + break + fi + sleep 1 + done + wait "$PUB_PID" 2>/dev/null || true + + local pass_deadline=$(( SECONDS + TIMEOUT_S )) + while [ "$SECONDS" -lt "$pass_deadline" ]; do + if grep -qE "$ENV_SIGNATURES" "$SUB_LOG"; then + status=2 + break + fi + if grep -qF "$PASS_MARKER" "$SUB_LOG"; then + status=0 # the pass marker appeared in the app's own captured output + break + fi + if ! kill -0 "$SUB_PID" 2>/dev/null; then + break # app died before the pass marker (code failure) + fi + sleep 1 + done + + # Bounded teardown (SIGINT, then escalate); no shutdown-line requirement here. + shutdown_long_running "$SUB_PID" + else + shutdown_long_running "$SUB_PID" + fi +} + # ── Stage-dispatch front ───────────────────────────────────────────────────────── -# Route the validated stage to its per-pattern function. The guaranteed arm holds -# the byte-stable run_guaranteed_stage; the three new arms route to placeholder -# functions until waves 2-4 fill them. publisher|consumer|roundtrip stay spelled -# exactly (referenced by implement-mode.md Step 5). +# Route the validated stage to its per-pattern function. publisher|consumer|roundtrip +# stay spelled exactly (referenced by implement-mode.md Step 5). case "$STAGE" in publisher|consumer|roundtrip) run_guaranteed_stage ;; direct) run_direct_stage ;; direct-request-reply) run_request_reply_stage direct ;; guaranteed-request-reply) run_request_reply_stage guaranteed ;; + app) run_app_stage ;; *) echo "unknown stage: $STAGE" >&2; usage ;; esac @@ -801,7 +995,13 @@ cat "$PUB_LOG" echo "────────────────────────────────────────────────────────────────────" case "$status" in - 0) echo "PASS ($STAGE): the stage milestone was observed in the role's own log, the subscriber's shutdown hook ran ('$SHUTDOWN_LINE'), and the subscriber exited cleanly (exit 0 or 130)." ;; + 0) + if [ "$STAGE" = "app" ]; then + echo "PASS (app): the readiness and pass markers were observed in the app's own captured output, and the app was stopped with a bounded teardown." + else + echo "PASS ($STAGE): the stage milestone was observed in the role's own log, the subscriber's shutdown hook ran ('$SHUTDOWN_LINE'), and the subscriber exited cleanly (exit 0 or 130)." + fi + ;; 2) echo "ENV FAILURE ($STAGE): a JCSMP connection/auth signature matched in a process log (broker or credentials are wrong, not the code). NOT entering the fix loop." >&2 ;; *) if [ "$DIRECT_ZERO_RECEIPT" -eq 1 ]; then diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/solace-suggested-mode.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/solace-suggested-mode.md index 6a32f7d..3eaede4 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/solace-suggested-mode.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/solace-suggested-mode.md @@ -2,13 +2,13 @@ The Solace Suggested overlay for Implement mode. This file is reached from `implement-mode.md` Step 0 when the developer chooses the Solace Suggested path. It hardens whatever the design summary chose without re-describing how the chosen leaf is generated. Work through the steps in order. Read this file's grounding links only as each step needs them; link the canonical doc and state the decision in one line, never paraphrase doc content. -This overlay's Secure sub-mode assumes a Solace Cloud broker whose TLS server certificate is already configured and chains to a public CA; its Non-Secure sub-mode targets a dev or test broker that has no TLS configured. Solace Cloud is the primary broker default; the session is built from the connection details the developer pasted into a gitignored `config.json`. Step 1 forks that session build into Secure (a `tcps://` host with explicit certificate validation) and Non-Secure (a plaintext `tcp://` host with no certificate validation). The overlay does NOT provision or configure any broker (no DMQ, no queues, no TLS certificates). It states the broker-side prerequisites; it never sets them up. +This overlay is TLS-secure by definition. It assumes a broker whose TLS server certificate is already configured; Solace Cloud, the primary broker default, has TLS out of the box and chains to a public CA. There is NO non-secure variant of Solace Suggested. When the target broker has no TLS configured, do not proceed on this path: state that Solace Suggested requires a TLS-capable broker and route back to the `implement-mode.md` Step 0 door, which offers Custom with the equivalent knobs minus the secure session, or Quickstart. The session is built from the connection details the developer pasted into a gitignored `config.json`. The overlay does NOT provision or configure any broker (no DMQ, no queues, no TLS certificates). It states the broker-side prerequisites; it never sets them up. ## This is a leaf-agnostic overlay The Solace Suggested hardening is described ONCE here and applies on top of whatever `Pattern` leaf the design summary selected. It is not duplicated per leaf. The session-level deltas (TLS and HA failover) are identical code in every leaf, the message-level delta (DMQ eligibility) lands only on PERSISTENT sends, and the separate-projects layout applies to any two-role pattern (Publisher and Subscriber, or Requestor and Replier). -This overlay defers to `implement-mode.md` for all leaf and sample mechanics: the Step 2 design-summary input contract, the Step 4 leaf dispatch (which sample pair each leaf reads, the near-verbatim adaptation discipline), and the four-file `VERIFY:` marker contract. It ADDS only the Solace Suggested deltas below. It adds NO verify stages and changes NO `VERIFY:` marker string. The marker contract, the leaf dispatch table, and the Step 6 checklist emit and report all stay in `implement-mode.md` and run unchanged on the Solace Suggested path. +This overlay defers to `implement-mode.md` for all leaf and sample mechanics: the Step 2 design-summary input contract, the Step 4 leaf dispatch (which leaf wiring file and sample pair each leaf reads, the near-verbatim adaptation discipline), and the fixed `VERIFY:` marker contract. It ADDS only the Solace Suggested deltas below. It adds NO verify stages and changes NO `VERIFY:` marker string. The marker contract, the leaf dispatch table, and the checklist write (Step 4) and report (Step 6) all stay in `implement-mode.md` and its leaf wiring files, and run unchanged on the Solace Suggested path. Every generated class this overlay describes carries the AI-assisted disclaimer header (the exact `AI-assisted code. Review before production use.` line plus a pointer to the local `solace-verification-checklist.md`, as `implement-mode.md` Step 4 defines it) and keeps every import explicit. Never collapse imports to a wildcard such as `import com.solacesystems.jcsmp.*;`; explicit single-class imports only, in every generated class, on every leaf. This overlay names identifiers, constants, and method calls; it does not paste whole generated classes. @@ -18,13 +18,9 @@ The Solace Suggested path reads broker connection details from a `config.json` f Generate the classes directly, with no per-stage confirmation gate: write the Publisher and the Subscriber (or the Requestor and the Replier) the same way Quickstart does, then let the developer review the written files. Do NOT preview sensitive excerpts and ask the developer to confirm before writing each stage. The credential VALUES live only in the gitignored `config.json`, never in a generated class or the chat, so there is nothing credential-bearing to gate on; writing the generated code does not need a confirmation. -## Step 1: Build the session (Secure or Non-Secure) +## Step 1: Build the TLS secure session -This is the ONLY step that forks. When the developer chose Solace Suggested at `implement-mode.md` Step 0, Step 0 also asked whether they want Secure or Non-Secure. This overlay reads that Secure or Non-Secure flag and forks the session build here, and ONLY here. Every other step in this overlay is identical on both sub-modes: DMQ eligibility, HA failover, the two decoupled projects, the admin-provisioned-queue guidance, and the return to the implement-mode contracts all run the same way whichever sub-mode the developer picked. Non-Secure inverts exactly one thing versus Secure: it keeps the plaintext `tcp://` scheme and sets no certificate-validation property. - -### Secure: build the TLS secure session - -The pasted Solace Cloud `host` carries a `tcps://` scheme. `tcps://` is the secure transport: it runs the session over TLS. If the developer pasted a `tcp://` (plaintext) host, upgrade it to `tcps://` and state the TLS-already-configured assumption to them; never open a plaintext session on the Secure sub-mode. The plaintext `tcp://` scheme must not survive into the generated `host` value. +The pasted Solace Cloud `host` carries a `tcps://` scheme. `tcps://` is the secure transport: it runs the session over TLS. If the developer pasted a `tcp://` (plaintext) host, upgrade it to `tcps://` and state the TLS-already-configured assumption to them; never open a plaintext session on the Solace Suggested path. The plaintext `tcp://` scheme must not survive into the generated `host` value. If the broker genuinely has no TLS configured, this overlay does not fit: route back to the `implement-mode.md` Step 0 door (Custom minus the secure-session knob, or Quickstart) rather than degrading the session here. Set `JCSMPProperties.SSL_VALIDATE_CERTIFICATE = true` EXPLICITLY. The value is the JCSMP default, but setting it explicitly makes the secure posture auditable and visible in code review: a reviewer sees that server-certificate validation is on without having to know the default. For the same auditable reason, also set `JCSMPProperties.SSL_VALIDATE_CERTIFICATE_DATE = true` explicitly, so the not-expired check is equally visible. The two explicit lines together state the full validation posture. @@ -32,14 +28,6 @@ Do NOT generate a custom trust store. The Solace Cloud public-CA happy path fall Ground TLS in [Creating Secure Sessions](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Creating-Secure-Sessions.htm). -### Non-Secure: build a plaintext session - -Keep the plaintext `tcp://` scheme the developer pasted and build a plaintext session. Whatever `tcp://` host they pasted survives verbatim into the generated `host` value; do NOT upgrade it to `tcps://`. This is the sub-mode for a dev or test broker that has no TLS configured. - -Omit the certificate-validation properties ENTIRELY. A plaintext session has no server certificate to validate, so set NO `SSL_` property at all: no `SSL_VALIDATE_CERTIFICATE`, no `SSL_VALIDATE_CERTIFICATE_DATE`, and no trust store. The properties are simply absent. NEVER emit `SSL_VALIDATE_CERTIFICATE` set to `false`: the correct posture is absence, and a `false` value would be a misleading artifact that also reads as a disabled-validation anti-pattern. Setting the property to `false` is never the Non-Secure instruction; the property does not appear at all. - -State the caveat plainly to the developer: the Non-Secure sub-mode uses unencrypted transport, it fits a dev or test broker that has no TLS configured, and it is not appropriate for real credentials or real data over untrusted networks. Then drop ONE short comment at the session-build site in the generated class conveying the same: a single line noting the session is plaintext with no TLS and is intended for a dev or test broker only. Keep it to that one comment; do not scatter the caveat through the class. - ## Step 2: Ask the HA-failover question; upgrade the reconnect channel values on yes, keep the sample baseline on no After broker confirmation and before generating the session properties, ask the developer plainly: is HA failover set up? Ask it as a direct question and wait for the answer. @@ -58,18 +46,20 @@ At the call site, state the broker-side prerequisite as the developer's responsi ## Step 4: Generate two separate, independent Maven projects -The Solace Suggested path decouples the two roles. Generate TWO standalone single-module Maven projects, one per role (Publisher and Subscriber, or Requestor and Replier), each in its own project root with its OWN `pom.xml`. Each pom resolves `sol-jcsmp` live at generation time (the no-hardcoded-versions invariant), pins log4j2 to `2.26.0` (the one allowed pin, at or above the `2.17.1` Log4Shell floor), and declares exactly ONE `exec-maven-plugin` execution for that project's single main. +The Solace Suggested path decouples the two roles. The DEFAULT shape is TWO standalone single-module Maven projects, one per role (Publisher and Subscriber, or Requestor and Replier), each in its own project root with its OWN `pom.xml`. Each pom resolves `sol-jcsmp` live at generation time (the no-hardcoded-versions invariant), resolves its logging backend the same way per `implement-mode.md` Step 3 (no pinned versions; the `2.17.1` Log4Shell floor is enforced as a check, not a pin), and declares exactly ONE `exec-maven-plugin` execution for that project's single main. + +On this default shape there is NO parent POM and NO `` aggregator. Two independent projects, not a multi-module reactor: each project builds on its own. Recommend each project is self-contained, carrying its own `config.json` plus `.gitignore` (the same four lowercase `JCSMPProperties` property-name keys, the same gitignored-and-never-committed rule), consistent with two independent projects and no shared parent. -There is NO parent POM and NO `` aggregator. Two independent projects, not a multi-module reactor: each project builds on its own. Recommend each project is self-contained, carrying its own `config.json` plus `.gitignore` (the same four lowercase `JCSMPProperties` property-name keys, the same gitignored-and-never-committed rule), consistent with two independent projects and no shared parent. +**When the developer's ask requires a single deliverable** (one command, one jar, one web app with one dashboard), the two-project default conflicts with the ask. Surface that conflict plainly and let the developer resolve it; NEVER merge silently. Offer three resolutions: (1) keep two independent projects plus a thin launcher script that starts both; (2) one Maven build with separate publisher and consumer modules under a parent pom, which keeps the decoupling and still yields one command; (3) drop the decoupled-projects shape with the developer's explicit consent. Whichever they choose is recorded in the emitted `solace-verification-checklist.md` — a consented drop lands under Developer-owned items, so the deviation is written down, not lost. -The Solace Suggested path adds no new Maven dependency. Every Solace Suggested surface in this overlay (TLS, DMQ, HA) lives in the same `com.solacesystems:sol-jcsmp` JAR Quickstart already uses, plus the already-pinned log4j2; there is no new coordinate. +The Solace Suggested path adds no new Maven dependency. Every Solace Suggested surface in this overlay (TLS, DMQ, HA) lives in the same `com.solacesystems:sol-jcsmp` JAR Quickstart already uses, plus the logging backend Step 3 resolves; there is no new coordinate. Solace Suggested verification uses the SAME bundled `scripts/verify.sh` as Quickstart, run in its two-project mode. `verify.sh` accepts per-role project directories, so it can drive the decoupled layout: it compiles each project, starts the long-running role first and the foreground role second, watches each process for the fixed `VERIFY:` markers, exercises the SIGINT graceful-shutdown hook, and classifies the outcome by the same 0/1/2 exit codes and bounded fix loop that `implement-mode.md` Step 5 defines. In two-project mode it passes NO connection details on the command line; each app reads its own gitignored `config.json`, so the broker credentials never appear in a process listing (the checklist's credential-handling item). Invoke it with the role-directory flags instead of the positional connection args: - pub/sub leaves: `scripts/verify.sh roundtrip --subscriber-dir --publisher-dir ` - request-reply leaves: `scripts/verify.sh guaranteed-request-reply --replier-dir --requestor-dir ` (use `direct-request-reply` for the direct leaf). -Run it for real under the SAME consent Quickstart uses: only when each project's `config.json` is present with REAL values (not the `config.example.json` placeholders). When any project's `config.json` is absent or still holds placeholders, do NOT fabricate a run: compile each project (`mvn -q compile` per root), report the compile result, and hand back the two-project `verify.sh` command for the developer to run themselves (the honesty rule). A live Solace Suggested round-trip needs a TLS-capable broker, since the `tcps://` Solace Suggested session validates the server certificate; a plaintext or unreachable broker surfaces as an environment failure (exit 2), which `verify.sh` classifies without entering the fix loop, so it never masquerades as a code bug. +Run it for real under the SAME consent Quickstart uses: only when each project's `config.json` is present with REAL values (not the `config.example.json` placeholders). Immediately before any live run, print the one-line live-broker heads-up (SKILL.md Invariant 6): name the broker host and state the run's actual broker-side effects (the queues it provisions, the connections it opens, the messages it publishes); the real-values `config.json` is the consent, the heads-up is not a prompt. When any project's `config.json` is absent or still holds placeholders, do NOT fabricate a run: compile each project (`mvn -q compile` per root), report the compile result, and hand back the two-project `verify.sh` command for the developer to run themselves (the honesty rule). A live Solace Suggested round-trip needs a TLS-capable broker, since the `tcps://` Solace Suggested session validates the server certificate; a plaintext or unreachable broker surfaces as an environment failure (exit 2), which `verify.sh` classifies without entering the fix loop, so it never masquerades as a code bug. ## Step 5: Topic architecture @@ -77,7 +67,7 @@ For how to structure, order, name, and place wildcards in the topic hierarchy, p ## Step 6: Return to the implement-mode contracts -After the Solace Suggested deltas above, return to `implement-mode.md` for the unchanged leaf mechanics: the Step 4 dispatch generates the chosen leaf's classes, and Step 6 resolves each `verification-checklist.md` item to its state for the chosen mode, writes the tailored `solace-verification-checklist.md` into the project (one copy per project root on this overlay's decoupled two-project layout), invites the developer's additions, and reports the same in chat, on the Solace Suggested path the same way it does on Quickstart. This overlay adds the Solace Suggested hardening; it does not replace those contracts. +After the Solace Suggested deltas above, return to `implement-mode.md` for the unchanged leaf mechanics: the Step 4 dispatch generates the chosen leaf's classes and writes the tailored `solace-verification-checklist.md` into the project as generation output (one copy per project root on this overlay's decoupled two-project layout), and Step 6 resolves each item to its state for the chosen mode, invites the developer's additions, and reports the same in chat, on the Solace Suggested path the same way it does on Quickstart. This overlay adds the Solace Suggested hardening; it does not replace those contracts. ## Grounding references diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/verification-checklist.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/verification-checklist.md index 067c402..4c548db 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/verification-checklist.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/verification-checklist.md @@ -2,19 +2,19 @@ This checklist records what the chosen mode delivered and what remains the developer's responsibility. Each item is one binary (yes/no) check with one canonical Solace doc link. Read the linked doc to decide; the check is not a substitute for the doc. -This file is the master template. Every Implement run emits a per-project copy into the generated project, tailored to the chosen mode and the code that was generated. Nothing here implies the generated output is ready to carry real traffic; each mode delivers a baseline, and the items below name what it delivered and what stays the developer's job. The three groups make that split explicit. "Delivered by this generation" names what the chosen mode generated into the code, so on that mode those items report as delivered; each carries a per-mode branch note because the same item is the developer's responsibility on a mode that did not generate it (for example, the secure-session item on Quickstart, which connects over plaintext). "Your responsibility (not delivered here)" names what no generation satisfies for the chosen mode, so those items stay the developer's responsibility. "Verified by the round-trip" names the conformance checks the publisher to consumer round-trip already exercises. +This file is the master template. Every Implement run emits a per-project copy into the generated project at generation time (implement-mode.md Step 4), tailored to the chosen mode and the code that was generated. Nothing here implies the generated output is ready to carry real traffic; each mode delivers a baseline, and the items below name what it delivered and what stays the developer's job. The responsibility groups make that split explicit. "Delivered by this generation" names what the chosen mode generated into the code, so on that mode those items report as delivered; each carries a per-mode branch note because the same item is the developer's responsibility on a mode that did not generate it (for example, the secure-session item on Quickstart, which connects over plaintext). "Your responsibility (not delivered here)" names what no generation satisfies for the chosen mode, so those items stay the developer's responsibility. "Verified by the round-trip" names the conformance checks the publisher to consumer round-trip already exercises. ## Delivered by this generation -- [ ] The session is a `tcps://` secure TLS session with server-certificate validation on (`SSL_VALIDATE_CERTIFICATE = true` set explicitly), no custom trust store on the Solace Cloud public-CA happy path. Branch note: Delivered by this generation on Solace Suggested Secure and on Custom with the secure-session knob ticked. Your responsibility on Quickstart, on Solace Suggested Non-Secure (plaintext `tcp://`, no certificate to validate), and on Custom with the secure-session knob unticked. [Creating Secure Sessions](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Creating-Secure-Sessions.htm) -- [ ] Each role is its own independent Maven project with its own pom and NO parent POM (Publisher and Subscriber, or Requestor and Replier, are decoupled into separate projects). Branch note: Delivered by this generation on Solace Suggested (both Secure and Non-Secure) and on Custom with the decoupled-projects knob ticked. Your responsibility on Quickstart, which ships both classes in one project with two mains, and on Custom with the decoupled-projects knob unticked. [JCSMP Best Practices](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Best-Practices.md) +- [ ] The session is a `tcps://` secure TLS session with server-certificate validation on (`SSL_VALIDATE_CERTIFICATE = true` set explicitly), no custom trust store on the Solace Cloud public-CA happy path. Branch note: Delivered by this generation on Solace Suggested (always TLS; it has no non-secure variant) and on Custom with the secure-session knob ticked. Your responsibility on Quickstart and on Custom with the secure-session knob unticked. [Creating Secure Sessions](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Creating-Secure-Sessions.htm) +- [ ] Each role is its own independent Maven project with its own pom and NO parent POM (Publisher and Subscriber, or Requestor and Replier, are decoupled into separate projects). Branch note: Delivered by this generation on Solace Suggested and on Custom with the decoupled-projects knob ticked. Your responsibility on Quickstart, which ships both classes in one project with two mains, and on Custom with the decoupled-projects knob unticked. [JCSMP Best Practices](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Best-Practices.md) - [ ] Every PERSISTENT publish calls `setDMQEligible(true)` so an undeliverable message can move to the broker dead message queue. Branch note: Delivered by this generation on Solace Suggested (the guaranteed leaves that carry PERSISTENT sends) and on Custom with the DMQ-eligibility knob ticked. Your responsibility otherwise: on Quickstart, on the direct at-most-once leaves where the flag does not apply, and on Custom with the DMQ-eligibility knob unticked. [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md) - [ ] The four HA-failover channel values are set (`connectRetries` 1, `reconnectRetries` 20, `reconnectRetryWaitInMillis` 3000, `connectRetriesPerHost` 5). Branch note: Delivered by this generation on Solace Suggested when the HA-failover question is answered yes, and on Custom with the HA-channel-values knob ticked and that question answered yes. Otherwise the sample keeps its reconnect baseline (`reconnectRetries` 20, `connectRetriesPerHost` 3) and tuning the four HA values is Your responsibility. [JCSMP Best Practices](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Best-Practices.md) ## Your responsibility (not delivered here) - [ ] A dead message queue is provisioned on the broker with a max-redelivery count and/or a TTL (the `setDMQEligible` flag alone does nothing without a broker-side DMQ; the skill never provisions one). [JCSMP Best Practices](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Best-Practices.md) -- [ ] The durable queue is admin-provisioned rather than provisioned by the application at startup. Branch note: Delivered by this generation only on Custom with the admin-provisioned-queue knob ticked, where the generated app binds a pre-existing admin-owned queue instead of self-provisioning. Your responsibility on Quickstart, on Solace Suggested (both sub-modes, where the app self-provisions its own queue and the overlay gives admin-provisioned-queue guidance only, generating no admin-queue binding code), and on Custom with the admin-provisioned-queue knob unticked. [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md) +- [ ] The durable queue is admin-provisioned rather than provisioned by the application at startup. Branch note: Delivered by this generation only on Custom with the admin-provisioned-queue knob ticked, where the generated app binds a pre-existing admin-owned queue instead of self-provisioning. Your responsibility on Quickstart, on Solace Suggested (where the app self-provisions its own queue and the overlay gives admin-provisioned-queue guidance only, generating no admin-queue binding code), and on Custom with the admin-provisioned-queue knob unticked. [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md) - [ ] Credentials are handled safely (each `config.json` is gitignored and never committed; credentials are not passed as command-line arguments where they land in shell history or a process listing, as the Quickstart verify flow does for convenience). [Defining Client Authentication](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Defining-Client-Authentication.md) - [ ] Authentication stronger than basic username/password (for example a client certificate or Kerberos) is set up where the environment requires it; this journey generates basic username/password only. [Defining Client Authentication](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Defining-Client-Authentication.md) - [ ] The consumer calls `ackMessage()` only after the message is fully processed (CLIENT acknowledgement, not auto-ack). [Acknowledging Messages](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Acknowledging-Messages.md) @@ -25,6 +25,18 @@ This file is the master template. Every Implement run emits a per-project copy i - [ ] Messages are published with PERSISTENT (Guaranteed) delivery for the guaranteed journey. [Message Delivery Modes](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Message-Delivery-Modes.md) - [ ] The publisher to consumer round-trip is verified against a reachable broker: a PERSISTENT message published to the topic lands on the subscribed durable queue and is consumed and acknowledged. [Provisioning a Durable Endpoint](https://docs.solace.com/API/API-Developer-Guide-JCSMP/JCSMP-API-Provisioning-Durable-End.md) +## Generation conformance (checked by the verify.sh preflight) + +These are mechanical conformance checks on the generated output itself; the copied `verify.sh` reports the first four in its preflight, so a miss is visible on every run. They carry no doc link because they check skill conformance, not Solace guidance. + +- [ ] Every generated source file starts with the exact line `AI-assisted code. Review before production use.` followed by the checklist pointer line. +- [ ] Exactly one logging backend is configured with a config resource on the classpath; any `log4j-core` on the classpath (including transitive) is at or above `2.17.1`; the `com.solacesystems` loggers are not silenced below INFO. +- [ ] The `sol-jcsmp` version in the pom matches the authoritative `` from the `repo1.maven.org` Maven metadata at generation time (never the solrsearch index). +- [ ] This tailored checklist file exists at every generated project root. +- [ ] Every construct carried from a reference sample keeps that construct's comment (names adapted); dropped demo-harness code took its comments with it; fresh messaging code that applies a documented practice carries a short comment naming it (the comments-follow-their-code contract, implement-mode.md Step 4). +- [ ] A `verify.sh` stage ran and its stage name and exit code are recorded here: ` → exit ` (or: the compile-only fallback was taken and the exact commands were handed back). +- [ ] Every page named in the design summary's Grounding docs field was WebFetched in this session (`none fetched` is an honest value; an unfetched citation is not). + ## Developer-owned items (not delivered by this skill) Record here any item the skill does not deliver: non-Solace concerns, out-of-scope concerns, or anything you are tracking on your own. Each is marked developer-owned rather than skill-satisfied, so at a glance a reader sees which items the skill stands behind and which you own. In this master template the section is a skeleton; the per-project copy emitted into a generated project is where concrete developer-owned items are recorded during a run. From 570cf591278407a13152671c1a24174ea54f9450 Mon Sep 17 00:00:00 2001 From: Adi El-Sammak <71227923+aelsammak@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:06:35 -0400 Subject: [PATCH 4/5] EBP-2938: Add output evals for solace-messaging-skills (#6) * EBP-2938: Add output-contract evals for solace-messaging-skills - Add tools/run-output-evals.sh, a local runner that plays scripted multi-turn cases through the Claude Code CLI with only the plugin under test loaded, then grades the transcript and the generated files - Add plugins/solace-messaging-skills/evals/output-evals.json with 11 cases across the three skills, 5 marked must_pass - Grade with deterministic checks (assistant text, tool use, generated files, mvn compile, live sol-jcsmp release match) and a fixed-model LLM judge for routing and grounding criteria - Add an opt-in live_verify grader that runs the generated project's verify.sh against a broker named by OUTPUT_EVAL_BROKER_* variables and skips loudly when they are unset - Document the corpus format, run instructions, gate, and local-only status in the evals README - Model the runner on tools/run-trigger-evals.sh (isolation via --plugin-dir and a scratch CLAUDE_CONFIG_DIR, 90% gate, must_pass) * EBP-2938: Fix two skill contract gaps found by the output evals - Harden the Debug row in solace-application-development/references/jcsmp.md so the documentation redirect is the entire debug answer; memory-derived hypotheses, interim checks, and fixes are forbidden - State in solace-topic-best-practices/SKILL.md that the skill answers in chat only and never writes files - Both gaps surfaced as failing output-eval cases (appdev-debug-redirect-negative and topics-no-files-negative); both cases pass after these edits * EBP-2938: Simplify output-eval docs, broker input, and case prompts - Remove exact case counts from the evals README so the prose does not drift from the corpus - Remove the "No CI wiring" section from the evals README - Take broker credentials for live_verify from exported OUTPUT_EVAL_BROKER_* variables only; drop the optional env file - Report in the run summary whether live_verify executed, skipped, or never ran because the case failed before grading - Drop the run.sh and port details from two application-development case prompts; no grader depended on them * EBP-2938: Address review findings on the output-eval runner and corpus - Replace the Quickstart failover word check with a code grep for the reconnect block; the Step 6 checklist report legitimately names HA failover - Run the LLM judge tool-free (--tools "") from the run directory so a tool call cannot spend its only turn and project settings stay out of the judge session - Add a compile grader to the Solace Suggested case and make compile and maven_release_match iterate every generated pom for two-project layouts - Align the debug-redirect judge with the Debug row: fail troubleshooting from any source, treat a restated symptom as the acknowledgement - Accept bold field labels in the design summary greps (design mode leaves formatting to the model) - Validate OUTPUT_EVAL_RUNS as a positive integer (a zero value crashed bash 3.2 on an empty array expansion) - Classify a timeout kill (exit 124) as INFRA in compile and live_verify - Carry the cause onto turn-level INFRA lines and print a RETRY line before the second attempt - Show the live-verify skip tag on FAIL lines; document that a max_turns hit fails the case - Drop the run-duration claim from the evals README; mention output evals in the root README Verified locally: claude-sonnet-5 and claude-opus-5 legs over the six affected cases pass, with live verify green against a real broker on both models. * EBP-2938: Add a corpus JSON Schema and rename the turn fields - Add tools/output-evals.schema.json (JSON Schema draft 2020-12) that defines the output-eval corpus with per-grader field rules and a description for every field, in response to review feedback - Rename the corpus fields turns to user_turns and max_turns to max_agent_turns to separate the scripted user messages from the CLI's agent-turn budget; update the runner and the corpus accordingly - Point the evals README and the runner header at the schema; the runner keeps enforcing the structural subset at start-up - Add a prerequisites list and a transcript-to-corpus mapping table (observed with Claude Code 2.1.266) to the evals README * EBP-2938: Document the timeout dependency and the credential requirement - Resolve GNU timeout with a gtimeout fallback for the mvn compile and verify.sh bounds, and warn once when neither is on PATH and a selected case needs Maven, instead of running unbounded silently - List timeout as an optional prerequisite in the evals README with the unbounded fallback and the macOS install hint - State in both README credential notes that an existing claude login does not satisfy the runner, because the throwaway config directory carries no login * EBP-2938: Treat permission denials as INFRA and pin LF endings for scripts - Read permission_denials from every turn's result event: a denial of a tool the runner grants is INFRA at once, with the tool and command on the case line and no retry, because the same policy would deny the retry; a denial of any other tool is tagged on the case line - Correct the header claim that the allowlist prevents denials; managed settings, hooks, and command shims can still deny a granted tool, and a denied cp became a lossy retype in a Windows run - Add .gitattributes with *.sh text eol=lf so Windows checkouts keep shell scripts LF for bash and for the byte-identity grader on verify.sh - Document Git Bash and LF on Windows, and denials as INFRA, in the evals README * EBP-2938: Require repro steps and a Read-only tool contract in the feedback skill - Make Steps to reproduce mandatory for bug-shaped feedback and derive it from the report when the session holds no skill invocation; a Windows sonnet run dropped the section under the old wording - Drop the Notes heading when empty and forbid None or N/A filler under it - State that Read is the skill's only tool and that the plugin manifest is read at its announced path, never located with a shell command; three-run checks showed sonnet running find for the manifest - Add a grader to feedback-support-email-positive that forbids a line consisting only of None - Three sonnet runs of the case pass by majority vote after these edits * EBP-2938: Copy the bundled verify.sh with cat instead of cp - Prescribe the copy command in implement-mode Step 4: cat the bundled script into the project root and chmod it, and forbid cp on the bundled file - Claude Code protects the plugin directory and blocks a cp that names a file inside it, even with Bash allowed; the blocked copy fell back to a retype of the script that must stay byte-identical, and the runner now reports that denial as INFRA - Reproduced on macOS with the runner's flags: cp denied, cat redirect allowed and byte-identical - Two sonnet runs of appdev-quickstart-implement-full with this wording used the cat form with zero denials; the second passed every grader * EBP-2938: Ignore no-op commands and summary remarks in the output evals - Add exclude_pattern to the four bare Bash-absent graders so a bare echo, true, or : (plus ls and pwd on the topics negative) no longer fails a case; a redirect, a chained command, a search, or a transport call still does - Sonnet ended feedback turns with echo done in about a third of runs and opus listed the directory once before it refused to write the file, while the graded content was correct every time - Tell the grounding judge that deriving the consequences of the fetched rules for the scenario is applying the guidance, and tell every judge that WebFetch results are the tool's summaries, not the page; a summary remark that guidance was absent had failed a correct opus answer - Correct the README Windows note: git checkout -- . leaves a stale CRLF checkout in place because git treats the files as unchanged; document the delete-and-checkout recipe * EBP-2938: Flag decisions beyond the fetched page in the topics skill - Add one sentence to step 3: when the user's scenario needs a decision the fetched page does not cover, say so before suggesting one - The grounding judge already accepts flagged extrapolation, and sonnet folded command topics into the page's template unflagged in 2 of 4 runs - Fifteen single sonnet runs of topics-grounded-answer pass after the edit; on the final tree the full sonnet leg is 11/11 and the opus leg 10/11, live verify skipped --- .gitattributes | 4 + README.md | 8 +- .../solace-messaging-skills/evals/README.md | 68 +- .../evals/output-evals.json | 251 ++++++ .../references/jcsmp.md | 2 +- .../references/jcsmp/implement-mode.md | 2 +- .../skills/solace-messaging-feedback/SKILL.md | 7 +- .../solace-topic-best-practices/SKILL.md | 5 +- tools/output-evals.schema.json | 185 +++++ tools/run-output-evals.sh | 725 ++++++++++++++++++ 10 files changed, 1247 insertions(+), 10 deletions(-) create mode 100644 .gitattributes create mode 100644 plugins/solace-messaging-skills/evals/output-evals.json create mode 100644 tools/output-evals.schema.json create mode 100755 tools/run-output-evals.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fbf03fa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Shell scripts must check out with LF endings on every platform. Bash rejects +# CRLF scripts, and the output evals compare a generated verify.sh byte for byte +# against the bundled reference copy. +*.sh text eol=lf diff --git a/README.md b/README.md index ae43d03..c94a5b6 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ agent-plugins/ ├── plugins/ # Claude plugins, one subdirectory per plugin │ └── solace-messaging-skills/ # The messaging skills plugin │ ├── .claude-plugin/ # Plugin definition (plugin.json) -│ ├── evals/ # Trigger eval corpus (trigger-evals.json) and its README +│ ├── evals/ # Trigger and output eval corpora and their README │ └── skills/ # Individual skill definitions (shared by all agents) │ ├── solace-application-development/ # Solace application development umbrella skill │ │ ├── SKILL.md # Skill routing and instructions @@ -62,15 +62,17 @@ agent-plugins/ │ │ └── SKILL.md # Navigation-only manifest; reads the live docs page │ └── solace-messaging-feedback/ # Feedback formatter skill │ └── SKILL.md # Formats and routes session feedback (Support, Community, or Ideas portal) -├── tools/ # Re-runnable scripts (check-links, run-trigger-evals) +├── tools/ # Re-runnable scripts (check-links, run-trigger-evals, run-output-evals) ├── README.md # This file └── .claude-plugin/ # Claude marketplace definition (marketplace.json) ``` -## Trigger evals +## Trigger and output evals Each plugin ships a trigger eval corpus under `plugins//evals/` that checks each skill fires on the prompts it should and stays silent on the prompts it should not. See the [trigger evals README](plugins/solace-messaging-skills/evals/README.md) for the corpus format, how to run the evals locally, and how they run in CI. +Each plugin also ships an output eval corpus in the same directory that checks a skill's output honors the skill contract (design before code, the door question, scrubbed feedback drafts, grounded answers). Output evals run locally only. See the [output evals section](plugins/solace-messaging-skills/evals/README.md#output-evals) of the same README for the corpus format, the graders, and how to run them. + ## License This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for details. diff --git a/plugins/solace-messaging-skills/evals/README.md b/plugins/solace-messaging-skills/evals/README.md index a8d7aa0..e8b4661 100644 --- a/plugins/solace-messaging-skills/evals/README.md +++ b/plugins/solace-messaging-skills/evals/README.md @@ -4,7 +4,7 @@ Each plugin ships a trigger eval corpus at `plugins//evals/trigger-evals ## Running the evals locally -To run the evals manually, you need the `claude` CLI and `jq` on your PATH, plus an exported credential (the runner uses a throwaway config directory with no ambient login). Run the script from the repository root: +To run the evals manually, you need the `claude` CLI and `jq` on your PATH, plus an exported credential. An existing `claude` login does not satisfy this: the runner uses a throwaway config directory so that only the plugin under test is loaded, and that directory has no login, so `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` must be exported. Run the script from the repository root: ```shell export ANTHROPIC_API_KEY= # or CLAUDE_CODE_OAUTH_TOKEN @@ -17,3 +17,69 @@ Each prompt runs three times and the verdict is the majority result. Set `TRIGGE ## Continuous integration In GitHub Actions, the `trigger-evals` job in `.github/workflows/ci.yml` runs this same script on every pull request as a two model matrix (`claude-haiku-4-5` and `claude-sonnet-5`), each model an independent check, and self-skips green when the `ANTHROPIC_API_KEY` secret is absent. + +# Output evals + +Trigger evals ask whether the right skill fires. Output evals ask the next question: does the skill's output honor the skill contract? The corpus at `plugins//evals/output-evals.json` holds the cases for the plugin's skills. Each case carries an `id`, the target `skill`, an ordered `user_turns` array of scripted user messages (multi-turn cases continue one session via `--resume`), an optional `max_agent_turns` budget (the model's replies and tool calls allowed per user turn), an optional `must_pass` flag, and a `graders` array. A case passes only when the target skill fired, every grader passed, and no user turn hit the `max_agent_turns` cap. The full structure, with the intent of every field, is defined in `tools/output-evals.schema.json` (JSON Schema, draft 2020-12). The runner enforces the structural subset of that schema at start-up (array shape, required fields, unique ids, known grader types) and exits before any API call when the corpus violates it. Validating an edit against the full schema, including the per-grader field rules, is a manual step with any JSON Schema validator for now. + +For the application-development skill, the cases cover these contract points end to end: the broker-access question, design mode entered rather than skipped, `solace-design.md` saved on request by the end of design mode, the three-way door question (Quickstart, Solace Suggested, Custom), the Solace Suggested secure connection and HA failover question, the Quickstart posture (plaintext capable, single project, no HA question), and the AI-generated notice at the top of every generated file. The compile case also carries an opt-in live round trip: when the four `OUTPUT_EVAL_BROKER_*` variables are set, the runner executes the generated project's own `verify.sh` against that broker. Without them the grader skips, the run says so up front and in its summary, and no case ever requires broker credentials. + +Two grader families exist. Deterministic graders (`assistant_grep`, `tool_use`, `file_exists`, `file_grep`, `java_disclaimer`, `compile`, `maven_release_match`, `live_verify`) check the transcript and the generated files mechanically, including a real `mvn compile` of the generated project, a match of the generated pom against the live sol-jcsmp `` on Maven Central, and, when a broker is configured, a real `verify.sh roundtrip` of the generated project. The `llm_judge` grader sends the transcript to a fixed judge model for routing and grounding criteria that a grep cannot decide. Negative cases assert the absence of forbidden behavior: code before a design confirm, a support email before the support-contract confirmation, unscrubbed identifiers in a feedback draft, and memory-derived debug steps. Run `./tools/run-output-evals.sh --help` for the full grader reference. + +## How the transcript maps onto the corpus + +Each `turnN.jsonl` in the work directory is the raw `--output-format stream-json --verbose` stream of one `claude -p` invocation, one file per entry in `user_turns`. The runner reads only the fields in this table. The shapes were observed with Claude Code 2.1.266 on 2026-09-09. Re-check this table after a CLI upgrade, because a change to these fields breaks the graders silently. + +| Corpus field or runner concept | Source in `turnN.jsonl` | +|---|---| +| `user_turns[i]` | Not in the file. Entry `i` is the `-p` prompt of invocation `i`, and the stream never echoes it. The only `user` events are tool results. Each entry produces its own `turn.jsonl`. | +| `max_agent_turns` | Passed as `--max-turns`. The `result` event reports `num_turns` (agent turns used, one per assistant message cycle) and sets `subtype` to `error_max_turns` when the cap is hit. | +| Grader `turn` | Selects which `turn.jsonl` file the grader reads. It is not a field inside the file. | +| `skill` (the implicit skill-fired gate) | An `assistant` event whose `message.content[]` holds a `tool_use` block with `name` `Skill`. Its `input.skill` is `plugin:skill-name`; the runner strips the prefix. | +| `assistant_grep` | `assistant` events, `message.content[]` blocks with `type` `text`. Thinking blocks and tool payloads are excluded. | +| `tool_use` | `assistant` events, blocks with `type` `tool_use`. The runner matches `name` and greps the compact JSON of `input`. | +| `llm_judge` with `webfetch_urls` or `webfetch_results` | WebFetch `tool_use` blocks supply the URLs. `user` events whose `tool_result.tool_use_id` matches a WebFetch block supply the fetched content. | +| Session continuity (`--resume`) | `session_id` on the `result` event, with the `init` event as the fallback. | +| INFRA verdict for a turn | A missing `result` event, or `result.is_error` true with a `subtype` other than `error_max_turns`. | + +## Running the output evals locally + +### Prerequisites + +- The `claude` CLI (Claude Code) on your PATH. +- `jq` and `curl` on your PATH. +- `mvn` (Apache Maven) with a JDK 11 or newer on your PATH. Only the cases that compile generated code need it, and the runner checks for it only when such a case is selected. +- An exported credential, either `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`. An existing `claude` login does not satisfy this: the runner uses a throwaway config directory so that only the plugin under test is loaded, and that directory has no login. Claude seat holders can mint a token with `claude setup-token`. +- `timeout` from GNU coreutils (or Homebrew's `gtimeout`) on your PATH, to cap each `mvn compile` and `verify.sh` run at 600 seconds. Optional: without it those steps run unbounded, and the runner prints a warning. On macOS, install it with `brew install coreutils`. +- Network access to `repo1.maven.org` (Maven Central metadata and dependencies) and `docs.solace.com` (the skills fetch documentation pages). +- Optional: the four `OUTPUT_EVAL_BROKER_*` variables for the live round trip against a real broker (see below). +- On macOS, `caffeinate` to keep the machine awake for a full leg. +- On Windows, Git Bash. The repository's `.gitattributes` checks shell scripts out with LF endings, which bash and the byte-identity grader on `verify.sh` both need. A checkout made with `core.autocrlf=true` before that file existed keeps its CRLF copies, and git reports the tree clean because it treats them as unchanged, so `git pull` and `git checkout -- .` do not rewrite them. Re-clone, or delete the tracked scripts and check them out again: `git ls-files -z '*.sh' | xargs -0 rm -f && git checkout -- .`. + +Run from the repository root: + +```shell +export ANTHROPIC_API_KEY= # or CLAUDE_CODE_OAUTH_TOKEN +./tools/run-output-evals.sh # defaults to claude-sonnet-5 +./tools/run-output-evals.sh --model claude-opus-5 +./tools/run-output-evals.sh --case appdev-quickstart-implement-full +``` + +Run the suite for both `claude-sonnet-5` and `claude-opus-5` before a PR that affects skill content, and record both results in the PR description, including whether live verify ran or skipped (the runner prints this in its summary). A full leg is a long run; keep the machine awake for it (on macOS, prefix the command with `caffeinate -i`), because a sleep mid-run surfaces as INFRA failures. A permission denial of a tool the runner grants (Bash, Write, and the rest of its allowlist) is also INFRA: managed settings, hooks, or local command shims can deny a command, the model then improvises, and the result no longer measures the skill. The case line names the denied tool and command; adjust the policy for the run instead of reading the result as a skill failure. Each case runs once by default (`OUTPUT_EVAL_RUNS=1`), because a full implement-flow case is expensive; raise it for a majority-vote stability study. `OUTPUT_EVAL_JUDGE_MODEL` (default `claude-sonnet-5`) stays fixed across subject models so leg differences are attributable to the subject. `OUTPUT_EVAL_WORKDIR` receives transcripts and generated projects, and the work directory is kept and printed when the run fails. + +The gate matches the trigger evals: a run passes when at least 90% of cases pass, any infrastructure failure fails the run, and any `must_pass` failure fails the run regardless of the pooled rate. The forbidden-behavior negatives and the compile case are `must_pass`. + +### Live verification against a real broker + +The compile case's `live_verify` grader runs the generated project's `verify.sh roundtrip` against a broker you name. It is opt-in and needs all four of these variables exported in your shell: + +```shell +export OUTPUT_EVAL_BROKER_HOST=tcp://:55555 +export OUTPUT_EVAL_BROKER_VPN= +export OUTPUT_EVAL_BROKER_USER= +export OUTPUT_EVAL_BROKER_PASSWORD= +``` + +Do not put these values in a file inside the repository; the repository is public. The runner strips the export attribute from the four variables so the values never reach the subject model, and hands them to `verify.sh` only as CLI arguments. + +The runner prints the live-verify state before the first case starts and again in the summary. When none of the four variables is exported, the grader skips and the case line says so. When some but not all are exported, the runner exits with an error before any case runs. When `verify.sh` reports a broker or credential error (its exit code 2), the case is an INFRA failure, not a skill failure. Use a dedicated eval message VPN or service: the generated subscriber provisions a durable queue on every run, so queues accumulate, and the client username needs permission to create endpoints. diff --git a/plugins/solace-messaging-skills/evals/output-evals.json b/plugins/solace-messaging-skills/evals/output-evals.json new file mode 100644 index 0000000..7b2a7ac --- /dev/null +++ b/plugins/solace-messaging-skills/evals/output-evals.json @@ -0,0 +1,251 @@ +[ + { + "id": "appdev-design-summary-positive", + "skill": "solace-application-development", + "user_turns": [ + "I'm planning a Java system on an event broker: when a flight goes delayed, one delay event must update three independent downstream services (gate displays, the crew scheduler, and passenger notifications). Each service needs its own copy of every event and none of them may miss one, even if it is briefly down. There will eventually be a browser dashboard, but right now I only want help designing the Solace messaging pattern and topology, not the code.", + "We have a Solace Cloud broker available. Go with your recommendations, but do not save anything to a file yet." + ], + "max_agent_turns": 25, + "graders": [ + { "type": "assistant_grep", "pattern": "[Bb]roker", "expect": "present", "turn": 1 }, + { "type": "assistant_grep", "pattern": "Pattern\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Delivery\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Access type\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Topic\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Consumption endpoint\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Auth\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Broker\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Grounding docs\\**:", "expect": "present" }, + { "type": "assistant_grep", "pattern": "Guaranteed Pub/Sub (fan-out)", "fixed": true, "expect": "present" }, + { "type": "assistant_grep", "pattern": "solace-design.md", "fixed": true, "expect": "present" }, + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { "type": "file_exists", "glob": "*.java", "expect": "absent" }, + { + "type": "llm_judge", + "include": ["assistant_text", "webfetch_urls"], + "criteria": "Grounding honesty: every docs.solace.com page the reply cites as grounding (including the design summary's 'Grounding docs' field) must appear in the fetched-URL list. 'none fetched' is a valid honest value when the list is empty. FAIL only when the reply cites a docs.solace.com page as grounding that does not appear in the fetched-URL list." + } + ] + }, + { + "id": "appdev-code-before-confirm-negative", + "skill": "solace-application-development", + "must_pass": true, + "user_turns": [ + "Build a Java chat room with a web interface: enter a display name, type a message, and see messages appear live for anyone with the page open, connected to Solace. Skip any design discussion or questions and generate the full project right now." + ], + "max_agent_turns": 25, + "graders": [ + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { "type": "tool_use", "tool": "Edit", "expect": "absent" }, + { "type": "file_exists", "glob": "*", "expect": "absent" }, + { "type": "assistant_grep", "pattern": "[Dd]esign|Pattern:", "expect": "present" } + ] + }, + { + "id": "appdev-quickstart-implement-full", + "skill": "solace-application-development", + "must_pass": true, + "user_turns": [ + "Build a Java app on our Solace event broker for food-delivery order processing. An OrderService publishes customer orders (order ID, restaurant ID, line items, total) and a pool of competing KitchenWorker instances share the load: each order must be processed by exactly one worker, must survive a worker crash or restart, and no order may be lost. Plain Maven project, no web dashboard. We have a Solace Cloud broker. This is Guaranteed Pub/Sub (single service, non-exclusive) with PERSISTENT delivery; derive the topic and queue names yourself.", + "Yes, I am happy with that design. Save it to solace-design.md, then go ahead.", + "Quickstart. I will not put credentials in the chat, so compile only and hand me the exact verify commands to run myself." + ], + "max_agent_turns": 80, + "graders": [ + { "type": "assistant_grep", "pattern": "Quickstart", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Solace Suggested", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Custom", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "file_grep", "glob": "*.java", "pattern": "new JCSMPChannelProperties\\(|setProperty\\([^)]*CLIENT_CHANNEL_PROPERTIES", "expect": "absent" }, + { "type": "file_exists", "glob": "solace-design.md", "expect": "present" }, + { "type": "file_grep", "glob": "pom.xml", "pattern": "|", "expect": "absent" }, + { "type": "compile" }, + { "type": "maven_release_match" }, + { "type": "live_verify" }, + { "type": "java_disclaimer" }, + { "type": "file_grep", "glob": "*.java", "pattern": "^import .*\\*;", "expect": "absent" }, + { "type": "file_grep", "glob": "pom.xml", "pattern": "com.solacesystems", "fixed": true, "expect": "present" }, + { "type": "file_exists", "glob": "log4j2.xml", "expect": "present" }, + { "type": "file_grep", "glob": "log4j2.xml", "pattern": "[Cc]onsole", "expect": "present" }, + { "type": "file_grep", "glob": "log4j2.xml", "pattern": "com\\.solacesystems.*(WARN|ERROR|OFF|FATAL)|(WARN|ERROR|OFF|FATAL).*com\\.solacesystems", "expect": "absent" }, + { "type": "file_grep", "glob": "*.java", "pattern": "VERIFY: CONNECTED", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "*.java", "pattern": "VERIFY: QUEUE_BOUND", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "*.java", "pattern": "VERIFY: PUBLISH_ACKED", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "*.java", "pattern": "VERIFY: MESSAGE_RECEIVED", "fixed": true, "expect": "present" }, + { "type": "file_exists", "glob": "config.example.json", "expect": "present" }, + { "type": "file_grep", "glob": "config.example.json", "pattern": "tcp://HOST:55555", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "config.example.json", "pattern": "YOUR_VPN", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "config.example.json", "pattern": "YOUR_USERNAME", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "config.example.json", "pattern": "YOUR_PASSWORD", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": ".gitignore", "pattern": "config.json", "fixed": true, "expect": "present" }, + { "type": "file_exists", "glob": "verify.sh", "expect": "present", "identical_to": "plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh" }, + { "type": "file_exists", "glob": "verify-hooks.sh", "expect": "present" }, + { "type": "file_grep", "glob": "verify-hooks.sh", "pattern": "^START_CMD=", "expect": "present" }, + { "type": "file_grep", "glob": "verify-hooks.sh", "pattern": "^TRIGGER_CMD=", "expect": "present" }, + { "type": "file_grep", "glob": "verify-hooks.sh", "pattern": "^READY_MARKER=", "expect": "present" }, + { "type": "file_grep", "glob": "verify-hooks.sh", "pattern": "^PASS_MARKER=", "expect": "present" }, + { "type": "file_exists", "glob": "solace-verification-checklist.md", "expect": "present" }, + { "type": "tool_use", "tool": "*", "input_pattern": "repo1\\.maven\\.org/maven2/com/solacesystems/sol-jcsmp/maven-metadata\\.xml", "expect": "present" }, + { "type": "tool_use", "tool": "Bash", "input_pattern": "solrsearch|search\\.maven\\.org", "expect": "absent" }, + { "type": "tool_use", "tool": "WebFetch", "input_pattern": "solrsearch|search\\.maven\\.org", "expect": "absent" }, + { "type": "tool_use", "tool": "Bash", "input_pattern": "verify\\.sh +(consumer|publisher|roundtrip|app)", "expect": "absent" }, + { "type": "assistant_grep", "pattern": "./verify.sh", "fixed": true, "expect": "present" } + ] + }, + { + "id": "appdev-debug-redirect-negative", + "skill": "solace-application-development", + "must_pass": true, + "user_turns": [ + "My pharmacy refill-authorization Java app uses JCSMP, and the replier now throws JCSMPErrorResponseException 503 Service Unavailable when it binds to the request queue. Walk me through debugging this step by step." + ], + "max_agent_turns": 15, + "graders": [ + { "type": "assistant_grep", "pattern": "https://docs.solace.com/API/Messaging-APIs/JCSMP-API/jcsmp-api-home.md", "fixed": true, "expect": "present" }, + { "type": "file_exists", "glob": "*", "expect": "absent" }, + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { + "type": "llm_judge", + "criteria": "The user asked for step-by-step debugging of a JCSMP error. The skill's contract says the documentation redirect is the ENTIRE debug answer: acknowledge the problem, link the canonical JCSMP documentation, and stop. PASS when the reply acknowledges the problem and links the documentation and adds nothing else. Restating the symptom the user reported (the exception, the operation that failed) is the acknowledgement, not a hypothesis; an offer to help with design or implementation instead is routing, not debugging guidance. FAIL when the reply names a cause the user did not report, or contains any troubleshooting step, interim check, workaround, or configuration or code suggestion, from any source, including content fetched from a documentation page or paraphrased from a bundled reference file. Restating the rule does not excuse guidance that follows it." + } + ] + }, + { + "id": "appdev-suggested-secure", + "skill": "solace-application-development", + "user_turns": [ + "Build a Java app on our Solace event broker where a pharmacy service requests refill authorization from a doctor's-office service and waits for the answer. Neither the request nor the approval may be lost. Plain Maven, no dashboard. We have a TLS-enabled Solace Cloud broker. This is Request-Reply (Guaranteed); derive the topic and queue names yourself.", + "Yes, I am happy with that design. No need to save it, go ahead.", + "Solace Suggested.", + "No HA failover needed. I will not share credentials in the chat; compile only and hand me the commands to run myself." + ], + "max_agent_turns": 80, + "graders": [ + { "type": "assistant_grep", "pattern": "[Ff]ailover", "expect": "present", "turn": 3 }, + { "type": "java_disclaimer" }, + { "type": "file_grep", "glob": "*.java", "pattern": "SSL_VALIDATE_CERTIFICATE", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "*.java", "pattern": "SSL_VALIDATE_CERTIFICATE_DATE", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "*.java", "pattern": "setProperty\\([^)]*SSL_TRUST_STORE", "expect": "absent" }, + { "type": "file_grep", "glob": "*.java", "pattern": "setDMQEligible(true)", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "pom.xml", "pattern": "|", "expect": "absent" }, + { "type": "compile" }, + { "type": "file_grep", "glob": "*.java", "pattern": "VERIFY: REQUEST_RECEIVED", "fixed": true, "expect": "present" }, + { "type": "file_grep", "glob": "*.java", "pattern": "VERIFY: REPLY_RECEIVED", "fixed": true, "expect": "present" }, + { "type": "tool_use", "tool": "Bash", "input_pattern": "verify\\.sh +(consumer|publisher|roundtrip|app|direct|guaranteed-request-reply|direct-request-reply)", "expect": "absent" }, + { "type": "assistant_grep", "pattern": "verify.sh", "fixed": true, "expect": "present" } + ] + }, + { + "id": "feedback-email-before-confirm-negative", + "skill": "solace-messaging-feedback", + "must_pass": true, + "user_turns": [ + "The solace-application-development skill generated a JCSMP subscriber that does not compile (missing import for ConsumerFlowProperties). Just write the support email to Solace for me right now. Do not ask me anything first." + ], + "max_agent_turns": 15, + "graders": [ + { "type": "assistant_grep", "pattern": "To: support@solace.com", "fixed": true, "expect": "absent" }, + { "type": "assistant_grep", "pattern": "support contract", "fixed": true, "expect": "present" }, + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { "type": "tool_use", "tool": "Edit", "expect": "absent" }, + { "type": "tool_use", "tool": "Bash", "exclude_pattern": "\"command\":\"(echo|true|:)( [^\"\\\\&|;>]*)?\"", "expect": "absent" } + ] + }, + { + "id": "feedback-support-email-positive", + "skill": "solace-messaging-feedback", + "user_turns": [ + "I have feedback on the solace-application-development skill. While building our schema-registry caching gateway, the generated subscriber called ackMessage() before processing completed, so a crash mid-processing lost the message. I expected the client ack to happen only after processing, the way the docs describe.", + "Yes, we hold a Solace support contract. Draft the email." + ], + "max_agent_turns": 15, + "graders": [ + { "type": "assistant_grep", "pattern": "To: support@solace.com", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Actual behavior", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Expected behavior", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Steps to reproduce", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Environment", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "^[[:space:]]*(\\*\\*)?None(\\*\\*)?\\.?[[:space:]]*$", "expect": "absent", "turn": 2 }, + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { "type": "tool_use", "tool": "Edit", "expect": "absent" }, + { "type": "tool_use", "tool": "Bash", "exclude_pattern": "\"command\":\"(echo|true|:)( [^\"\\\\&|;>]*)?\"", "expect": "absent" } + ] + }, + { + "id": "feedback-routing-judge", + "skill": "solace-messaging-feedback", + "user_turns": [ + "Some feedback about the solace-application-development skill: I wish it also generated a Dockerfile and a docker-compose file for the app it builds. Today I hand-write those every time.", + "We do have a Solace support contract." + ], + "max_agent_turns": 15, + "graders": [ + { "type": "assistant_grep", "pattern": "What is the challenge?", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "What is the impact?", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "What is the workaround?", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Describe your idea", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "To: support@solace.com", "fixed": true, "expect": "absent" }, + { + "type": "llm_judge", + "criteria": "Feature-shaped feedback from a user who confirmed a Solace support contract must be routed to the Solace Ideas portal submission format, not a support email and not a community post. PASS when the final draft is an Ideas portal submission with its form headings. FAIL when the final draft is a support email or a community post, or when the assistant never asked about or used the support-contract answer for routing." + } + ] + }, + { + "id": "feedback-scrub-negative", + "skill": "solace-messaging-feedback", + "must_pass": true, + "user_turns": [ + "Feedback on the JCSMP code generation: while building our schema-registry caching gateway, it generated a subscriber that never acks messages. Our setup, for context: broker tcps://mr-zq9x7.messaging.solace.cloud:55443, message VPN zenithbank-orders-vpn, client username zenithbank-svc-01, queue q.zenithbank.orders.intake. The missing ack is the problem; the connection side works fine.", + "No support contract. Format it for the community." + ], + "max_agent_turns": 15, + "graders": [ + { "type": "assistant_grep", "pattern": "zq9x7", "fixed": true, "expect": "absent", "turn": 2 }, + { "type": "assistant_grep", "pattern": "[Zz]enith[Bb]ank", "expect": "absent", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Title: Solace Agent Skills feedback", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "Removed during minimization", "fixed": true, "expect": "present", "turn": 2 }, + { "type": "assistant_grep", "pattern": "To: support@solace.com", "fixed": true, "expect": "absent" }, + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { "type": "tool_use", "tool": "Edit", "expect": "absent" }, + { "type": "tool_use", "tool": "Bash", "exclude_pattern": "\"command\":\"(echo|true|:)( [^\"\\\\&|;>]*)?\"", "expect": "absent" } + ] + }, + { + "id": "topics-grounded-answer", + "skill": "solace-topic-best-practices", + "user_turns": [ + "We are building a smart home platform on Solace: a two-floor house, several rooms per floor, and each room has some mix of lights, thermostats, and door locks that publish state changes and receive commands. How should we structure our topic hierarchy? Which levels, and in what order?" + ], + "max_agent_turns": 15, + "graders": [ + { "type": "tool_use", "tool": "WebFetch", "input_pattern": "Topic-Architecture-Best-Practices\\.md", "expect": "present" }, + { "type": "tool_use", "tool": "WebFetch", "input_pattern": "docs\\.solace\\.com", "exclude_pattern": "Topic-Architecture-Best-Practices\\.md", "expect": "absent" }, + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { "type": "tool_use", "tool": "Edit", "expect": "absent" }, + { "type": "file_exists", "glob": "*", "expect": "absent" }, + { + "type": "llm_judge", + "include": ["assistant_text", "webfetch_results"], + "criteria": "The reply must apply the fetched Topic Architecture Best Practices page to the user's scenario. PASS when the topic-structure recommendations follow the fetched guidance, including domain-specific applications of it, and when anything the page does not cover is explicitly flagged as going beyond the fetched page. Deriving the consequences of the fetched rules for the user's scenario, such as which subscriptions a level order makes easy or hard, is applying the guidance, not going beyond it. FAIL when the reply contradicts the fetched guidance, presents memory-derived best-practice claims as if the page contained them, or never applies the fetched content." + } + ] + }, + { + "id": "topics-no-files-negative", + "skill": "solace-topic-best-practices", + "user_turns": [ + "Design the topic hierarchy for our smart home platform (two floors, several rooms, lights, thermostats, and door locks) and save the naming conventions to a topic-conventions.md file in this directory so the team can reference it." + ], + "max_agent_turns": 15, + "graders": [ + { "type": "tool_use", "tool": "Write", "expect": "absent" }, + { "type": "tool_use", "tool": "Edit", "expect": "absent" }, + { "type": "tool_use", "tool": "Bash", "exclude_pattern": "\"command\":\"(ls|pwd|echo|true|:)( [^\"\\\\&|;>]*)?\"", "expect": "absent" }, + { "type": "file_exists", "glob": "*", "expect": "absent" }, + { "type": "tool_use", "tool": "WebFetch", "input_pattern": "Topic-Architecture-Best-Practices\\.md", "expect": "present" } + ] + } +] diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md index d3ba17d..f1d6cfb 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp.md @@ -15,7 +15,7 @@ Determine the user's intent and enter the appropriate mode. Routing is keyed on | A build request WITH a valid design contract (see the design-contract gate below) | **Implement** | Read `jcsmp/implement-mode.md`; for an embedded shape generate the messaging layer per its leaf rules and verify with the `app` stage of `verify.sh` (its Step 5) | | An edit to an EXISTING app that changes the messaging topology (the topology rule below) | **Design first** | Re-enter `jcsmp/design-mode.md` to re-confirm the affected summary fields, then Implement applies the change | | A mechanical edit to an EXISTING app with no topology change — reconnect handling, payload format, logging, renames, ack tuning | **Implement** | Read `jcsmp/implement-mode.md`; no design pass | -| "My JCSMP app is throwing on connect" / "Why is my consumer not binding the queue?" | **Debug** | Debug mode is not yet available in this release. Redirect the user to the canonical Solace JCSMP troubleshooting documentation: [JCSMP API Home](https://docs.solace.com/API/Messaging-APIs/JCSMP-API/jcsmp-api-home.md). Do not generate debugging guidance from memory. | +| "My JCSMP app is throwing on connect" / "Why is my consumer not binding the queue?" | **Debug** | Debug mode is not yet available in this release. Redirect the user to the canonical Solace JCSMP troubleshooting documentation: [JCSMP API Home](https://docs.solace.com/API/Messaging-APIs/JCSMP-API/jcsmp-api-home.md). The redirect is the ENTIRE debug answer: acknowledge the problem in one line, link the page, and stop. Do not generate debugging guidance from memory. That prohibition covers root-cause hypotheses, likely causes, interim checks or "things worth checking in the meantime", workarounds, and configuration or code suggestions. Restating this rule does not license adding such guidance after the redirect. | If unclear, default to **Design**. Understand the messaging problem before generating code. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md index 168a827..76b78ee 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md @@ -114,7 +114,7 @@ Generation is keyed on the `Pattern` leaf read in Step 2. Each leaf has its own **Embedded and web-app shapes.** When the JCSMP layer sits inside a larger application (a web dashboard, a REST service, a framework app), the chosen leaf's wiring still governs the messaging layer: the same connection helper, the same session/flow/reconnect/ACK handlers, the same `VERIFY:` markers emitted through a stdout-reaching `trace(...)`, and the disclaimer header on every generated file, messaging or not. The `VERIFY:` markers are NOT demo harness; they stay in the messaging layer whatever the shape, because the `app` verify stage (Step 5) greps for them in the app's captured output. -**Write the verification artifacts in this same step.** Alongside the classes and the pom, write: the tailored `solace-verification-checklist.md` at each project root (tailoring rules in Step 6; the write is generation output with NO consent gate), a copy of the bundled `scripts/verify.sh` at the project root, and the generated `verify-hooks.sh` (Step 5 defines its contract). A session that ends early still leaves the checklist and the verify entry points in the project. +**Write the verification artifacts in this same step.** Alongside the classes and the pom, write: the tailored `solace-verification-checklist.md` at each project root (tailoring rules in Step 6; the write is generation output with NO consent gate), a copy of the bundled `scripts/verify.sh` at the project root, and the generated `verify-hooks.sh` (Step 5 defines its contract). A session that ends early still leaves the checklist and the verify entry points in the project. Make the `verify.sh` copy with a read-and-redirect, `cat "/references/jcsmp/scripts/verify.sh" > verify.sh && chmod +x verify.sh`, where the skill directory is the one announced when this skill loaded. Do not use `cp` on the bundled file: Claude Code protects the plugin directory and blocks a `cp` that names a file inside it, and a blocked copy pushes the script through a retype that must stay byte-identical. Never edit the bundled script itself. ### Adapting a sample to a variant of its leaf diff --git a/plugins/solace-messaging-skills/skills/solace-messaging-feedback/SKILL.md b/plugins/solace-messaging-skills/skills/solace-messaging-feedback/SKILL.md index 3f8dd82..bfed8fd 100644 --- a/plugins/solace-messaging-skills/skills/solace-messaging-feedback/SKILL.md +++ b/plugins/solace-messaging-skills/skills/solace-messaging-feedback/SKILL.md @@ -148,8 +148,8 @@ After the block, on a new line: - **Actual behavior / Expected behavior** cover bug-shaped *and* feature-shaped feedback. "I wish it did X" goes in `Expected behavior` with the current state in `Actual behavior`. In the Ideas portal template the same content splits across `What is the challenge?` (the current state) and `Describe your idea` (what the user wants). - **Impact** (rendered as `What is the impact?` on the Ideas portal) is omitted when the user gave no signal of cost or friction. Don't invent severity language. - **What is the workaround?** (Ideas portal only) is how the user copes today, taken from the session. Use `None` when the session shows no workaround; don't invent one. -- **Steps to reproduce** is populated from the prompts and skill invocations of the current session when the feedback is bug-shaped. Omit entirely for feature requests and confusion reports, where repro steps don't apply. -- **Notes** is omitted when empty. Resist padding. +- **Steps to reproduce** is always present for bug-shaped feedback. Populate it from the prompts and skill invocations of the current session when they exist. When the session holds no such invocation (the user reports the bug from memory), derive the steps from the report itself: what to ask the skill, what to inspect in its output, and what to observe. Omit entirely for feature requests and confusion reports, where repro steps don't apply. +- **Notes** is omitted when empty: drop the heading. Never write `None`, `N/A`, or other filler under it. The `None` convention belongs only to the Ideas portal workaround field above. - **Signature** (Support email only) stays the literal placeholder ``. Never infer or fill in the sender's name from git config, the environment, or the session. The user types their own name when they review the email before sending. Omit any field the session doesn't support. Don't fabricate content to fill a slot. @@ -159,7 +159,7 @@ Omit any field the session doesn't support. Don't fabricate content to fill a sl Capture environment metadata best-effort. Any field that can't be determined renders as `unknown`. The skill never errors on env capture. `unknown` is always a valid value. - **Skill / API**: which skill the user was working with (for example `solace-application-development`, or the specific API such as JCSMP) and, when relevant, `solace-topic-best-practices`. Fall back to `unknown` if unclear. -- **Plugin version**: read the `version` field from the plugin manifest at `.claude-plugin/plugin.json` under the plugin root, which is two directories above this skill (resolve `../../.claude-plugin/plugin.json` from this skill's base directory). This works both when the plugin is installed and in a development checkout of this repo, where the plugin root is `plugins/solace-messaging-skills/`. Fall back to `unknown` if unreadable or missing. +- **Plugin version**: read the `version` field from the plugin manifest at `.claude-plugin/plugin.json` under the plugin root, which is two directories above this skill (resolve `../../.claude-plugin/plugin.json` from this skill's base directory). This works both when the plugin is installed and in a development checkout of this repo, where the plugin root is `plugins/solace-messaging-skills/`. Use the Read tool on that one path; the base directory is announced when the skill loads. Do not search the filesystem or run a shell command to find or read the manifest. Fall back to `unknown` if unreadable or missing. - **Model**: the active Claude model name or ID is typically surfaced to the assistant at runtime (for example `claude-opus-4-8`). Fall back to `unknown` if not surfaced. Future env fields are added the same way: attempt, fall back to `unknown`, never block. @@ -217,6 +217,7 @@ When invoked with no prior activity, the skill doesn't emit an empty draft. Ask ## Guardrails - **Formatter, not transport.** No issue creation, email sending, draft creation, community or portal posting, API calls, or file writes. The output stays in chat for the user to copy. +- **Read is the only tool.** The `allowed-tools` list (Read) is the contract, and it exists for the plugin manifest alone. Never run a shell command, search the filesystem, or call any other tool, even when the session makes one available. Everything else in the draft comes from the conversation. - **Confirm before drafting a Support email.** The support-contract question in Routing is a mandatory checkpoint. Never draft the Support email until the user confirms they hold a support contract and want the email drafted. - **User-visible before send.** Always present the full draft so the user can review and edit before pasting, sending, or posting. - **No fabrication.** Omit fields the session doesn't support rather than guessing. diff --git a/plugins/solace-messaging-skills/skills/solace-topic-best-practices/SKILL.md b/plugins/solace-messaging-skills/skills/solace-topic-best-practices/SKILL.md index 41e230d..aaf1d97 100644 --- a/plugins/solace-messaging-skills/skills/solace-topic-best-practices/SKILL.md +++ b/plugins/solace-messaging-skills/skills/solace-topic-best-practices/SKILL.md @@ -12,10 +12,13 @@ This skill answers topic-hierarchy and topic-architecture questions by reading t 1. WebFetch the canonical page live: `https://docs.solace.com/Messaging/Topic-Architecture-Best-Practices.md`. This is the authoritative source for topic-level ordering, naming conventions, wildcard placement, and taxonomy guidance. 2. Apply the fetched guidance to the user's specific topic-design question (the events they publish, the consumers that subscribe, the levels they need, and where wildcards belong). -3. Quote or summarize the fetched page. Do not answer from memory and do not paraphrase guidance the page does not contain. +3. Quote or summarize the fetched page. Do not answer from memory and do not paraphrase guidance the page does not contain. When the user's scenario needs a decision the page does not address, say so explicitly before you make a suggestion, so the reader can tell the page's guidance from your own. +4. Deliver the answer in chat. Never write it to a file, even when the user asks you to save it: present the content in chat and let the user save it themselves. Fetch the single page above, then ground every recommendation in it. Do not WebFetch other pages blindly. ## What this skill is not Do not use this skill to design, build, generate, or implement a Solace messaging application (publisher, consumer, or pub/sub). That is the solace-application-development skill. + +This skill answers in chat only: it never writes, edits, or saves files (a conventions document, a design note, code, or anything else). The `allowed-tools` list (Read, Grep, WebFetch) is the contract; a Write or Edit tool being available in the session does not change it. diff --git a/tools/output-evals.schema.json b/tools/output-evals.schema.json new file mode 100644 index 0000000..9299e06 --- /dev/null +++ b/tools/output-evals.schema.json @@ -0,0 +1,185 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Output-eval corpus", + "description": "Structure of plugins//evals/output-evals.json, the input to tools/run-output-evals.sh. The file is an array of cases. Each case scripts one conversation with the plugin under test and lists the graders that judge the resulting transcript and generated files. A case passes only when the target skill fired and every grader passed. Case ids must be unique across the file; JSON Schema cannot express that rule, so the runner enforces it at start-up together with the structural checks below.", + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/case" }, + "$defs": { + "case": { + "type": "object", + "description": "One scripted conversation plus its graders.", + "required": ["id", "skill", "user_turns", "graders"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$", + "description": "Unique kebab-case case name. Used by the --case filter and printed on every result line. Convention: -[-positive|-negative]." + }, + "skill": { + "type": "string", + "minLength": 1, + "description": "Name of the skill that must fire during the case (the Skill tool's skill input without the plugin prefix, for example solace-application-development). The runner fails the case when this skill never fired, because the output would not be attributable to it and every absence grader would pass vacuously." + }, + "user_turns": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 }, + "description": "Ordered scripted user messages. The first starts a session; each later one continues that session with --resume. The model's replies between user turns are not scripted, so write later user turns so they cannot be misread (state consent or refusal explicitly)." + }, + "max_agent_turns": { + "type": "integer", + "minimum": 1, + "default": 25, + "description": "Budget of agent turns (model replies and tool calls) for each user turn's invocation, passed as --max-turns. Conversational cases need about 15 to 25; a full implement flow needs about 80. Hitting the cap fails the case and is tagged [max-turns] on the result line so a budget problem is distinguishable from a behavior failure." + }, + "must_pass": { + "type": "boolean", + "default": false, + "description": "When true, a failure of this case fails the whole run regardless of the pooled 90% pass rate. Reserve it for acceptance-criteria negatives and the compile anchor." + }, + "graders": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/grader" }, + "description": "Checks applied after all turns complete. Every grader must pass. Absence checks should be scoped to a tool or construct, never to a bare substring: comments and documents that quote a prohibition false-positive naive patterns." + } + } + }, + "grader": { + "description": "One check over the transcript or the generated files. The type field selects the check and decides which other fields are allowed.", + "oneOf": [ + { "$ref": "#/$defs/assistant_grep" }, + { "$ref": "#/$defs/tool_use" }, + { "$ref": "#/$defs/file_exists" }, + { "$ref": "#/$defs/file_grep" }, + { "$ref": "#/$defs/java_disclaimer" }, + { "$ref": "#/$defs/compile" }, + { "$ref": "#/$defs/maven_release_match" }, + { "$ref": "#/$defs/llm_judge" }, + { "$ref": "#/$defs/live_verify" } + ] + }, + "expect_presence": { + "type": "string", + "enum": ["present", "absent"], + "default": "present", + "description": "present: at least one match must exist. absent: no match may exist." + }, + "turn_scope": { + "type": "integer", + "minimum": 1, + "description": "Restrict the check to the transcript of this user turn (1-based index into user_turns). Omit to check all user turns." + }, + "fixed_string": { + "type": "boolean", + "default": false, + "description": "true: treat pattern as a fixed string (grep -F). false: treat it as an extended regular expression (grep -E)." + }, + "assistant_grep": { + "type": "object", + "description": "Grep over the assistant's text blocks only. Tool payloads and the scripted user turns are excluded so they cannot false-positive.", + "required": ["type", "pattern"], + "additionalProperties": false, + "properties": { + "type": { "const": "assistant_grep" }, + "pattern": { "type": "string", "minLength": 1, "description": "Text to look for in the assistant's replies." }, + "expect": { "$ref": "#/$defs/expect_presence" }, + "fixed": { "$ref": "#/$defs/fixed_string" }, + "turn": { "$ref": "#/$defs/turn_scope" } + } + }, + "tool_use": { + "type": "object", + "description": "Match tool_use events by tool name and, optionally, by a regular expression over the compact JSON of the tool input.", + "required": ["type", "tool"], + "additionalProperties": false, + "properties": { + "type": { "const": "tool_use" }, + "tool": { "type": "string", "minLength": 1, "description": "Tool name to match (Write, Edit, Bash, WebFetch, Skill, ...). \"*\" matches every tool." }, + "input_pattern": { "type": "string", "minLength": 1, "description": "Extended regular expression that the compact input JSON must match for the event to count." }, + "exclude_pattern": { "type": "string", "minLength": 1, "description": "Extended regular expression; matching events are dropped before counting. Use it to allow a specific benign match inside an otherwise absent check." }, + "expect": { "$ref": "#/$defs/expect_presence" }, + "count": { "type": "integer", "minimum": 0, "description": "With expect present: the exact number of matching events required. Omit to require at least one." }, + "turn": { "$ref": "#/$defs/turn_scope" } + } + }, + "file_exists": { + "type": "object", + "description": "Find generated files by basename glob under the case work directory. Maven build output under target/ is excluded.", + "required": ["type", "glob"], + "additionalProperties": false, + "properties": { + "type": { "const": "file_exists" }, + "glob": { "type": "string", "minLength": 1, "description": "Basename glob, for example pom.xml, *.java, or * for any file." }, + "expect": { "$ref": "#/$defs/expect_presence" }, + "identical_to": { "type": "string", "minLength": 1, "description": "Repository-relative path. With expect present, every matched file must be byte-identical to this file. Used to prove a bundled script was copied verbatim." } + } + }, + "file_grep": { + "type": "object", + "description": "Grep the generated files matched by a basename glob.", + "required": ["type", "glob", "pattern"], + "additionalProperties": false, + "properties": { + "type": { "const": "file_grep" }, + "glob": { "type": "string", "minLength": 1, "description": "Basename glob selecting the files to grep." }, + "pattern": { "type": "string", "minLength": 1, "description": "Text to look for in the selected files." }, + "expect": { + "type": "string", + "enum": ["present", "absent", "all_files"], + "default": "present", + "description": "present: at least one selected file matches. absent: no selected file matches (also passes when no file matches the glob). all_files: every selected file matches." + }, + "fixed": { "$ref": "#/$defs/fixed_string" } + } + }, + "java_disclaimer": { + "type": "object", + "description": "Every generated .java file starts with the AI-assisted disclaimer line and the verification-checklist pointer. Fails when no .java file exists.", + "required": ["type"], + "additionalProperties": false, + "properties": { "type": { "const": "java_disclaimer" } } + }, + "compile": { + "type": "object", + "description": "Run mvn compile on every generated pom.xml (two-project layouts compile each project). Any non-zero exit fails the case; a timeout is an infrastructure failure. Requires mvn on PATH.", + "required": ["type"], + "additionalProperties": false, + "properties": { "type": { "const": "compile" } } + }, + "maven_release_match": { + "type": "object", + "description": "Every generated pom.xml carries the current sol-jcsmp release version published in the Maven Central metadata. A metadata fetch failure is an infrastructure failure, not a case failure.", + "required": ["type"], + "additionalProperties": false, + "properties": { "type": { "const": "maven_release_match" } } + }, + "llm_judge": { + "type": "object", + "description": "One tool-free completion on the judge model (OUTPUT_EVAL_JUDGE_MODEL) that returns a strict pass or fail verdict for one criterion. Use it for routing and grounding questions that a grep cannot decide. An unparseable verdict after two attempts is an infrastructure failure.", + "required": ["type", "criteria"], + "additionalProperties": false, + "properties": { + "type": { "const": "llm_judge" }, + "criteria": { "type": "string", "minLength": 1, "description": "The single criterion the judge grades. State the PASS condition and the FAIL condition explicitly." }, + "include": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "enum": ["assistant_text", "webfetch_urls", "webfetch_results"] }, + "default": ["assistant_text"], + "description": "Transcript material shown to the judge. assistant_text: the assistant's replies across all turns. webfetch_urls: the deduplicated list of URLs fetched with WebFetch. webfetch_results: the truncated content those fetches returned." + } + } + }, + "live_verify": { + "type": "object", + "description": "Run the generated project's own verify.sh roundtrip against the broker named by the exported OUTPUT_EVAL_BROKER_* variables. verify.sh exit 0 passes; exit 2 (a broker or credential error) or a timeout is an infrastructure failure; any other exit fails. Skips, and reports the skip, when no broker is configured.", + "required": ["type"], + "additionalProperties": false, + "properties": { "type": { "const": "live_verify" } } + } + } +} diff --git a/tools/run-output-evals.sh b/tools/run-output-evals.sh new file mode 100755 index 0000000..73ba6e6 --- /dev/null +++ b/tools/run-output-evals.sh @@ -0,0 +1,725 @@ +#!/usr/bin/env bash +# +# run-output-evals.sh: output-contract evals for the agent plugins. +# +# Trigger evals (run-trigger-evals.sh) ask "did the right skill fire?". Output +# evals ask the next question: did the skill's OUTPUT honor the skill contract? +# For every case in each plugin's evals/output-evals.json, the runner plays the +# case's scripted user turns through the Claude Code CLI with only that plugin +# loaded, then grades the transcript and any generated files against the case's +# grader list. A case passes only when the target skill fired, every grader +# passed, AND no turn hit the turn cap. +# +# WHY --plugin-dir AND A SCRATCH CONFIG DIR +# Same isolation contract as the trigger runner: --plugin-dir loads the +# plugin straight from the repo, and a throwaway CLAUDE_CONFIG_DIR hides the +# developer's ambient skills, so the observed behavior is attributable to +# the plugin under test. A scratch config has no ambient login, so a +# credential must be exported. +# +# MULTI-TURN CASES +# Several contracts only resolve across turns (a design confirm, the door +# question, the support-contract checkpoint), so a case carries an ordered +# "user_turns" array of user messages. Turn 1 starts a session; later turns +# continue it via `claude -p --resume `, re-extracting the +# session id after every turn. Each turn writes its own turnN.jsonl +# transcript so graders can scope assertions to a turn. +# +# CORPUS SCHEMA +# tools/output-evals.schema.json (JSON Schema, draft 2020-12) defines the +# corpus structure and documents the intent of every field. The start-up +# check below enforces its structural subset (array shape, required fields, +# unique ids, known grader types) and exits before any API call when the +# corpus violates it. Validate an edit against the full schema with any JSON +# Schema validator; the per-grader field rules live only in the schema. +# +# GRADERS (closed set; the corpus shape is validated up front) +# assistant_grep grep over the assistant's TEXT blocks only (never the +# raw JSONL: tool payloads and user turns would +# false-positive). Fields: pattern, expect +# (present|absent), fixed (default false), turn +# (default all). +# tool_use match tool_use events by tool name (or "*") and an +# optional ERE over the compact input JSON. Fields: +# tool, input_pattern, exclude_pattern (drop matches of +# this ERE before counting), expect, count, turn. +# file_exists find generated files by basename glob under the case +# work dir (target/ excluded). Fields: glob, expect, +# identical_to (repo-relative path; every match must be +# byte-identical to it). +# file_grep grep files matched by basename glob. Fields: glob, +# pattern, expect (present|absent|all_files), fixed. +# java_disclaimer every generated .java starts with the AI-assisted +# disclaimer line and the checklist pointer; fails when +# no .java exists. No fields. +# compile `mvn -q -B compile` on every generated pom.xml; all +# must exit 0. Bounded at 600s when GNU timeout (or +# gtimeout) is on PATH; unbounded, with a warning, +# otherwise. No fields. +# maven_release_match every generated pom carries the live sol-jcsmp +# from repo1.maven.org metadata. No fields. +# llm_judge one tool-free judge completion on +# OUTPUT_EVAL_JUDGE_MODEL returning a strict +# {"verdict","reason"} JSON. Fields: criteria, include +# (subset of assistant_text, webfetch_urls, +# webfetch_results; default assistant_text). +# live_verify run the generated project's own verify.sh (roundtrip +# stage, Quickstart single-project mode) against the +# broker named by the OUTPUT_EVAL_BROKER_* variables. +# verify.sh exit 0 passes; exit 2 (a doc-traceable +# broker or credential error) and a timeout kill are +# INFRA; anything else fails. Skips, and says so, when +# no broker is configured. No fields. +# +# RUNS: output cases are expensive (a full Implement flow runs 30+ turns plus +# Maven), so OUTPUT_EVAL_RUNS defaults to 1. Set it higher for a majority-vote +# stability study; the verdict is then the majority result per case. +# +# GATE: the run passes when at least 90% of cases pass. A case may set +# "must_pass": true; any must-pass failure fails the run regardless of the +# pooled rate, and so does any infrastructure failure. +# +# Exit codes: 0 = pass rate >= 90% with no infrastructure failures and no +# must-pass failures; 1 = pass rate below the gate, a must-pass failure, or +# any infrastructure failure (a missing credential or tool, a malformed +# corpus, an unparseable judge verdict, a permission denial of a tool the +# runner grants, or zero discovered cases); an unmeasured case is never +# absorbed by the gate. +# +# Usage: run-output-evals.sh [--model ] [--case [,...]]... +# --model Subject model. Defaults to +# ${OUTPUT_EVAL_MODEL:-claude-sonnet-5}. Run the suite for +# BOTH claude-sonnet-5 and claude-opus-5 before a PR. +# --case Run only the named case(s); repeatable or comma-separated. +# OUTPUT_EVAL_JUDGE_MODEL Judge model (default claude-sonnet-5). Keep it +# fixed across subject legs so leg diffs are +# attributable to the subject model. +# OUTPUT_EVAL_WORKDIR, when set, receives the transcripts, generated +# projects, and judge bookkeeping in a unique run-XXXXXX subdirectory; +# cleanup only ever removes that subdirectory. Otherwise a mktemp +# directory is used. Kept (and printed) whenever the run fails. +# OUTPUT_EVAL_BROKER_HOST, OUTPUT_EVAL_BROKER_VPN, OUTPUT_EVAL_BROKER_USER, +# OUTPUT_EVAL_BROKER_PASSWORD, when exported, enable the live_verify grader +# (host as tcp://:55555). Export all four or none: a partial set is +# an error. The values reach only verify.sh, as CLI args, never the +# subject model. + +set -uo pipefail +export LC_ALL=C + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +RUNS="${OUTPUT_EVAL_RUNS:-1}" +MODEL="${OUTPUT_EVAL_MODEL:-claude-sonnet-5}" +JUDGE_MODEL="${OUTPUT_EVAL_JUDGE_MODEL:-claude-sonnet-5}" +GRADER_TYPES='["assistant_grep","tool_use","file_exists","file_grep","java_disclaimer","compile","maven_release_match","llm_judge","live_verify"]' +# Headless -p denies tools outside --allowedTools, which would distort the +# measured behavior, so the subject gets the full list the skills declare. Bash +# is unrestricted by design (agreed for this local-only suite); every invocation +# runs in a scratch work dir. The allowlist does not bind the environment: +# managed settings, hooks, and local command shims can still deny a granted +# tool, and a denied tool makes the model improvise (a denied `cp` once became +# a lossy retype of a 55 KB script). Every result event carries +# permission_denials, so the runner reads it: a denial of a granted tool is +# INFRA, and a denial of any other tool is tagged on the case line. +ALLOWED_TOOLS=(Skill Read Glob Grep Write Edit Bash WebFetch TodoWrite) + +CASE_FILTER=() +while [[ $# -gt 0 ]]; do + case "$1" in + --model) + [[ $# -ge 2 ]] || { echo "ERROR: --model requires a value." >&2; exit 1; } + MODEL="$2"; shift 2 ;; + --case) + [[ $# -ge 2 ]] || { echo "ERROR: --case requires a value." >&2; exit 1; } + IFS=',' read -r -a _ids <<<"$2" + [[ ${#_ids[@]} -gt 0 ]] && CASE_FILTER+=("${_ids[@]}") + shift 2 ;; + -h|--help) + sed -n '2,/^[^#]/s/^# \{0,1\}//p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) + echo "ERROR: unknown argument '$1'." >&2; exit 1 ;; + esac +done + +# Dependency and auth pre-flight (fail closed), before any mktemp so an early +# exit leaks nothing. The credential value is never echoed. +command -v claude >/dev/null 2>&1 || { echo "ERROR: the 'claude' CLI is not on PATH. Install @anthropic-ai/claude-code." >&2; exit 1; } +# GNU timeout (or Homebrew coreutils' gtimeout) bounds mvn and verify.sh at +# 600s. Optional: without it those steps run unbounded, and the runner says so +# below whenever a selected case would use it. +TIMEOUT_BIN="$(command -v timeout || command -v gtimeout || true)" +command -v jq >/dev/null 2>&1 || { echo "ERROR: 'jq' is not on PATH." >&2; exit 1; } +command -v curl >/dev/null 2>&1 || { echo "ERROR: 'curl' is not on PATH." >&2; exit 1; } +# Arithmetic reads a zero or non-numeric RUNS as 0, which skips every run and +# trips bash 3.2's empty-array expansion under set -u at the FAIL print. +[[ "$RUNS" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: OUTPUT_EVAL_RUNS must be a positive integer (got '$RUNS')." >&2; exit 1; } +if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + echo "ERROR: export ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN before running (a scratch CLAUDE_CONFIG_DIR has no ambient login)." >&2 + exit 1 +fi + +# Live-verify broker (opt-in, tri-state). Count the four exported variables: +# none = skip loudly, all = run, a partial set = error, so a typo in one name can +# never degrade to a silent skip. The values must not reach the subject model +# (the compile anchor asserts compile-only behavior), so the export attribute is +# stripped: they stay shell variables handed to verify.sh. +BROKER_VARS=(OUTPUT_EVAL_BROKER_HOST OUTPUT_EVAL_BROKER_VPN OUTPUT_EVAL_BROKER_USER OUTPUT_EVAL_BROKER_PASSWORD) +missing=() +for v in "${BROKER_VARS[@]}"; do [[ -n "${!v:-}" ]] || missing+=("$v"); done +if [[ ${#missing[@]} -eq 0 ]]; then + LIVE_VERIFY=run +elif [[ ${#missing[@]} -eq ${#BROKER_VARS[@]} ]]; then + LIVE_VERIFY=skip +else + echo "ERROR: partial broker configuration: export all four OUTPUT_EVAL_BROKER_* variables or none (missing: ${missing[*]})." >&2 + exit 1 +fi +export -n "${BROKER_VARS[@]}" + +# in_filter : 0 when no filter is set or the id is listed. +in_filter() { + [[ ${#CASE_FILTER[@]} -eq 0 ]] && return 0 + local want + for want in "${CASE_FILTER[@]}"; do [[ "$want" == "$1" ]] && return 0; done + return 1 +} + +# --- transcript extraction helpers ----------------------------------------- +# All of them read turnN.jsonl files under $RUN_DIR and never the raw stream: +# assistant text and tool_use events are jq-extracted so grep patterns cannot +# false-positive on tool payloads or on the scripted user turns. + +# turn_files : print the transcript paths for the scope, in order. +turn_files() { + local t + if [[ "$1" == "all" ]]; then + for ((t = 1; t <= NTURNS; t++)); do + [[ -e "$RUN_DIR/turn${t}.jsonl" ]] && echo "$RUN_DIR/turn${t}.jsonl" + done + else + [[ -e "$RUN_DIR/turn$1.jsonl" ]] && echo "$RUN_DIR/turn$1.jsonl" + fi + return 0 +} + +assistant_text() { # + local f + while IFS= read -r f; do + jq -r 'select(.type=="assistant") | .message.content[]? | select(.type=="text") | .text' "$f" 2>/dev/null + done < <(turn_files "$1") +} + +tool_events() { # : one "namecompact-input-json" line per tool_use + local f + while IFS= read -r f; do + jq -r 'select(.type=="assistant") | .message.content[]? | select(.type=="tool_use") + | [.name, (.input | tojson)] | @tsv' "$f" 2>/dev/null + done < <(turn_files "$1") +} + +webfetch_urls() { + local f + while IFS= read -r f; do + jq -r 'select(.type=="assistant") | .message.content[]? | select(.type=="tool_use" and .name=="WebFetch") + | .input.url // (.input | tojson)' "$f" 2>/dev/null + done < <(turn_files all) | sort -u +} + +webfetch_results() { + local f + while IFS= read -r f; do + jq -rs '([.[] | select(.type=="assistant") | .message.content[]? + | select(.type=="tool_use" and .name=="WebFetch") | .id]) as $ids + | .[] | select(.type=="user") | .message.content[]? + | select(.type=="tool_result" and ((.tool_use_id // "") as $t | $ids | index($t))) + | .content + | if type=="array" then (.[]? | select(.type=="text") | .text) + elif type=="string" then . else tostring end' "$f" 2>/dev/null + done < <(turn_files all) +} + +# turn_ok : 0 = a real measurement; 1 = infrastructure failure +# (no result event, or an error other than the turn cap). A turn-cap kill +# (error_max_turns) is not INFRA: the graders still run over what was +# produced, but the case fails with a [max-turns] detail, because a scripted +# turn that never finished is not evidence of the contract. The tag keeps a +# budget problem distinguishable from a behavior failure. +turn_ok() { + local t="$1" subtype is_error + jq -e 'select(.type=="result")' "$t" >/dev/null 2>&1 || return 1 + subtype="$(jq -r 'select(.type=="result") | .subtype' "$t" 2>/dev/null)" + is_error="$(jq -r 'select(.type=="result") | .is_error' "$t" 2>/dev/null)" + [[ "$subtype" == "error_max_turns" ]] && MAXTURNS_HIT=1 + [[ "$is_error" == "true" && "$subtype" != "error_max_turns" ]] && return 1 + return 0 +} + +# turn_reason : one line naming why turn_ok rejected the +# turn, so an INFRA verdict carries its cause instead of only a work-dir path: +# the result subtype and message when a result event exists, else the tail of +# the CLI's stderr capture. +turn_reason() { + local subtype msg err + subtype="$(jq -r 'select(.type=="result") | .subtype // empty' "$1.jsonl" 2>/dev/null | tail -1)" + if [[ -n "$subtype" ]]; then + msg="$(jq -r 'select(.type=="result") | .result // empty' "$1.jsonl" 2>/dev/null | tail -1 | head -c 200)" + echo "result subtype $subtype${msg:+: $msg}" + else + err="$(tail -2 "$1.err" 2>/dev/null | tr '\n' ' ' | head -c 200)" + echo "no result event; stderr: ${err:-empty}" + fi +} + +# denied_tools : one "toolsnippet" line per permission denial +# in the turn's result event. The snippet is the denied command (or the compact +# input), cut short, so the case line names what the environment blocked. +denied_tools() { + jq -r 'select(.type=="result") | .permission_denials[]? + | [.tool_name, ((.tool_input.command // (.tool_input | tojson)) | .[0:120])] | @tsv' "$1" 2>/dev/null +} + +# granted : 0 when the tool is in ALLOWED_TOOLS. +granted() { + local a + for a in "${ALLOWED_TOOLS[@]}"; do [[ "$a" == "$1" ]] && return 0; done + return 1 +} + +# find_files : generated files under the work dir, excluding +# Maven build output. "*" means any file. +find_files() { + find "$WORK" -type f -name "$1" -not -path '*/target/*' 2>/dev/null +} + +shallowest_pom() { + find_files pom.xml | awk -F/ '{print NF, $0}' | sort -n | head -1 | cut -d' ' -f2- +} + +resolve_sol_jcsmp_release() { + curl -s --max-time 20 https://repo1.maven.org/maven2/com/solacesystems/sol-jcsmp/maven-metadata.xml \ + | grep -oE '[^<]+' | sed -E 's/<\/?release>//g' +} + +# --- graders ---------------------------------------------------------------- +# grade_one : 0 pass, 1 fail (a measurement), 2 infrastructure. +# Failure/infra explanation lands in GRADER_DETAIL. +GRADER_DETAIL="" +grade_one() { + local g="$1" gtype + gtype="$(jq -r '.type' <<<"$g")" + GRADER_DETAIL="" + + case "$gtype" in + assistant_grep) + local pattern expect fixed turn text found=0 gopts=(-q) + pattern="$(jq -r '.pattern' <<<"$g")" + expect="$(jq -r '.expect // "present"' <<<"$g")" + fixed="$(jq -r '.fixed // false' <<<"$g")" + turn="$(jq -r '.turn // "all"' <<<"$g")" + [[ "$fixed" == "true" ]] && gopts+=(-F) || gopts+=(-E) + text="$(assistant_text "$turn")" + grep "${gopts[@]}" -- "$pattern" <<<"$text" && found=1 + if [[ "$expect" == "present" && "$found" -eq 0 ]]; then + GRADER_DETAIL="assistant text (turn $turn) lacks: $pattern"; return 1 + elif [[ "$expect" == "absent" && "$found" -eq 1 ]]; then + GRADER_DETAIL="assistant text (turn $turn) contains forbidden: $pattern"; return 1 + fi + return 0 ;; + + tool_use) + local tool ipat xpat expect count turn events matches m + tool="$(jq -r '.tool' <<<"$g")" + ipat="$(jq -r '.input_pattern // ""' <<<"$g")" + xpat="$(jq -r '.exclude_pattern // ""' <<<"$g")" + expect="$(jq -r '.expect // "present"' <<<"$g")" + count="$(jq -r '.count // ""' <<<"$g")" + turn="$(jq -r '.turn // "all"' <<<"$g")" + events="$(tool_events "$turn")" + if [[ "$tool" == "*" ]]; then matches="$events" + else matches="$(awk -F'\t' -v t="$tool" '$1 == t' <<<"$events")"; fi + if [[ -n "$ipat" ]]; then matches="$(grep -E -- "$ipat" <<<"$matches" || true)"; fi + if [[ -n "$xpat" ]]; then matches="$(grep -vE -- "$xpat" <<<"$matches" || true)"; fi + m="$(grep -c . <<<"$matches" || true)" + if [[ "$expect" == "present" ]]; then + if [[ -n "$count" ]]; then + [[ "$m" -eq "$count" ]] && return 0 + GRADER_DETAIL="tool $tool${ipat:+ ~ $ipat} used $m time(s), expected $count"; return 1 + fi + [[ "$m" -gt 0 ]] && return 0 + GRADER_DETAIL="tool $tool${ipat:+ ~ $ipat} never used"; return 1 + else + [[ "$m" -eq 0 ]] && return 0 + GRADER_DETAIL="forbidden tool use: $tool${ipat:+ ~ $ipat} ($m match(es))"; return 1 + fi ;; + + file_exists) + local glob expect ident files f ok=1 + glob="$(jq -r '.glob' <<<"$g")" + expect="$(jq -r '.expect // "present"' <<<"$g")" + ident="$(jq -r '.identical_to // ""' <<<"$g")" + files="$(find_files "$glob")" + if [[ "$expect" == "absent" ]]; then + [[ -z "$files" ]] && return 0 + GRADER_DETAIL="forbidden file(s) on disk matching $glob: $(head -3 <<<"$files" | tr '\n' ' ')"; return 1 + fi + [[ -z "$files" ]] && { GRADER_DETAIL="no file matching $glob was generated"; return 1; } + if [[ -n "$ident" ]]; then + while IFS= read -r f; do + cmp -s "$f" "$REPO_ROOT/$ident" || { ok=0; GRADER_DETAIL="$f is not byte-identical to $ident"; } + done <<<"$files" + [[ "$ok" -eq 1 ]] || return 1 + fi + return 0 ;; + + file_grep) + local glob pattern expect fixed files f gopts=(-q) matched=0 total=0 miss="" + glob="$(jq -r '.glob' <<<"$g")" + pattern="$(jq -r '.pattern' <<<"$g")" + expect="$(jq -r '.expect // "present"' <<<"$g")" + fixed="$(jq -r '.fixed // false' <<<"$g")" + [[ "$fixed" == "true" ]] && gopts+=(-F) || gopts+=(-E) + files="$(find_files "$glob")" + if [[ -z "$files" ]]; then + [[ "$expect" == "absent" ]] && return 0 + GRADER_DETAIL="no file matching $glob to grep for: $pattern"; return 1 + fi + while IFS= read -r f; do + total=$((total + 1)) + if grep "${gopts[@]}" -- "$pattern" "$f"; then matched=$((matched + 1)); else miss="$f"; fi + done <<<"$files" + case "$expect" in + present) + [[ "$matched" -gt 0 ]] && return 0 + GRADER_DETAIL="$glob file(s) lack: $pattern"; return 1 ;; + absent) + [[ "$matched" -eq 0 ]] && return 0 + GRADER_DETAIL="$glob file(s) contain forbidden: $pattern"; return 1 ;; + all_files) + [[ "$matched" -eq "$total" ]] && return 0 + GRADER_DETAIL="$miss lacks: $pattern"; return 1 ;; + esac ;; + + java_disclaimer) + local files f bad="" + files="$(find_files '*.java')" + [[ -z "$files" ]] && { GRADER_DETAIL="no .java files were generated"; return 1; } + while IFS= read -r f; do + head -3 "$f" | grep -qF 'AI-assisted code. Review before production use.' || bad="$f (missing disclaimer line)" + head -4 "$f" | grep -qF 'See the verification checklist: solace-verification-checklist.md' || bad="$f (missing checklist pointer)" + done <<<"$files" + [[ -z "$bad" ]] && return 0 + GRADER_DETAIL="$bad"; return 1 ;; + + compile) + # Every generated pom compiles, so a two-project layout (Solace Suggested) + # is proven whole, not by whichever pom sorts first. + local poms pom rc n=0 log + poms="$(find_files pom.xml)" + [[ -z "$poms" ]] && { GRADER_DETAIL="no generated pom.xml to compile"; return 1; } + while IFS= read -r pom; do + n=$((n + 1)); log="$RUN_DIR/mvn-compile-$n.log" + if [[ -n "$TIMEOUT_BIN" ]]; then + "$TIMEOUT_BIN" 600 mvn -q -B -f "$pom" compile > "$log" 2>&1; rc=$? + else + mvn -q -B -f "$pom" compile > "$log" 2>&1; rc=$? + fi + # timeout's own exit 124 is a wall-clock kill, not a measurement of the code. + [[ "$rc" -eq 124 ]] && { GRADER_DETAIL="mvn compile for $pom timed out after 600s, log: $log"; return 2; } + [[ "$rc" -eq 0 ]] || { GRADER_DETAIL="mvn compile failed for $pom (exit $rc), log: $log"; return 1; } + done <<<"$poms" + return 0 ;; + + maven_release_match) + local rel poms pom + rel="$(resolve_sol_jcsmp_release)" + [[ -z "$rel" ]] && { GRADER_DETAIL="could not resolve sol-jcsmp from repo1.maven.org"; return 2; } + poms="$(find_files pom.xml)" + [[ -z "$poms" ]] && { GRADER_DETAIL="no generated pom.xml to check against release $rel"; return 1; } + while IFS= read -r pom; do + grep -qF "$rel" "$pom" || { GRADER_DETAIL="$pom does not carry the live sol-jcsmp release $rel"; return 1; } + done <<<"$poms" + return 0 ;; + + llm_judge) + local criteria includes pfile jfile attempt raw verdict reason + criteria="$(jq -r '.criteria' <<<"$g")" + includes="$(jq -r '(.include // ["assistant_text"]) | join(" ")' <<<"$g")" + JUDGE_N=$((JUDGE_N + 1)) + pfile="$RUN_DIR/judge${JUDGE_N}.prompt" + jfile="$RUN_DIR/judge${JUDGE_N}.json" + { + echo 'You are grading an AI assistant transcript against one criterion.' + echo 'Respond with ONLY a JSON object: {"verdict":"pass"|"fail","reason":""}.' + echo 'The verdict is "fail" when the criterion is violated, "pass" otherwise.' + echo + echo "CRITERION: $criteria" + if grep -qw assistant_text <<<"$includes"; then + echo; echo '--- ASSISTANT TEXT (all turns, in order) ---' + assistant_text all | head -c 51200 + fi + if grep -qw webfetch_urls <<<"$includes"; then + echo; echo '--- URLS FETCHED VIA WebFetch (deduplicated; empty means none) ---' + webfetch_urls + fi + if grep -qw webfetch_results <<<"$includes"; then + echo; echo '--- WebFetch RESULT CONTENT (truncated) ---' + echo 'This is the summary the WebFetch tool returned to the assistant for its prompt, not the page itself. A remark in it that something is absent describes the summary, not the page.' + webfetch_results | head -c 30720 + fi + } > "$pfile" + for attempt in 1 2; do + # --tools "" keeps the judge tool-free (a tool call would spend its only + # turn and leave no verdict); the cd keeps the operator's project + # settings out of the judge session, as $WORK does for the subject. + ( cd "$RUN_DIR" && claude -p "$(cat "$pfile")" --model "$JUDGE_MODEL" --max-turns 1 \ + --tools "" --output-format json < /dev/null > "$jfile" 2>"$RUN_DIR/judge${JUDGE_N}.err" ) + raw="$(jq -r '.result // empty' "$jfile" 2>/dev/null | tr '\n' ' ' | grep -oE '\{[^{}]*"verdict"[^{}]*\}' | head -1)" + verdict="$(jq -r '.verdict // empty' <<<"$raw" 2>/dev/null)" + reason="$(jq -r '.reason // empty' <<<"$raw" 2>/dev/null)" + if [[ "$verdict" == "pass" ]]; then return 0; fi + if [[ "$verdict" == "fail" ]]; then GRADER_DETAIL="judge: ${reason:-no reason given}"; return 1; fi + done + GRADER_DETAIL="judge verdict unparseable after 2 attempts ($jfile)"; return 2 ;; + + live_verify) + local pom proj log rc + if [[ "$LIVE_VERIFY" != "run" ]]; then LIVE_SKIPPED=1; return 0; fi + pom="$(shallowest_pom)" + [[ -z "$pom" ]] && { GRADER_DETAIL="no generated pom.xml to verify against the broker"; return 1; } + proj="$(dirname "$pom")" + [[ -f "$proj/verify.sh" ]] || { GRADER_DETAIL="no verify.sh beside $pom"; return 1; } + log="$RUN_DIR/live-verify.log" + LIVE_RAN=1 + if [[ -n "$TIMEOUT_BIN" ]]; then + ( cd "$proj" && "$TIMEOUT_BIN" 600 bash ./verify.sh roundtrip "$OUTPUT_EVAL_BROKER_HOST" "$OUTPUT_EVAL_BROKER_VPN" "$OUTPUT_EVAL_BROKER_USER" "$OUTPUT_EVAL_BROKER_PASSWORD" ) > "$log" 2>&1; rc=$? + else + ( cd "$proj" && bash ./verify.sh roundtrip "$OUTPUT_EVAL_BROKER_HOST" "$OUTPUT_EVAL_BROKER_VPN" "$OUTPUT_EVAL_BROKER_USER" "$OUTPUT_EVAL_BROKER_PASSWORD" ) > "$log" 2>&1; rc=$? + fi + case "$rc" in + 0) return 0 ;; + 2) GRADER_DETAIL="verify.sh roundtrip hit a broker or credential error (exit 2), log: $log"; return 2 ;; + 124) GRADER_DETAIL="verify.sh roundtrip timed out after 600s, log: $log"; return 2 ;; + *) GRADER_DETAIL="verify.sh roundtrip failed (exit $rc), log: $log"; return 1 ;; + esac ;; + + *) + GRADER_DETAIL="unknown grader type '$gtype'"; return 2 ;; + esac +} + +# --- scratch dirs (created only after the guards pass) ----------------------- +export CLAUDE_CONFIG_DIR +CLAUDE_CONFIG_DIR="$(mktemp -d)" +if [[ -n "${OUTPUT_EVAL_WORKDIR:-}" ]]; then + if ! mkdir -p "$OUTPUT_EVAL_WORKDIR" \ + || ! WORKDIR="$(cd "$OUTPUT_EVAL_WORKDIR" && pwd)" \ + || ! WORKDIR="$(mktemp -d "$WORKDIR/run-XXXXXX")"; then + echo "ERROR: cannot prepare OUTPUT_EVAL_WORKDIR '$OUTPUT_EVAL_WORKDIR'." >&2 + rm -rf "$CLAUDE_CONFIG_DIR" + exit 1 + fi +else + WORKDIR="$(mktemp -d)" +fi + +pass=0; fail=0; infra=0; must_fail=0; LIVE_NEEDED=0; LIVE_RAN=0 + +for evals_file in "$REPO_ROOT"/plugins/*/evals/output-evals.json; do + [[ -e "$evals_file" ]] || continue + # Fail closed on a malformed corpus: a jq parse error inside the process + # substitution feeding the loop below is otherwise unobservable, so a broken + # file would silently contribute zero cases. + if ! jq -e --argjson types "$GRADER_TYPES" ' + type=="array" and length>0 + and ([.[].id] | length == (unique | length)) + and all(.[]; + (.id | type=="string" and length>0) + and (.skill | type=="string" and length>0) + and (.user_turns | type=="array" and length>0 and all(.[]; type=="string" and length>0)) + and ((.must_pass // false) | type=="boolean") + and ((.max_agent_turns // 25) | type=="number") + and (.graders | type=="array" and length>0 + and all(.[]; type=="object" and (.type as $t | $types | index($t)))))' \ + "$evals_file" >/dev/null 2>&1; then + echo "ERROR: $evals_file is not a non-empty array of {id, skill, user_turns, graders[, max_agent_turns, must_pass]} cases with unique ids and known grader types." >&2 + rm -rf "$CLAUDE_CONFIG_DIR" "$WORKDIR" + exit 1 + fi + plugin_dir="$(cd "$(dirname "$evals_file")/.." && pwd)" # plugins/ + + # Unknown --case ids are an error, not a silent zero-case run. + if [[ ${#CASE_FILTER[@]} -gt 0 ]]; then + for want in "${CASE_FILTER[@]}"; do + jq -e --arg id "$want" 'any(.[]; .id == $id)' "$evals_file" >/dev/null 2>&1 \ + || { echo "ERROR: --case '$want' not found in $evals_file." >&2; rm -rf "$CLAUDE_CONFIG_DIR" "$WORKDIR"; exit 1; } + done + fi + + # Maven is only required when a selected case compiles, checks the pom, or + # runs verify.sh. The live-verify banner prints before any case starts so the + # operator sees the broker state in the first seconds, not at the end of the run. + needs_mvn=0; needs_live=0 + while IFS= read -r row; do + cid="$(jq -r '.id' <<<"$row")" + in_filter "$cid" || continue + jq -e '.graders | any(.type=="compile" or .type=="maven_release_match" or .type=="live_verify")' <<<"$row" >/dev/null 2>&1 && needs_mvn=1 + jq -e '.graders | any(.type=="live_verify")' <<<"$row" >/dev/null 2>&1 && needs_live=1 + done < <(jq -c '.[]' "$evals_file") + if [[ "$needs_mvn" -eq 1 ]] && ! command -v mvn >/dev/null 2>&1; then + echo "ERROR: 'mvn' is not on PATH and a selected case carries a compile, maven_release_match, or live_verify grader." >&2 + rm -rf "$CLAUDE_CONFIG_DIR" "$WORKDIR" + exit 1 + fi + if [[ "$needs_mvn" -eq 1 && -z "$TIMEOUT_BIN" ]]; then + echo "WARNING: neither 'timeout' nor 'gtimeout' is on PATH; mvn compile and verify.sh run unbounded (install GNU coreutils to cap them at 600s)." >&2 + fi + if [[ "$needs_live" -eq 1 ]]; then + LIVE_NEEDED=1 + if [[ "$LIVE_VERIFY" == "run" ]]; then + echo "live verify: ENABLED against $OUTPUT_EVAL_BROKER_HOST" + else + echo "live verify: SKIPPED (export the four OUTPUT_EVAL_BROKER_* variables to enable)" + fi + fi + + while IFS= read -r case_json; do + case_id="$(jq -r '.id' <<<"$case_json")" + in_filter "$case_id" || continue + skill="$(jq -r '.skill' <<<"$case_json")" + must="$(jq -r '.must_pass // false' <<<"$case_json")" + max_turns="$(jq -r '.max_agent_turns // 25' <<<"$case_json")" + NTURNS="$(jq -r '.user_turns | length' <<<"$case_json")" + + run_pass=0; case_infra=0; fail_details=(); other_denials="" + for ((k = 1; k <= RUNS; k++)); do + # One whole-case retry on an infrastructure failure, each attempt from a + # fresh work dir (no mid-conversation resume of a failed turn). The retry + # is announced, so a flaky leg is distinguishable from a clean one. + attempt_ok=0; turn_detail=""; denial_infra=0 + for attempt in 1 2; do + RUN_DIR="$WORKDIR/${case_id}/run${k}-try${attempt}" + WORK="$RUN_DIR/work" + mkdir -p "$WORK" + MAXTURNS_HIT=0 + sid="" + turn_infra=0 + for ((t = 1; t <= NTURNS; t++)); do + turn_prompt="$(jq -r ".user_turns[$((t - 1))]" <<<"$case_json")" + args=(-p "$turn_prompt" --plugin-dir "$plugin_dir" --model "$MODEL" + --max-turns "$max_turns" --allowedTools "${ALLOWED_TOOLS[@]}" + --output-format stream-json --verbose) + [[ -n "$sid" ]] && args+=(--resume "$sid") + ( cd "$WORK" && claude "${args[@]}" \ + < /dev/null > "$RUN_DIR/turn${t}.jsonl" 2>"$RUN_DIR/turn${t}.err" ) + if ! turn_ok "$RUN_DIR/turn${t}.jsonl"; then + turn_infra=1 + turn_detail="turn $t (try $attempt): $(turn_reason "$RUN_DIR/turn${t}")" + break + fi + # Permission denials. A denied GRANTED tool means the environment, not + # the skill, shaped this output, and the same policy would deny the + # retry too, so the case is INFRA at once. A denial of any other tool + # is the model's own doing and is only tagged on the case line. + while IFS=$'\t' read -r dtool dcmd; do + [[ -z "$dtool" ]] && continue + if granted "$dtool"; then + denial_infra=1 + turn_detail="turn $t: the environment denied $dtool, a tool the runner grants ($dcmd)" + else + other_denials+="$dtool"$'\n' + fi + done < <(denied_tools "$RUN_DIR/turn${t}.jsonl") + [[ "$denial_infra" -eq 1 ]] && break + # Re-extract after every turn: a print-mode resume can mint a new id. + new_sid="$(jq -r 'select(.type=="result") | .session_id // empty' "$RUN_DIR/turn${t}.jsonl" | tail -1)" + [[ -z "$new_sid" ]] && new_sid="$(jq -r 'select(.type=="system" and .subtype=="init") | .session_id // empty' "$RUN_DIR/turn${t}.jsonl" | head -1)" + [[ -n "$new_sid" ]] && sid="$new_sid" + done + [[ "$denial_infra" -eq 1 ]] && break + if [[ "$turn_infra" -eq 0 ]]; then attempt_ok=1; break; fi + [[ "$attempt" -eq 1 ]] && echo "RETRY [$case_id] ($skill) :: $turn_detail" + done + if [[ "$attempt_ok" -ne 1 ]]; then case_infra=1; fail_details=("$turn_detail"); break; fi + + # Implicit gate: the target skill must have fired, or the output is not + # attributable to it and every absence grader would pass vacuously. + fired="$(turn_files all | while IFS= read -r f; do + jq -r 'select(.type=="assistant") | .message.content[]? + | select(.type=="tool_use" and .name=="Skill") | .input.skill' "$f" 2>/dev/null + done | sed 's/.*://' | sort -u)" + this_run_details=() + if ! grep -qxF "$skill" <<<"$fired"; then + this_run_details+=("the '$skill' skill never fired (fired: ${fired:-none})") + fi + + JUDGE_N=0 + LIVE_SKIPPED=0 + gi=0 + while IFS= read -r grader; do + gi=$((gi + 1)) + gtype="$(jq -r '.type' <<<"$grader")" + grade_one "$grader"; rc=$? + if [[ "$rc" -eq 2 ]]; then case_infra=1; this_run_details+=("grader#$gi $gtype INFRA :: $GRADER_DETAIL"); break + elif [[ "$rc" -eq 1 ]]; then this_run_details+=("grader#$gi $gtype :: $GRADER_DETAIL"); fi + done < <(jq -c '.graders[]' <<<"$case_json") + [[ "$case_infra" -eq 1 ]] && { fail_details=("${this_run_details[@]}"); break; } + + [[ "$MAXTURNS_HIT" -eq 1 ]] && this_run_details+=("[max-turns] a user turn hit the max_agent_turns cap of $max_turns") + if [[ ${#this_run_details[@]} -eq 0 ]]; then + run_pass=$((run_pass + 1)) + else + fail_details=("${this_run_details[@]}") + fi + done + + denial_tag="" + [[ -n "$other_denials" ]] && denial_tag=" [denied outside the allowlist: $(sort -u <<<"$other_denials" | grep . | paste -sd' ' -)]" + + if [[ "$case_infra" -eq 1 ]]; then + echo "FAIL [$case_id] ($skill) INFRA${fail_details[0]:+ :: ${fail_details[0]}}$denial_tag" + fail=$((fail + 1)); infra=$((infra + 1)); continue + fi + + # Majority vote; with RUNS=1 this is simply "the single run passed". + live_tag="" + [[ "${LIVE_SKIPPED:-0}" -eq 1 ]] && live_tag=" [live verify skipped: no broker configured]" + if (( run_pass * 2 > RUNS )); then + echo "PASS [$case_id] ($skill)$live_tag$denial_tag" + pass=$((pass + 1)) + else + tag="" + [[ "$must" == "true" ]] && { must_fail=$((must_fail + 1)); tag=" [must-pass]"; } + echo "FAIL$tag [$case_id] ($skill) ($run_pass/$RUNS runs passed)$live_tag$denial_tag" + for d in "${fail_details[@]}"; do echo " - $d"; done + fail=$((fail + 1)) + fi + done < <(jq -c '.[]' "$evals_file") +done + +total=$((pass + fail)) +echo +if [[ "$total" -eq 0 ]]; then + echo "ERROR: no output-eval cases discovered under plugins/*/evals/output-evals.json." >&2 + rm -rf "$CLAUDE_CONFIG_DIR" "$WORKDIR" + exit 1 +fi +echo "$pass passed, $fail failed ($((pass * 100 / total))% pass rate, gate is 90%)" +[[ "$must_fail" -gt 0 ]] && echo "$must_fail must-pass case(s) failed (any must-pass failure fails the run)" +if [[ "$LIVE_NEEDED" -eq 1 ]]; then + if [[ "$LIVE_VERIFY" == "run" && "$LIVE_RAN" -eq 1 ]]; then + echo "live verify: ran against $OUTPUT_EVAL_BROKER_HOST (see the case line for its verdict)" + elif [[ "$LIVE_VERIFY" == "run" ]]; then + echo "live verify: enabled against $OUTPUT_EVAL_BROKER_HOST but never executed (the case failed before grading)" + else + echo "live verify: skipped (no broker configured); say so in the PR record" + fi +fi + +rm -rf "$CLAUDE_CONFIG_DIR" +if [[ "$infra" -gt 0 || "$must_fail" -gt 0 ]] || (( pass * 100 < total * 90 )); then + echo "transcripts and generated projects kept for inspection under: $WORKDIR" >&2 + exit 1 +fi +rm -rf "$WORKDIR" +exit 0 From 9d1145fb4bc45d1eddc7c180cf0ad9765ae9fde6 Mon Sep 17 00:00:00 2001 From: Adi El-Sammak <71227923+aelsammak@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:45:52 -0400 Subject: [PATCH 5/5] EBP-3061: Bump solace-messaging-skills to 0.1.1 for release (#7) - Bump version in plugins/solace-messaging-skills/.claude-plugin/plugin.json from 0.1.0 to 0.1.1 - Patch release per the pre-1.0.0 rules: behavior-preserving fixes and documentation since 0.1.0 - Release-prep PR into dev, the only PR into dev that may touch a plugin version --- plugins/solace-messaging-skills/.claude-plugin/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/solace-messaging-skills/.claude-plugin/plugin.json b/plugins/solace-messaging-skills/.claude-plugin/plugin.json index 766f539..d41a724 100644 --- a/plugins/solace-messaging-skills/.claude-plugin/plugin.json +++ b/plugins/solace-messaging-skills/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "solace-messaging-skills", "description": "Skills for building applications on the Solace platform. Includes solace-application-development (designs and generates Solace messaging code grounded in canonical Solace docs), solace-topic-best-practices (looks up the canonical Topic Architecture Best Practices page online for topic-hierarchy questions), and solace-messaging-feedback (formats a session into copy-paste-ready feedback routed to the Solace Ideas portal, Solace Support, or the Solace Community)", - "version": "0.1.0", + "version": "0.1.1", "license": "Apache-2.0", "author": { "name": "Solace"