test/docs: E2E prod build, artifact uploads, reset recovery, and report reference

.github/workflows/ci.yml

@@ -43,8 +43,17 @@ jobs:
           cache-dependency-path: esure-frontend/package-lock.json
       - run: npm ci
       - run: npm run check
+      - run: npm run build
       - run: npx playwright install --with-deps chromium
-      - run: npm run test:e2e
+      - run: npm run test:e2e:prod
+      - name: Upload test artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report-${{ matrix.node-version || 'default' }}
+          path: esure-frontend/playwright-report/
+          retention-days: 7
+          if-no-files-found: ignore
 
   cli:
     name: CLI checks

esure-docs/API.md

@@ -166,3 +166,169 @@ Errors must not expose secret keys, raw internal exceptions, or configuration.
 - `IDEMPOTENCY_CONFLICT`
 - `IDEMPOTENCY_CAPACITY`
 - `INTERNAL_ERROR`
+
+## Report field reference
+
+The final sanitized report is the authoritative record of a completed run. This section explains each field using a complete annotated example.
+
+### Terminal vs. active reports
+
+- **Terminal reports** (status `passed` or `failed`) are complete and may be downloaded using `GET /api/v1/runs/{runId}/report`.
+- **Active reports** (status `requested`, `validating`, or `running`) are not yet complete. Downloading an active report returns `409 CONFLICT` with message "run is not complete". Polling `GET /api/v1/runs/{runId}` returns the run metadata but not the downloadable report attachment.
+
+### Report structure
+
+```json
+{
+  "id": "48f2c1a3-7d22-4b1f-8c3e-9a5e2b1c4d0f",
+  "scenarioId": "issued-asset-payment",
+  "scenarioVersion": 1,
+  "scenarioSchemaVersion": 1,
+  "scenarioContentHash": "sha256:3e8f4a7c1b2d5f6e9a0c8d3e7f2a4b6c5d8e1f3a4b7c9d2e5f8a0c3e6f9b1d4",
+  "network": "testnet",
+  "status": "passed",
+  "createdAt": "2026-10-20T14:32:15.000Z",
+  "completedAt": "2026-10-20T14:32:23.000Z",
+  "steps": [
+    {
+      "id": "create-trustline",
+      "type": "changeTrust",
+      "status": "passed",
+      "transactionHash": "8d1a9c2f4e7b6a3d5c8e1f9a2b4d6e8f7c9a1b3d5e7f9a0c2d4e6f8a9b1c3d",
+      "ledger": 899456,
+      "stellarTransactionCode": "tx_success",
+      "stellarOperationCodes": [],
+      "message": "Account GXYZ... established trustline to TESTUSD issued by GABC...",
+      "startedAt": "2026-10-20T14:32:16.000Z",
+      "completedAt": "2026-10-20T14:32:18.000Z",
+      "durationMs": 2150
+    },
+    {
+      "id": "issue-asset",
+      "type": "payment",
+      "status": "passed",
+      "transactionHash": "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a",
+      "ledger": 899457,
+      "stellarTransactionCode": "tx_success",
+      "stellarOperationCodes": [],
+      "message": "Issuer GABC... paid 100.0000000 TESTUSD to customer GXYZ...",
+      "startedAt": "2026-10-20T14:32:18.500Z",
+      "completedAt": "2026-10-20T14:32:20.000Z",
+      "durationMs": 1500
+    }
+  ],
+  "assertions": [
+    {
+      "type": "balanceEquals",
+      "status": "passed",
+      "expected": "100.0000000",
+      "actual": "100.0000000",
+      "message": "Account GXYZ... holds exactly 100.0000000 TESTUSD"
+    },
+    {
+      "type": "trustlineExists",
+      "status": "passed",
+      "expected": true,
+      "actual": true,
+      "message": "Account GXYZ... has an active trustline to TESTUSD issued by GABC..."
+    },
+    {
+      "type": "stepSucceeded",
+      "status": "passed",
+      "expected": true,
+      "actual": true,
+      "message": "Step issue-asset succeeded"
+    }
+  ],
+  "summary": {
+    "stepsPassed": 2,
+    "stepsFailed": 0,
+    "assertionsPassed": 3,
+    "assertionsFailed": 0
+  }
+}
+```
+
+### Field explanations
+
+#### Provenance
+- **id** (UUID): Unique identifier for this run. Use it to retrieve the run status or download this report.
+- **scenarioId** (string): The scenario definition that was executed (e.g., `issued-asset-payment`).
+- **scenarioVersion** (integer): The version of the scenario definition executed (e.g., `1`).
+- **scenarioSchemaVersion** (integer): Always `1` in the current API.
+- **scenarioContentHash** (string): SHA-256 digest of the scenario definition; used to detect changes to a scenario across runs.
+- **network** (string): Always `testnet` in the current API.
+
+#### Timeline
+- **createdAt** (ISO-8601): When the run was requested.
+- **completedAt** (ISO-8601, optional): When the run finished execution. Absent if the run is still active.
+
+#### Run outcome
+- **status** (string): One of `requested`, `validating`, `running`, `passed`, or `failed`.
+  - `requested`: Run accepted but not yet started.
+  - `validating`: Scenario definition is being validated.
+  - `running`: Steps and assertions are executing.
+  - `passed`: All assertions passed.
+  - `failed`: One or more assertions failed, or an execution error occurred.
+
+#### Steps (executed operations)
+Each step in the `steps` array represents one submitted transaction:
+
+- **id** (string): Step identifier from the scenario definition (e.g., `create-trustline`).
+- **type** (string): Operation type (e.g., `changeTrust`, `payment`, `manageSellOffer`).
+- **status** (string): `passed` if Stellar accepted the transaction, `failed` if rejected.
+- **transactionHash** (string, optional): The transaction hash on Horizon. Present only if the transaction was submitted to the ledger.
+- **ledger** (integer, optional): The ledger sequence number that included this transaction.
+- **stellarTransactionCode** (string): The top-level response from Stellar (e.g., `tx_success`, `tx_failed`, `tx_too_late`).
+- **stellarOperationCodes** (array): Per-operation result codes from Stellar. Empty if the transaction succeeded. Contains one or more codes if operations failed.
+- **message** (string): Plain-English summary of the step result.
+- **startedAt** (ISO-8601, optional): When this step execution began.
+- **completedAt** (ISO-8601, optional): When this step execution finished.
+- **durationMs** (integer, optional): Elapsed time in milliseconds, derived from a monotonic clock.
+
+#### Assertions (test outcomes)
+Each assertion in the `assertions` array represents a deterministic check evaluated after execution:
+
+- **type** (string): Assertion type (e.g., `balanceEquals`, `stepSucceeded`, `stepFailedWith`).
+- **status** (string): `passed` if the assertion matched, `failed` if it did not.
+- **expected** (mixed): The expected value from the scenario definition. Type depends on the assertion (string for balances, boolean for existence checks, array of codes for `stepFailedWith`).
+- **actual** (mixed): The observed value. May be `null` if the evidence could not be retrieved (e.g., a transaction lookup failed).
+- **message** (string): Plain-English explanation of the assertion result.
+
+#### Summary
+- **stepsPassed** (integer): Count of steps with status `passed`.
+- **stepsFailed** (integer): Count of steps with status `failed`.
+- **assertionsPassed** (integer): Count of assertions with status `passed`.
+- **assertionsFailed** (integer): Count of assertions with status `failed`.
+
+#### Error (optional)
+If the run status is `failed`, an `error` object may be present:
+
+```json
+{
+  "code": "TRANSACTION_FAILED",
+  "message": "Step 'submit-payment' failed with Stellar error tx_failed / op_underfunded",
+  "category": "stellar",
+  "retryable": true,
+  "failedStepId": "submit-payment",
+  "stellarTransactionCode": "tx_failed",
+  "stellarOperationCodes": ["op_underfunded"]
+}
+```
+
+- **code** (string): Machine-readable error identifier.
+- **message** (string): Human-readable error explanation.
+- **category** (string): Error category (`stellar`, `network`, `timeout`, `capacity`, `validation`, or `internal`).
+- **retryable** (boolean): Whether retrying the run is likely to succeed.
+- **failedStepId** (string, optional): The step ID that caused the error.
+- **stellarTransactionCode** (string, optional): Stellar's transaction-level error code.
+- **stellarOperationCodes** (array, optional): Stellar's per-operation error codes.
+
+### Decimal strings
+
+All numeric amounts (balances, transaction values, fees) are represented as decimal strings, never as JSON numbers. This preserves precision and avoids floating-point rounding errors:
+
+- ✓ Correct: `"amount": "100.0000000"`, `"expected": "25.5000001"`
+- ✗ Incorrect: `"amount": 100, "expected": 25.5`
+
+Decimal strings follow Stellar's stroop representation (at most 7 fractional digits).

esure-docs/TESTNET_RESET_RECOVERY.md

@@ -0,0 +1,108 @@
+# Testnet Reset Recovery Runbook
+
+When the Stellar Testnet undergoes a reset or network downtime, historical transaction links and generated accounts may become unavailable. This runbook explains how to recognize network-related failures and safely recover by rerunning scenarios with fresh accounts.
+
+## Symptoms of different failure types
+
+### Network reset
+A network reset clears all Testnet ledger state, account balances, and transaction history. You may observe:
+- Friendbot funding fails with "no such account" or "account not found"
+- Previously funded accounts report zero balances
+- Transaction lookups return "not found" on Horizon
+- New account generation works, but old accounts cannot be reused
+- Error logs reference account or transaction state that no longer exists
+
+Recovery action: [Rerun scenarios with fresh accounts](#rerunning-scenarios-with-fresh-accounts)
+
+### Backend restart without network reset
+The Esure backend or Horizon service may restart independently. You may observe:
+- Transient connection timeouts or "service unavailable" errors
+- Existing accounts and transactions remain available
+- Retrying the same run with the same account succeeds
+- Network state is consistent within a few seconds after restart
+
+Recovery action: Wait briefly and [retry the run](#retrying-failed-runs); no fresh accounts needed.
+
+### Report retention expiration
+Esure reports are retained for a configured duration (default: 1 hour). You may observe:
+- Run reports return `409 CONFLICT` or "not complete" on download
+- Recent transaction lookups on Horizon succeed (the network did not reset)
+- The run ID is correct but the report is no longer available locally
+- The transaction hash still resolves on Horizon
+
+Recovery action: The run itself is finished; no recovery is needed. Save reports immediately after runs complete, or configure a longer `RUN_RETENTION_MS` in the backend if you need longer access.
+
+## Official network status
+
+Testnet network resets are announced on the [Stellar Community Forum](https://stellar.community/) and the [Stellar Status page](https://stellar.status.io/). Before declaring a network reset:
+
+1. Check the [Testnet Horizon health endpoint](https://horizon-testnet.stellar.org/health)
+2. Visit the Stellar Community Forum and search for recent reset announcements
+3. Verify your own Friendbot requests against a fresh generated account
+
+Do not hardcode reset schedules or monitor dates in your code. Always consult the official sources above.
+
+## Rerunning scenarios with fresh accounts
+
+When you confirm a network reset has occurred, reruns are safe and expected:
+
+1. **Retrieve your scenario definition**: If you saved the scenario YAML/JSON file, reuse the exact same file. If you lost the definition, recreate it from your requirements.
+2. **Generate fresh accounts**: Esure scenarios always use `generate: true` for accounts, so each run automatically creates fresh accounts. No manual account creation is needed.
+3. **Resubmit the scenario**: POST to `/api/v1/runs` or use the CLI with your scenario definition.
+4. **Verify transaction links**: After the run completes, check that new transaction links resolve on Horizon. If they do not, the network reset may still be in progress; wait and retry.
+5. **Save the report immediately**: Use `GET /api/v1/runs/{runId}/report` to download the sanitized report before the retention window expires.
+
+Example using the CLI:
+
+```bash
+cd esure-cli
+npm ci
+npm run build
+node dist/cli.js run ./my-scenario.yaml --output json > report-after-reset.json
+```
+
+## Preserving offline reports
+
+Esure reports are JSON documents and can be safely stored offline:
+
+1. Download the report from `GET /api/v1/runs/{runId}/report` immediately after a run completes.
+2. Save it as a JSON file in your version-control system, artifact store, or documentation.
+3. Offline reports do not expire and can be shared, analyzed, or reaudited later.
+4. Transaction links in old reports may stop resolving on Horizon after a network reset, but the historical data in the report itself remains valid.
+
+Example saving a report:
+
+```bash
+curl https://esure.onrender.com/api/v1/runs/abc123/report > historical-report.json
+```
+
+## Troubleshooting
+
+### "Friendbot unavailable" errors
+Friendbot (the Testnet faucet) may be temporarily down independently of a network reset. Verify:
+- Recent transaction lookups on Horizon work (network is online)
+- New accounts generate successfully (your code runs)
+- Fresh Friendbot requests on simple accounts succeed (faucet is available)
+
+If Friendbot is down, wait 5-10 minutes and retry with a fresh run. If the issue persists, check the [Stellar Community Forum](https://stellar.community/) for status updates.
+
+### "Transaction not found" on Horizon
+After a run succeeds, transaction links may fail to resolve if:
+- The network reset after the transaction was confirmed (expected, see [Network reset](#network-reset))
+- Report retention expired and the run record was deleted from Esure (your local copy, if saved, is still accurate)
+- A rare network synchronization issue occurred (wait a minute and retry the link)
+
+If a recent transaction link does not resolve but Horizon is online, consult the [Stellar Status page](https://stellar.status.io/) and wait for network synchronization.
+
+### Multiple failed runs
+If several consecutive runs fail with consistent network errors:
+1. Verify the backend is online: `curl https://esure.onrender.com/health`
+2. Check the [Stellar Status page](https://stellar.status.io/) for ongoing incidents
+3. Wait 10-15 minutes, as Testnet may be recovering from an outage
+4. Retry a simple bundled scenario like `xlm-payment` to isolate your scenario definition from network issues
+
+## See also
+
+- [Scenario Schema](./SCENARIOS.md) — Scenario definition format and validation
+- [Backend API](./API.md) — Run creation, status polling, and report endpoints
+- [System Architecture](./ARCHITECTURE.md) — How Esure runs scenarios and persists reports

esure-frontend/package.json

@@ -8,6 +8,7 @@
     "start": "next start",
     "test": "vitest run",
     "test:e2e": "playwright test",
+    "test:e2e:prod": "TEST_MODE=production playwright test",
     "typecheck": "tsc --noEmit",
     "check": "npm run typecheck && npm test && npm run build"
   },

esure-frontend/playwright.config.ts

@@ -1,14 +1,21 @@
 import { defineConfig, devices } from "@playwright/test";
 
+const port = process.env.PORT || 3000;
+const baseURL = `http://localhost:${port}`;
+const isProduction = process.env.TEST_MODE === "production";
+
 export default defineConfig({
   testDir: "./e2e",
   fullyParallel: true,
   forbidOnly: Boolean(process.env.CI),
   retries: process.env.CI ? 2 : 0,
   workers: process.env.CI ? 1 : undefined,
-  reporter: "list",
+  reporter: [
+    "list",
+    ["html", { outputFolder: "playwright-report" }],
+  ],
   use: {
-    baseURL: `http://localhost:${process.env.PORT || 3000}`,
+    baseURL,
     trace: "on-first-retry",
   },
   projects: [
@@ -25,10 +32,17 @@ export default defineConfig({
       use: { ...devices["Desktop Safari"] },
     },
   ],
-  webServer: {
-    command: `npm run dev -- --port ${process.env.PORT || 3000}`,
-    url: `http://localhost:${process.env.PORT || 3000}`,
-    reuseExistingServer: !process.env.CI,
-    timeout: 120_000,
-  },
+  webServer: isProduction
+    ? {
+        command: `npm run start -- --port ${port}`,
+        url: baseURL,
+        reuseExistingServer: false,
+        timeout: 120_000,
+      }
+    : {
+        command: `npm run dev -- --port ${port}`,
+        url: baseURL,
+        reuseExistingServer: !process.env.CI,
+        timeout: 120_000,
+      },
 });