diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49da977..323423f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,17 @@ jobs: cache-dependency-path: esure-frontend/package-lock.json - run: npm ci - run: npm run check + - run: npm run build - run: npx playwright install --with-deps chromium - - run: npm run test:e2e + - run: npm run test:e2e:prod + - name: Upload test artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report-${{ matrix.node-version || 'default' }} + path: esure-frontend/playwright-report/ + retention-days: 7 + if-no-files-found: ignore cli: name: CLI checks diff --git a/esure-docs/API.md b/esure-docs/API.md index 44a1ab0..09e771e 100644 --- a/esure-docs/API.md +++ b/esure-docs/API.md @@ -166,3 +166,169 @@ Errors must not expose secret keys, raw internal exceptions, or configuration. - `IDEMPOTENCY_CONFLICT` - `IDEMPOTENCY_CAPACITY` - `INTERNAL_ERROR` + +## Report field reference + +The final sanitized report is the authoritative record of a completed run. This section explains each field using a complete annotated example. + +### Terminal vs. active reports + +- **Terminal reports** (status `passed` or `failed`) are complete and may be downloaded using `GET /api/v1/runs/{runId}/report`. +- **Active reports** (status `requested`, `validating`, or `running`) are not yet complete. Downloading an active report returns `409 CONFLICT` with message "run is not complete". Polling `GET /api/v1/runs/{runId}` returns the run metadata but not the downloadable report attachment. + +### Report structure + +```json +{ + "id": "48f2c1a3-7d22-4b1f-8c3e-9a5e2b1c4d0f", + "scenarioId": "issued-asset-payment", + "scenarioVersion": 1, + "scenarioSchemaVersion": 1, + "scenarioContentHash": "sha256:3e8f4a7c1b2d5f6e9a0c8d3e7f2a4b6c5d8e1f3a4b7c9d2e5f8a0c3e6f9b1d4", + "network": "testnet", + "status": "passed", + "createdAt": "2026-10-20T14:32:15.000Z", + "completedAt": "2026-10-20T14:32:23.000Z", + "steps": [ + { + "id": "create-trustline", + "type": "changeTrust", + "status": "passed", + "transactionHash": "8d1a9c2f4e7b6a3d5c8e1f9a2b4d6e8f7c9a1b3d5e7f9a0c2d4e6f8a9b1c3d", + "ledger": 899456, + "stellarTransactionCode": "tx_success", + "stellarOperationCodes": [], + "message": "Account GXYZ... established trustline to TESTUSD issued by GABC...", + "startedAt": "2026-10-20T14:32:16.000Z", + "completedAt": "2026-10-20T14:32:18.000Z", + "durationMs": 2150 + }, + { + "id": "issue-asset", + "type": "payment", + "status": "passed", + "transactionHash": "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a", + "ledger": 899457, + "stellarTransactionCode": "tx_success", + "stellarOperationCodes": [], + "message": "Issuer GABC... paid 100.0000000 TESTUSD to customer GXYZ...", + "startedAt": "2026-10-20T14:32:18.500Z", + "completedAt": "2026-10-20T14:32:20.000Z", + "durationMs": 1500 + } + ], + "assertions": [ + { + "type": "balanceEquals", + "status": "passed", + "expected": "100.0000000", + "actual": "100.0000000", + "message": "Account GXYZ... holds exactly 100.0000000 TESTUSD" + }, + { + "type": "trustlineExists", + "status": "passed", + "expected": true, + "actual": true, + "message": "Account GXYZ... has an active trustline to TESTUSD issued by GABC..." + }, + { + "type": "stepSucceeded", + "status": "passed", + "expected": true, + "actual": true, + "message": "Step issue-asset succeeded" + } + ], + "summary": { + "stepsPassed": 2, + "stepsFailed": 0, + "assertionsPassed": 3, + "assertionsFailed": 0 + } +} +``` + +### Field explanations + +#### Provenance +- **id** (UUID): Unique identifier for this run. Use it to retrieve the run status or download this report. +- **scenarioId** (string): The scenario definition that was executed (e.g., `issued-asset-payment`). +- **scenarioVersion** (integer): The version of the scenario definition executed (e.g., `1`). +- **scenarioSchemaVersion** (integer): Always `1` in the current API. +- **scenarioContentHash** (string): SHA-256 digest of the scenario definition; used to detect changes to a scenario across runs. +- **network** (string): Always `testnet` in the current API. + +#### Timeline +- **createdAt** (ISO-8601): When the run was requested. +- **completedAt** (ISO-8601, optional): When the run finished execution. Absent if the run is still active. + +#### Run outcome +- **status** (string): One of `requested`, `validating`, `running`, `passed`, or `failed`. + - `requested`: Run accepted but not yet started. + - `validating`: Scenario definition is being validated. + - `running`: Steps and assertions are executing. + - `passed`: All assertions passed. + - `failed`: One or more assertions failed, or an execution error occurred. + +#### Steps (executed operations) +Each step in the `steps` array represents one submitted transaction: + +- **id** (string): Step identifier from the scenario definition (e.g., `create-trustline`). +- **type** (string): Operation type (e.g., `changeTrust`, `payment`, `manageSellOffer`). +- **status** (string): `passed` if Stellar accepted the transaction, `failed` if rejected. +- **transactionHash** (string, optional): The transaction hash on Horizon. Present only if the transaction was submitted to the ledger. +- **ledger** (integer, optional): The ledger sequence number that included this transaction. +- **stellarTransactionCode** (string): The top-level response from Stellar (e.g., `tx_success`, `tx_failed`, `tx_too_late`). +- **stellarOperationCodes** (array): Per-operation result codes from Stellar. Empty if the transaction succeeded. Contains one or more codes if operations failed. +- **message** (string): Plain-English summary of the step result. +- **startedAt** (ISO-8601, optional): When this step execution began. +- **completedAt** (ISO-8601, optional): When this step execution finished. +- **durationMs** (integer, optional): Elapsed time in milliseconds, derived from a monotonic clock. + +#### Assertions (test outcomes) +Each assertion in the `assertions` array represents a deterministic check evaluated after execution: + +- **type** (string): Assertion type (e.g., `balanceEquals`, `stepSucceeded`, `stepFailedWith`). +- **status** (string): `passed` if the assertion matched, `failed` if it did not. +- **expected** (mixed): The expected value from the scenario definition. Type depends on the assertion (string for balances, boolean for existence checks, array of codes for `stepFailedWith`). +- **actual** (mixed): The observed value. May be `null` if the evidence could not be retrieved (e.g., a transaction lookup failed). +- **message** (string): Plain-English explanation of the assertion result. + +#### Summary +- **stepsPassed** (integer): Count of steps with status `passed`. +- **stepsFailed** (integer): Count of steps with status `failed`. +- **assertionsPassed** (integer): Count of assertions with status `passed`. +- **assertionsFailed** (integer): Count of assertions with status `failed`. + +#### Error (optional) +If the run status is `failed`, an `error` object may be present: + +```json +{ + "code": "TRANSACTION_FAILED", + "message": "Step 'submit-payment' failed with Stellar error tx_failed / op_underfunded", + "category": "stellar", + "retryable": true, + "failedStepId": "submit-payment", + "stellarTransactionCode": "tx_failed", + "stellarOperationCodes": ["op_underfunded"] +} +``` + +- **code** (string): Machine-readable error identifier. +- **message** (string): Human-readable error explanation. +- **category** (string): Error category (`stellar`, `network`, `timeout`, `capacity`, `validation`, or `internal`). +- **retryable** (boolean): Whether retrying the run is likely to succeed. +- **failedStepId** (string, optional): The step ID that caused the error. +- **stellarTransactionCode** (string, optional): Stellar's transaction-level error code. +- **stellarOperationCodes** (array, optional): Stellar's per-operation error codes. + +### Decimal strings + +All numeric amounts (balances, transaction values, fees) are represented as decimal strings, never as JSON numbers. This preserves precision and avoids floating-point rounding errors: + +- ✓ Correct: `"amount": "100.0000000"`, `"expected": "25.5000001"` +- ✗ Incorrect: `"amount": 100, "expected": 25.5` + +Decimal strings follow Stellar's stroop representation (at most 7 fractional digits). diff --git a/esure-docs/TESTNET_RESET_RECOVERY.md b/esure-docs/TESTNET_RESET_RECOVERY.md new file mode 100644 index 0000000..090f8d3 --- /dev/null +++ b/esure-docs/TESTNET_RESET_RECOVERY.md @@ -0,0 +1,108 @@ +# Testnet Reset Recovery Runbook + +When the Stellar Testnet undergoes a reset or network downtime, historical transaction links and generated accounts may become unavailable. This runbook explains how to recognize network-related failures and safely recover by rerunning scenarios with fresh accounts. + +## Symptoms of different failure types + +### Network reset +A network reset clears all Testnet ledger state, account balances, and transaction history. You may observe: +- Friendbot funding fails with "no such account" or "account not found" +- Previously funded accounts report zero balances +- Transaction lookups return "not found" on Horizon +- New account generation works, but old accounts cannot be reused +- Error logs reference account or transaction state that no longer exists + +Recovery action: [Rerun scenarios with fresh accounts](#rerunning-scenarios-with-fresh-accounts) + +### Backend restart without network reset +The Esure backend or Horizon service may restart independently. You may observe: +- Transient connection timeouts or "service unavailable" errors +- Existing accounts and transactions remain available +- Retrying the same run with the same account succeeds +- Network state is consistent within a few seconds after restart + +Recovery action: Wait briefly and [retry the run](#retrying-failed-runs); no fresh accounts needed. + +### Report retention expiration +Esure reports are retained for a configured duration (default: 1 hour). You may observe: +- Run reports return `409 CONFLICT` or "not complete" on download +- Recent transaction lookups on Horizon succeed (the network did not reset) +- The run ID is correct but the report is no longer available locally +- The transaction hash still resolves on Horizon + +Recovery action: The run itself is finished; no recovery is needed. Save reports immediately after runs complete, or configure a longer `RUN_RETENTION_MS` in the backend if you need longer access. + +## Official network status + +Testnet network resets are announced on the [Stellar Community Forum](https://stellar.community/) and the [Stellar Status page](https://stellar.status.io/). Before declaring a network reset: + +1. Check the [Testnet Horizon health endpoint](https://horizon-testnet.stellar.org/health) +2. Visit the Stellar Community Forum and search for recent reset announcements +3. Verify your own Friendbot requests against a fresh generated account + +Do not hardcode reset schedules or monitor dates in your code. Always consult the official sources above. + +## Rerunning scenarios with fresh accounts + +When you confirm a network reset has occurred, reruns are safe and expected: + +1. **Retrieve your scenario definition**: If you saved the scenario YAML/JSON file, reuse the exact same file. If you lost the definition, recreate it from your requirements. +2. **Generate fresh accounts**: Esure scenarios always use `generate: true` for accounts, so each run automatically creates fresh accounts. No manual account creation is needed. +3. **Resubmit the scenario**: POST to `/api/v1/runs` or use the CLI with your scenario definition. +4. **Verify transaction links**: After the run completes, check that new transaction links resolve on Horizon. If they do not, the network reset may still be in progress; wait and retry. +5. **Save the report immediately**: Use `GET /api/v1/runs/{runId}/report` to download the sanitized report before the retention window expires. + +Example using the CLI: + +```bash +cd esure-cli +npm ci +npm run build +node dist/cli.js run ./my-scenario.yaml --output json > report-after-reset.json +``` + +## Preserving offline reports + +Esure reports are JSON documents and can be safely stored offline: + +1. Download the report from `GET /api/v1/runs/{runId}/report` immediately after a run completes. +2. Save it as a JSON file in your version-control system, artifact store, or documentation. +3. Offline reports do not expire and can be shared, analyzed, or reaudited later. +4. Transaction links in old reports may stop resolving on Horizon after a network reset, but the historical data in the report itself remains valid. + +Example saving a report: + +```bash +curl https://esure.onrender.com/api/v1/runs/abc123/report > historical-report.json +``` + +## Troubleshooting + +### "Friendbot unavailable" errors +Friendbot (the Testnet faucet) may be temporarily down independently of a network reset. Verify: +- Recent transaction lookups on Horizon work (network is online) +- New accounts generate successfully (your code runs) +- Fresh Friendbot requests on simple accounts succeed (faucet is available) + +If Friendbot is down, wait 5-10 minutes and retry with a fresh run. If the issue persists, check the [Stellar Community Forum](https://stellar.community/) for status updates. + +### "Transaction not found" on Horizon +After a run succeeds, transaction links may fail to resolve if: +- The network reset after the transaction was confirmed (expected, see [Network reset](#network-reset)) +- Report retention expired and the run record was deleted from Esure (your local copy, if saved, is still accurate) +- A rare network synchronization issue occurred (wait a minute and retry the link) + +If a recent transaction link does not resolve but Horizon is online, consult the [Stellar Status page](https://stellar.status.io/) and wait for network synchronization. + +### Multiple failed runs +If several consecutive runs fail with consistent network errors: +1. Verify the backend is online: `curl https://esure.onrender.com/health` +2. Check the [Stellar Status page](https://stellar.status.io/) for ongoing incidents +3. Wait 10-15 minutes, as Testnet may be recovering from an outage +4. Retry a simple bundled scenario like `xlm-payment` to isolate your scenario definition from network issues + +## See also + +- [Scenario Schema](./SCENARIOS.md) — Scenario definition format and validation +- [Backend API](./API.md) — Run creation, status polling, and report endpoints +- [System Architecture](./ARCHITECTURE.md) — How Esure runs scenarios and persists reports diff --git a/esure-frontend/package.json b/esure-frontend/package.json index e7e6399..ebc0b3b 100644 --- a/esure-frontend/package.json +++ b/esure-frontend/package.json @@ -8,6 +8,7 @@ "start": "next start", "test": "vitest run", "test:e2e": "playwright test", + "test:e2e:prod": "TEST_MODE=production playwright test", "typecheck": "tsc --noEmit", "check": "npm run typecheck && npm test && npm run build" }, diff --git a/esure-frontend/playwright.config.ts b/esure-frontend/playwright.config.ts index 57129b1..848d948 100644 --- a/esure-frontend/playwright.config.ts +++ b/esure-frontend/playwright.config.ts @@ -1,14 +1,21 @@ import { defineConfig, devices } from "@playwright/test"; +const port = process.env.PORT || 3000; +const baseURL = `http://localhost:${port}`; +const isProduction = process.env.TEST_MODE === "production"; + export default defineConfig({ testDir: "./e2e", fullyParallel: true, forbidOnly: Boolean(process.env.CI), retries: process.env.CI ? 2 : 0, workers: process.env.CI ? 1 : undefined, - reporter: "list", + reporter: [ + "list", + ["html", { outputFolder: "playwright-report" }], + ], use: { - baseURL: `http://localhost:${process.env.PORT || 3000}`, + baseURL, trace: "on-first-retry", }, projects: [ @@ -25,10 +32,17 @@ export default defineConfig({ use: { ...devices["Desktop Safari"] }, }, ], - webServer: { - command: `npm run dev -- --port ${process.env.PORT || 3000}`, - url: `http://localhost:${process.env.PORT || 3000}`, - reuseExistingServer: !process.env.CI, - timeout: 120_000, - }, + webServer: isProduction + ? { + command: `npm run start -- --port ${port}`, + url: baseURL, + reuseExistingServer: false, + timeout: 120_000, + } + : { + command: `npm run dev -- --port ${port}`, + url: baseURL, + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, });