From 37cd394285195e41addc5a2875cf89bfb445ca62 Mon Sep 17 00:00:00 2001 From: BonelessWater Date: Wed, 14 Oct 2026 16:39:55 -0400 Subject: [PATCH 1/3] QT desk: local simulator (sim/) The real AlgoLens, the real engine (engine-rpc) and Postgres in Docker, on one clock that sim/simctl moves. libfaketime is preloaded in postgres, engine-rpc and the backend; all three read one offset from sim/clock/faketime.rc, so 09:29, 09:31 and 10:01 New York can be tested without waiting. Mailpit catches every e-mail. - sim/docker-compose.sim.yml, thin Dockerfiles adding libfaketime, an edge on :8080 with the production routes - sim/simctl (+ simctl.ps1): build, up, load, reset, save, clock, day, model-run, pass, status, mail, logs, psql - sim/snapshot.sh: read-only production snapshot over ssh (run by hand) - sim/seed: the sim users, and a synthetic fixture from trade-ngin's QT migrations for checking the stack without a snapshot - SIM_MODE=1 only: GET /api/sim and an App-level "SIMULATION" banner Claude-Session: https://claude.ai/code/session_01VCt4NHq2wcnzSiVtdabnEJ --- algolens-api/algolens/adapters/http/sim.py | 38 + .../infrastructure/config/app_factory.py | 5 + algolens-api/tests/test_sim_mode.py | 38 + algolens-frontend/src/App.tsx | 2 + .../src/components/SimBanner.tsx | 66 + .../src/domain/sim/simClock.test.ts | 21 + algolens-frontend/src/domain/sim/simClock.ts | 39 + .../src/infrastructure/api/simApi.ts | 14 + sim/.env.example | 7 + sim/.gitattributes | 3 + sim/.gitignore | 5 + sim/README.md | 212 +++ sim/backend/Dockerfile | 10 + sim/clock/.gitkeep | 0 sim/data/.gitkeep | 0 sim/docker-compose.sim.yml | 177 +++ sim/edge/edge.conf | 20 + sim/engine/Dockerfile | 27 + sim/engine/sim_config.py | 40 + sim/postgres/Dockerfile | 7 + sim/seed/build-synthetic.sh | 42 + sim/seed/synthetic-base.sql | 157 ++ sim/seed/synthetic-day.sql | 83 ++ sim/seed/synthetic.sql | 1311 +++++++++++++++++ sim/seed/users.sql | 34 + sim/simctl | 414 ++++++ sim/simctl.ps1 | 7 + sim/snapshot.sh | 114 ++ 28 files changed, 2893 insertions(+) create mode 100644 algolens-api/algolens/adapters/http/sim.py create mode 100644 algolens-api/tests/test_sim_mode.py create mode 100644 algolens-frontend/src/components/SimBanner.tsx create mode 100644 algolens-frontend/src/domain/sim/simClock.test.ts create mode 100644 algolens-frontend/src/domain/sim/simClock.ts create mode 100644 algolens-frontend/src/infrastructure/api/simApi.ts create mode 100644 sim/.env.example create mode 100644 sim/.gitattributes create mode 100644 sim/.gitignore create mode 100644 sim/README.md create mode 100644 sim/backend/Dockerfile create mode 100644 sim/clock/.gitkeep create mode 100644 sim/data/.gitkeep create mode 100644 sim/docker-compose.sim.yml create mode 100644 sim/edge/edge.conf create mode 100644 sim/engine/Dockerfile create mode 100644 sim/engine/sim_config.py create mode 100644 sim/postgres/Dockerfile create mode 100644 sim/seed/build-synthetic.sh create mode 100644 sim/seed/synthetic-base.sql create mode 100644 sim/seed/synthetic-day.sql create mode 100644 sim/seed/synthetic.sql create mode 100644 sim/seed/users.sql create mode 100644 sim/simctl create mode 100644 sim/simctl.ps1 create mode 100644 sim/snapshot.sh diff --git a/algolens-api/algolens/adapters/http/sim.py b/algolens-api/algolens/adapters/http/sim.py new file mode 100644 index 000000000..e8efc04c7 --- /dev/null +++ b/algolens-api/algolens/adapters/http/sim.py @@ -0,0 +1,38 @@ +"""Simulation mode: GET /api/sim, for the local simulator under sim/. + +Registered only when SIM_MODE=1 (sim/docker-compose.sim.yml sets it). The +frontend asks this route once on load; a 404 (every real deployment) means no +banner. With SIM_MODE=1 it answers the server's clock, which in the simulator +is the moved sim clock, so the banner shows the time the desk rules use. + +No login needed: the banner also shows on the login page. It reveals only the +server time. +""" + +import os +from datetime import datetime, timezone +from zoneinfo import ZoneInfo + +from flask import Blueprint, jsonify + +NEW_YORK = ZoneInfo("America/New_York") + +sim_bp = Blueprint("sim", __name__) + + +def sim_mode_enabled(environ=None): + env = os.environ if environ is None else environ + return (env.get("SIM_MODE") or "").strip() == "1" + + +def sim_state(now): + return { + "sim": True, + "serverTime": now.astimezone(timezone.utc).isoformat(), + "newYork": now.astimezone(NEW_YORK).isoformat(), + } + + +@sim_bp.route("", methods=["GET"]) +def get_sim(): + return jsonify(sim_state(datetime.now(timezone.utc))), 200 diff --git a/algolens-api/algolens/infrastructure/config/app_factory.py b/algolens-api/algolens/infrastructure/config/app_factory.py index 90c1c6db3..de3f8d084 100644 --- a/algolens-api/algolens/infrastructure/config/app_factory.py +++ b/algolens-api/algolens/infrastructure/config/app_factory.py @@ -16,6 +16,7 @@ from algolens.adapters.http.auth import auth_bp from algolens.adapters.http.desk import desk_bp from algolens.adapters.http.portfolio import portfolio_bp +from algolens.adapters.http.sim import sim_bp, sim_mode_enabled from algolens.infrastructure.db.postgres import get_db_connection from extensions import limiter @@ -205,6 +206,10 @@ def token_verification_failed_callback(jwt_header, jwt_payload): app.register_blueprint(portfolio_bp, url_prefix="/portfolio") app.register_blueprint(desk_bp, url_prefix="/portfolio/desk") app.register_blueprint(api_bp, url_prefix="/api") + if sim_mode_enabled(): + # Local simulator only (sim/): the banner's clock. Absent otherwise. + app.logger.warning("[SIM_MODE] SIM_MODE=1: GET /api/sim is on (simulator)") + app.register_blueprint(sim_bp, url_prefix="/api/sim") @app.route("/health/live", methods=["GET"]) def liveness_check(): diff --git a/algolens-api/tests/test_sim_mode.py b/algolens-api/tests/test_sim_mode.py new file mode 100644 index 000000000..15cc7a722 --- /dev/null +++ b/algolens-api/tests/test_sim_mode.py @@ -0,0 +1,38 @@ +"""GET /api/sim exists only with SIM_MODE=1 (the local simulator, sim/).""" + +from datetime import datetime, timezone + +from algolens.adapters.http.sim import sim_mode_enabled, sim_state +from algolens.infrastructure.config.app_factory import create_app + + +def test_sim_mode_flag(): + assert sim_mode_enabled({"SIM_MODE": "1"}) + assert not sim_mode_enabled({}) + assert not sim_mode_enabled({"SIM_MODE": "0"}) + assert not sim_mode_enabled({"SIM_MODE": "true"}) + + +def test_sim_state_reports_utc_and_new_york(): + state = sim_state(datetime(2026, 10, 15, 13, 29, 5, tzinfo=timezone.utc)) + assert state == { + "sim": True, + "serverTime": "2026-10-15T13:29:05+00:00", + "newYork": "2026-10-15T09:29:05-04:00", + } + + +def test_route_absent_without_sim_mode(monkeypatch): + monkeypatch.delenv("SIM_MODE", raising=False) + app = create_app() + assert app.test_client().get("/api/sim").status_code == 404 + + +def test_route_present_with_sim_mode(monkeypatch): + monkeypatch.setenv("SIM_MODE", "1") + app = create_app() + resp = app.test_client().get("/api/sim") + assert resp.status_code == 200 + body = resp.get_json() + assert body["sim"] is True + datetime.fromisoformat(body["serverTime"]) diff --git a/algolens-frontend/src/App.tsx b/algolens-frontend/src/App.tsx index b7f820a1a..8e5cf7e46 100644 --- a/algolens-frontend/src/App.tsx +++ b/algolens-frontend/src/App.tsx @@ -7,6 +7,7 @@ import { QtHome } from './components/qt/QtHome'; import { InvestorHome } from './components/investor/InvestorHome'; import { AdminHome } from './components/admin/AdminHome'; import { ScreenFrame } from './components/ScreenFrame'; +import { SimBanner } from './components/SimBanner'; import { AccessPending } from './components/AccessPending'; import { ThemeProvider, useTheme } from './adapters/react/ThemeContext'; import { AuthProvider, useAuth } from './adapters/react/AuthContext'; @@ -93,6 +94,7 @@ export default function App() { + ); diff --git a/algolens-frontend/src/components/SimBanner.tsx b/algolens-frontend/src/components/SimBanner.tsx new file mode 100644 index 000000000..0b3d18096 --- /dev/null +++ b/algolens-frontend/src/components/SimBanner.tsx @@ -0,0 +1,66 @@ +import { useEffect, useState } from 'react'; +import { SIM_POLL_MS, simBannerText, simOffsetMs } from '../domain/sim/simClock'; +import { fetchSimState } from '../infrastructure/api/simApi'; + +/** + * A fixed banner shown only by the local simulator (backend SIM_MODE=1). On a + * real deployment GET /api/sim is a 404 and this renders nothing and stops. + */ +export function SimBanner() { + const [offsetMs, setOffsetMs] = useState(null); + const [now, setNow] = useState(() => Date.now()); + + useEffect(() => { + let stopped = false; + let poll: ReturnType | undefined; + const refresh = async () => { + const state = await fetchSimState(); + if (stopped) return; + if (!state) { + if (poll) clearInterval(poll); + return; + } + const offset = simOffsetMs(state.serverTime, Date.now()); + if (Number.isFinite(offset)) setOffsetMs(offset); + }; + refresh().then(() => { + if (!stopped) poll = setInterval(refresh, SIM_POLL_MS); + }); + return () => { + stopped = true; + if (poll) clearInterval(poll); + }; + }, []); + + useEffect(() => { + if (offsetMs === null) return; + const tick = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(tick); + }, [offsetMs]); + + if (offsetMs === null) return null; + return ( + // Inline styles: index.css is a precompiled Tailwind build, so a class it + // does not already contain would do nothing. +
+ {simBannerText(now + offsetMs)} +
+ ); +} diff --git a/algolens-frontend/src/domain/sim/simClock.test.ts b/algolens-frontend/src/domain/sim/simClock.test.ts new file mode 100644 index 000000000..eb4201bdc --- /dev/null +++ b/algolens-frontend/src/domain/sim/simClock.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest'; +import { newYorkClock, simBannerText, simOffsetMs } from './simClock'; + +describe('sim banner clock', () => { + it('formats an instant in New York time', () => { + expect(newYorkClock(Date.parse('2026-10-15T13:29:05Z'))).toBe('2026-10-15 09:29:05'); + expect(newYorkClock(Date.parse('2026-12-01T15:01:00Z'))).toBe('2026-12-01 10:01:00'); + }); + + it('says SIMULATION with the clock', () => { + expect(simBannerText(Date.parse('2026-10-15T14:01:00Z'))).toBe( + 'SIMULATION — clock 2026-10-15 10:01:00 New York', + ); + }); + + it('keeps the server offset so the banner can tick', () => { + const browser = Date.parse('2026-10-14T20:00:00Z'); + expect(simOffsetMs('2026-10-15T13:30:00+00:00', browser)).toBe(17.5 * 3600 * 1000); + expect(simOffsetMs('not a time', browser)).toBeNaN(); + }); +}); diff --git a/algolens-frontend/src/domain/sim/simClock.ts b/algolens-frontend/src/domain/sim/simClock.ts new file mode 100644 index 000000000..e951344c3 --- /dev/null +++ b/algolens-frontend/src/domain/sim/simClock.ts @@ -0,0 +1,39 @@ +/** + * The simulator banner (sim/ in the repo). The backend answers GET /api/sim + * only when SIM_MODE=1; its serverTime is the moved sim clock. The banner + * keeps the offset between that clock and the browser's, so it can tick + * between polls. + */ + +export const SIM_POLL_MS = 10_000; + +export interface SimState { + sim: boolean; + serverTime: string; +} + +/** Server clock minus browser clock, in ms (NaN when serverTime is unreadable). */ +export function simOffsetMs(serverTime: string, browserNowMs: number): number { + const server = Date.parse(serverTime); + return Number.isFinite(server) ? server - browserNowMs : Number.NaN; +} + +/** "2026-10-15 09:29:05" in New York for an epoch in ms. */ +export function newYorkClock(epochMs: number): string { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: 'America/New_York', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hourCycle: 'h23', + }).formatToParts(new Date(epochMs)); + const get = (type: string) => parts.find(p => p.type === type)?.value ?? ''; + return `${get('year')}-${get('month')}-${get('day')} ${get('hour')}:${get('minute')}:${get('second')}`; +} + +export function simBannerText(epochMs: number): string { + return `SIMULATION — clock ${newYorkClock(epochMs)} New York`; +} diff --git a/algolens-frontend/src/infrastructure/api/simApi.ts b/algolens-frontend/src/infrastructure/api/simApi.ts new file mode 100644 index 000000000..e46d63ff7 --- /dev/null +++ b/algolens-frontend/src/infrastructure/api/simApi.ts @@ -0,0 +1,14 @@ +import type { SimState } from '../../domain/sim/simClock'; +import { API_BASE_URL } from './httpClient'; + +/** GET /api/sim: the sim clock, or null when this is not the simulator (404) or it fails. */ +export async function fetchSimState(): Promise { + try { + const response = await fetch(`${API_BASE_URL}/api/sim`, { credentials: 'include' }); + if (!response.ok) return null; + const data = (await response.json()) as SimState; + return data && data.sim === true && typeof data.serverTime === 'string' ? data : null; + } catch { + return null; + } +} diff --git a/sim/.env.example b/sim/.env.example new file mode 100644 index 000000000..b07bb44c1 --- /dev/null +++ b/sim/.env.example @@ -0,0 +1,7 @@ +# Copy to sim/.env to override the simulator's defaults (docker compose reads it). +# ENGINE_IMAGE=trade-ngin:rpc-0b9dbf6e +# PG_IMAGE=postgres:16 +# SIM_FLASK_ENV=development +# QT_CATCHUP_PORTFOLIOS=qt_conservative,qt_conservative_model +# QT_CATCHUP_ENABLED=1 +# QT_EVENING_RUN=0 diff --git a/sim/.gitattributes b/sim/.gitattributes new file mode 100644 index 000000000..dc2ac8052 --- /dev/null +++ b/sim/.gitattributes @@ -0,0 +1,3 @@ +# Scripts and SQL run inside Linux containers or Git Bash: keep LF on Windows checkouts. +* text=auto eol=lf +*.ps1 text eol=crlf diff --git a/sim/.gitignore b/sim/.gitignore new file mode 100644 index 000000000..901330b16 --- /dev/null +++ b/sim/.gitignore @@ -0,0 +1,5 @@ +# Prod snapshots (snapshot.sh) and the clock state are local only. +/data/* +!/data/.gitkeep +/clock/faketime.rc +/clock/day diff --git a/sim/README.md b/sim/README.md new file mode 100644 index 000000000..a0abbaaba --- /dev/null +++ b/sim/README.md @@ -0,0 +1,212 @@ +# QT desk simulator + +The real AlgoLens, the real engine (`engine-rpc` and the C++ binary it runs) and Postgres, all on +one clock you can move. Use it to see what the desk and the engine do at 09:29, 09:31 or 10:01 +New York without waiting for that time. + +Everything runs locally in Docker. Nothing here talks to production, except `snapshot.sh`, which +you run yourself to take a read-only copy of the data. + +| What | Where | +|---|---| +| AlgoLens | http://localhost:8080 | +| Mailpit (every e-mail the engine sends) | http://localhost:8025 | +| Postgres | `localhost:55432`, user `postgres`, password `sim`, database `new_algo_data` | +| Users | `qt@sim.local` (desk), `vp@sim.local`, `president@sim.local`; password `sim` | + +## How the clock works + +`libfaketime` is preloaded in the postgres, engine-rpc and backend containers. All three read +`sim/clock/faketime.rc`, which holds one offset in seconds from real time (for example +`+63000`). `simctl clock` writes it, and every process sees the change within a second. The +clock keeps ticking from the time you set. `now()` in Postgres, `datetime.now()` in the backend, +the desk service's scheduler and the C++ engine all read the same time. + +The sim day is in `sim/clock/day`. `simctl clock 09:29` means 09:29 New York on that day. + +The offset is used instead of libfaketime's absolute `@YYYY-MM-DD HH:MM:SS` form because `@` +starts the clock at that time in every new process: each engine run the desk service spawns +would start at the same second again. With an offset, all processes agree. + +AlgoLens shows an amber banner, "SIMULATION — clock ... New York", at the top of every page. +It appears only when the backend runs with `SIM_MODE=1`, which only this compose file sets. + +## Prerequisites (Windows 11) + +- Docker Desktop, running. If it hangs (`docker ps` does not answer), run + `docker desktop restart`. +- Git Bash (comes with Git for Windows). Run `simctl` from Git Bash as `bash sim/simctl ...`, + or from PowerShell as `.\sim\simctl.ps1 ...`. The examples below use Git Bash, from the + repository root. +- The engine image built locally: `docker images trade-ngin` should list + `trade-ngin:rpc-0b9dbf6e`. To use another tag, put `ENGINE_IMAGE=trade-ngin:rpc-` in + `sim/.env` (or export it) and run `bash sim/simctl build`. +- Two browser profiles (or one normal window and one private window), so the desk and the VP + can be logged in at the same time. Use `http://localhost:8080`, not `127.0.0.1`: the session + cookie is marked Secure, which browsers accept on plain HTTP only for `localhost`. + +## First run + +1. Take a snapshot of production. You run this yourself. It only reads; nothing is written on + the server, and the copy lands in `sim/data/`: + + ```bash + bash sim/snapshot.sh + ``` + + It connects with `prod-deploy.pem`, reads the login from + `~/.config/algogators/db-app-roles.env` on the host when that file sets `PGUSER` and + `PGPASSWORD` (otherwise it asks you), and streams back over ssh: + + - the schema of `trading`, `metadata`, `futures_data`, `auth`, `ops`, `investor`; + - all of `metadata.contract_metadata`, `futures_data.ohlcv_1d` (the engine's market data), + `ops.data_ready` and `trading.strategy_registry`; + - the rows of `QT_CONSERVATIVE_PORTFOLIO` and `QT_CONSERVATIVE_MODEL_PORTFOLIO` in + `trading.positions`, `executions`, `live_results`, `live_run_metadata`, `equity_curve`, + `position_overrides`, `strategy_config` and `signals`. + + No user rows are taken. The result is `sim/data/snapshot-/` (git-ignored). + +2. Build the images and load the snapshot: + + ```bash + bash sim/simctl build + bash sim/simctl load sim/data/snapshot- + ``` + + `load` restores the snapshot, adds the three sim users, and picks the sim day: the newest + QT day in the data. It removes that day's QT rows so the real engine can build the day + again. Use `--day YYYY-MM-DD` for another day, or `--keep` to keep the newest day as it was. + It sets the clock to 08:00 on the sim day and saves everything as a template, so + `simctl reset` can return to this point in seconds. + + If the snapshot came from a newer Postgres major version than 16, `load` warns; set + `PG_IMAGE=postgres:` in `sim/.env` and run `bash sim/simctl build` first. + +3. Start everything: + + ```bash + bash sim/simctl up + ``` + +4. Build the sim day's books with the real engine (the same command the scheduler runs): + + ```bash + bash sim/simctl model-run + ``` + + The scheduler would also do it on its own on the next pass (it runs every 5 minutes + between 06:30 and 10:30 sim time, or now with `bash sim/simctl pass`). + +5. Save this state as the template, so each `reset` starts from a built, unapproved day: + + ```bash + bash sim/simctl save + ``` + +6. Open http://localhost:8080. Log in as `qt@sim.local` / `sim` in one browser profile and as + `vp@sim.local` / `sim` in the other. Open http://localhost:8025 for the mail. + +## Everyday commands + +```bash +bash sim/simctl clock 09:29 # 09:29 New York on the sim day +bash sim/simctl clock +5m # forward 5 minutes (also -2m, +1h, +30s) +bash sim/simctl day 2026-10-16 # another sim day, same time of day +bash sim/simctl status # time in each container, the day's publish state, desk commands +bash sim/simctl pass # one scheduler pass now, instead of waiting for its 1-minute tick +bash sim/simctl reset # back to the template; clock 08:00; Mailpit emptied +bash sim/simctl logs # follow engine-rpc and backend logs +bash sim/simctl psql # psql on the sim database +bash sim/simctl down # stop (data kept); `destroy` also deletes the volumes +``` + +After a `reset` (or any move of the clock backwards) log in again: a session issued at 10:01 +is "not yet valid" at 08:00. + +After `git checkout `, run `bash sim/simctl build` and +`bash sim/simctl up` to run that branch's backend and frontend. The images are always built +from the working tree. + +## Test script + +Which flow you can test depends on the code you run. The "approve B" flow below (approve the +model's book unchanged, auto-approval, approve C after the dynamic optimisation) ships in +AlgoLens PR `qt/desk-pipeline` and trade-ngin PR #208. Until both are merged, check out +`qt/desk-pipeline` here (and build an engine image from #208, then set `ENGINE_IMAGE`), or +run today's flow: Save, then Approve. + +Start each scenario with `bash sim/simctl reset`, then log in again in both browsers. + +1. **Approve before 09:30, sent at 09:30.** + `clock 09:00`. As the desk, approve B unchanged: it is approved at once (auto-approved). + `status` shows the publish command done and `sent_at` empty. `clock 09:30`, then `pass` + (or wait a minute). Mailpit shows the day's e-mail, and `status` shows `sent_ny` 09:30. + +2. **A change that moves the dynamic optimisation.** + `reset`, `clock 09:10`. As the desk, set ZT to 8 and approve. The engine's dynamic + optimisation changes the book; the desk sees C. Approve C. Mailpit gets it at 09:30. + +3. **Override approved by the VP.** + `reset`, `clock 09:05`. As the desk, request an override for B. Mailpit shows the override + request e-mail to `vp@sim.local` and `president@sim.local`. Open its link in the VP's browser + (log in as `vp@sim.local`), approve. B is published; at 09:30 it is sent. + +4. **Approve between 09:30 and 10:00: sent at once.** + `reset`, `clock 09:45`. As the desk, approve. The e-mail goes out at once (Mailpit), and + `status` shows `sent_ny` about 09:45. + +5. **Nothing approved: the 10:00 fallback.** + `reset`, `clock 10:01`, then `pass`. The engine publishes the model's book A + (`publish_source` fallback, `published_by` `system:fallback-10am`) and e-mails it. The + President gets an alert. + +6. **Too late.** + Right after scenario 5 (or `reset`, `clock 10:02`), try to approve as the desk. AlgoLens + refuses: "Approval of the book closed at 10:00 New York". + +## Without a snapshot: the synthetic fixture + +`bash sim/simctl load synthetic` restores `sim/seed/synthetic.sql`: trade-ngin's QT schema +(migrations 021 to 029 read from the engine image) and one seeded QT day, 2026-10-15, with +three books of five symbols. The desk panel renders and the clock rules apply, but it is not +market data: every engine run (model run, approval, fallback) fails on it, and the scheduler +mails the President an alert. Use it to check the stack, not the engine. Rebuild it with +`bash sim/seed/build-synthetic.sh` after an engine change. + +## Files + +| File | What | +|---|---| +| `docker-compose.sim.yml` | postgres, mailpit, engine-rpc, backend, frontend, edge | +| `simctl`, `simctl.ps1` | the driver, and its PowerShell wrapper | +| `snapshot.sh` | the read-only production snapshot (you run it) | +| `postgres/Dockerfile` | postgres:16 with libfaketime | +| `engine/Dockerfile`, `engine/sim_config.py` | the engine image with libfaketime and `/app/config` (database: sim postgres; e-mail: Mailpit) | +| `backend/Dockerfile` | the AlgoLens backend image with libfaketime | +| `edge/edge.conf` | plain-HTTP edge with `deployment/edge-routes.conf` | +| `seed/users.sql` | the three sim users and their screens | +| `seed/synthetic*.sql`, `seed/build-synthetic.sh` | the synthetic fixture | +| `clock/` | `faketime.rc` (the offset) and `day` (the sim day), both git-ignored | +| `data/` | snapshots, git-ignored | + +## Notes + +- **Settings.** The backend runs with `FLASK_ENV=production` (CORS allow-list, JWT secret + required, Secure cookies), `QT_DESK_ENABLED=1`, `QT_APPROVERS=vp=vp@sim.local,president=president@sim.local` + and `SIM_MODE=1`. If a browser refuses the cookie, set `SIM_FLASK_ENV=development` in + `sim/.env` and `bash sim/simctl up`. The engine runs with `QT_CATCHUP_ENABLED=1`, + `QT_CATCHUP_PORTFOLIOS=qt_conservative` (the config directory of `QT_CONSERVATIVE_PORTFOLIO`; + add `qt_conservative_model` for the twin) and `QT_EVENING_RUN=0` (the 06:45 schedule, no + `ops.data_ready`); each can be overridden in `sim/.env`. +- **E-mail.** Both the C++ mailer and the desk service's Python mailer log in, and the C++ + mailer always asks for STARTTLS. Mailpit therefore runs with a throwaway self-signed + certificate and accepts any login (`sim`/`sim` in every `email.json`). All mail is caught; + nothing leaves your machine. +- **libfaketime.** `FAKETIME_DISABLE_SHM=1` is required: without it postgres hangs at start + (gosu's child blocks on a semaphore). The engine image gets libfaketime built from source: + Ubuntu 24.04's package breaks Python's `time.sleep()` under an offset. +- **Database login.** The apps connect as `postgres` (superuser) in the simulator. Production + grants (for example 025's revokes on `position_overrides`) are therefore not enforced here. +- **Restore errors.** `load` does not stop on an error in the schema (an extension the sim + lacks, for example). It reports the count and writes them to `sim/data/.load.log`. diff --git a/sim/backend/Dockerfile b/sim/backend/Dockerfile new file mode 100644 index 000000000..8ffa35b27 --- /dev/null +++ b/sim/backend/Dockerfile @@ -0,0 +1,10 @@ +# The AlgoLens backend image built from the checked-out branch (algolens-api/Dockerfile, tagged +# algolens-sim/backend-base:local by `simctl build`), with libfaketime added for the sim clock. +ARG BASE_IMAGE=algolens-sim/backend-base:local +FROM ${BASE_IMAGE} +USER root +RUN apt-get update \ + && apt-get install -y --no-install-recommends libfaketime \ + && rm -rf /var/lib/apt/lists/* \ + && ls -l /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1 +USER appuser diff --git a/sim/clock/.gitkeep b/sim/clock/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sim/data/.gitkeep b/sim/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sim/docker-compose.sim.yml b/sim/docker-compose.sim.yml new file mode 100644 index 000000000..c532b6392 --- /dev/null +++ b/sim/docker-compose.sim.yml @@ -0,0 +1,177 @@ +# The QT desk simulator: real AlgoLens, the real engine (engine-rpc) and Postgres, all on one +# clock you can move. Local only. See sim/README.md; drive it with sim/simctl. +# +# http://localhost:8080 AlgoLens (edge nginx with the production routes) +# http://localhost:8025 Mailpit (every e-mail the engine and the desk service send) +# localhost:55432 Postgres (user postgres, password sim, database new_algo_data) +# +# The clock: libfaketime is preloaded in postgres, engine-rpc and backend. All three read +# sim/clock/faketime.rc (bind-mounted at /sim-clock), which holds one offset in seconds from real +# time ("+63000"). simctl writes it; every process picks a change up within a second. +name: algolens-sim + +x-faketime: &faketime + LD_PRELOAD: /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1 + FAKETIME_TIMESTAMP_FILE: /sim-clock/faketime.rc + # Re-read the file at most once a second (FAKETIME_NO_CACHE=1 re-reads it on every call). + FAKETIME_CACHE_DURATION: "1" + # Without it a child process started by another user (postgres via gosu) blocks on the + # parent's semaphore and hangs. + FAKETIME_DISABLE_SHM: "1" + # Timers and timeouts (gRPC, gunicorn, postgres) keep real monotonic time, so a clock jump + # backwards never stalls them. + FAKETIME_DONT_FAKE_MONOTONIC: "1" + FAKETIME_DONT_RESET: "1" + +x-db-env: &db-env + DB_HOST: postgres + DB_PORT: "5432" + DB_USER: postgres + DB_PASSWORD: sim + DB_NAME: new_algo_data + +x-qt-approvers: &qt-approvers "vp=vp@sim.local,president=president@sim.local" + +services: + postgres: + build: + context: ./postgres + args: + PG_IMAGE: ${PG_IMAGE:-postgres:16} + image: algolens-sim/postgres:local + environment: + <<: *faketime + POSTGRES_PASSWORD: sim + POSTGRES_DB: new_algo_data + ports: + - "127.0.0.1:55432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + - ./clock:/sim-clock:ro + - ./data:/sim-data:ro + - ./seed:/sim-seed:ro + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 5s + timeout: 5s + retries: 20 + + mailpit: + image: axllent/mailpit:v1.27 + environment: + # The engine's C++ mailer always asks for STARTTLS and logs in, and the desk service's + # Python mailer logs in: a throwaway self-signed certificate and any login are accepted. + MP_SMTP_TLS_CERT: sans:mailpit + MP_SMTP_TLS_KEY: sans:mailpit + MP_SMTP_AUTH_ACCEPT_ANY: "true" + MP_SMTP_AUTH_ALLOW_INSECURE: "true" + ports: + - "127.0.0.1:8025:8025" + - "127.0.0.1:1025:1025" + + engine-rpc: + build: + context: ./engine + args: + ENGINE_IMAGE: ${ENGINE_IMAGE:-trade-ngin:rpc-0b9dbf6e} + image: algolens-sim/engine-rpc:local + command: rpc + init: true + depends_on: + postgres: + condition: service_healthy + mailpit: + condition: service_started + environment: + <<: [*faketime, *db-env] + QT_APPROVERS: *qt-approvers + QT_CATCHUP_ENABLED: ${QT_CATCHUP_ENABLED:-1} + # Config directory names under /app/config/portfolios (the scheduler's unit), not ids: + # qt_conservative is QT_CONSERVATIVE_PORTFOLIO. + QT_CATCHUP_PORTFOLIOS: ${QT_CATCHUP_PORTFOLIOS:-qt_conservative} + QT_EVENING_RUN: ${QT_EVENING_RUN:-0} + QT_APPROVE_URL_BASE: http://localhost:8080/qt/approve + QT_DESK_URL: http://localhost:8080 + QT_REVIEW_RECIPIENTS: desk@sim.local + QT_LOG_DIR: /var/log/qt-engine + volumes: + - ./clock:/sim-clock:ro + - qt-locks:/tmp/qt-locks + - qt-logs:/var/log/qt-engine + networks: + default: + aliases: [desk-agent] + healthcheck: + test: ["CMD", "sh", "-c", "cd /opt/rpc/app && exec /opt/rpc/venv/bin/python -m algogators_rpc.healthcheck"] + interval: 30s + timeout: 10s + start_period: 20s + retries: 3 + + # AlgoLens backend as CI builds it (algolens-api/Dockerfile of the checked-out branch), then + # libfaketime on top (sim/backend/Dockerfile). Rebuild after a checkout: simctl build. + backend-base: + build: + context: ../algolens-api + image: algolens-sim/backend-base:local + profiles: ["build"] + + backend: + build: + context: ./backend + args: + BASE_IMAGE: algolens-sim/backend-base:local + image: algolens-sim/backend:local + command: + - gunicorn + - --bind=0.0.0.0:5000 + - --workers=2 + - --threads=2 + - --timeout=60 + - wsgi:app + depends_on: + postgres: + condition: service_healthy + environment: + <<: [*faketime, *db-env] + # Production settings (CORS allow-list, JWT secret required, Secure cookies). Browsers + # accept Secure cookies on http://localhost; set SIM_FLASK_ENV=development otherwise. + FLASK_ENV: ${SIM_FLASK_ENV:-production} + CORS_ORIGINS: http://localhost:8080 + JWT_SECRET_KEY: sim-only-jwt-secret-not-for-production-0000 + QT_DESK_ENABLED: "1" + QT_APPROVERS: *qt-approvers + ENGINE_RPC_ADDR: engine-rpc:50051 + SIM_MODE: "1" + volumes: + - ./clock:/sim-clock:ro + healthcheck: + test: ["CMD", "curl", "-fsS", "--max-time", "5", "http://localhost:5000/health/live"] + interval: 15s + timeout: 10s + retries: 3 + start_period: 20s + + frontend: + build: + context: ../algolens-frontend + args: + VITE_API_URL: http://localhost:8080 + image: algolens-sim/frontend:local + + # The production edge routes (deployment/edge-routes.conf) behind plain HTTP on :8080. + edge: + image: nginx:1.27-alpine + depends_on: + - backend + - frontend + ports: + - "127.0.0.1:8080:80" + volumes: + - ./edge/edge.conf:/etc/nginx/conf.d/default.conf:ro + - ../deployment/edge-routes.conf:/etc/nginx/snippets/edge-routes.conf:ro + +volumes: + pgdata: + qt-locks: + qt-logs: diff --git a/sim/edge/edge.conf b/sim/edge/edge.conf new file mode 100644 index 000000000..7a334ea04 --- /dev/null +++ b/sim/edge/edge.conf @@ -0,0 +1,20 @@ +# The simulator's edge: deployment/edge.conf's legacy listener, on plain HTTP, with the same +# routes (deployment/edge-routes.conf is mounted from the checkout). +map $http_upgrade $edge_connection { + "" ""; + default upgrade; +} + +limit_req_zone $binary_remote_addr zone=edge_auth:5m rate=5r/s; +limit_req_status 429; +server_tokens off; + +server { + listen 80 default_server; + server_name _; + + set $fwd_for $remote_addr; + set $fwd_proto http; + + include /etc/nginx/snippets/edge-routes.conf; +} diff --git a/sim/engine/Dockerfile b/sim/engine/Dockerfile new file mode 100644 index 000000000..bfc1e6230 --- /dev/null +++ b/sim/engine/Dockerfile @@ -0,0 +1,27 @@ +# The trade-ngin image (engine-rpc entrypoint) with libfaketime added, so the gRPC server, the +# catch-up scheduler and every engine run it spawns (flock + the C++ binary) read the sim clock. +# +# libfaketime is built from source: Ubuntu 24.04's package (0.9.10-2.1) fails Python's +# time.sleep() with EINVAL under a clock offset (its clock_nanosleep shifts absolute monotonic +# deadlines), which kills every engine run the desk service starts. The commit below is the one +# Debian trixie packages (0.9.10+2024-06-05+gba9ed5b2), which postgres and the backend use. +# +# /app/config is built here from the image's own config_template (so portfolio.json and +# risk.json always match the engine version), with the database block pointed at the sim +# postgres and every portfolio's email.json pointed at Mailpit (sim_config.py). +ARG ENGINE_IMAGE=trade-ngin:rpc-0b9dbf6e + +FROM ubuntu:24.04 AS libfaketime +ARG LIBFAKETIME_COMMIT=ba9ed5b2 +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates gcc git libc6-dev make \ + && git clone https://github.com/wolfcw/libfaketime.git /src \ + && cd /src && git checkout "$LIBFAKETIME_COMMIT" \ + && make -C src libfaketime.so.1 \ + && install -D -m 0644 src/libfaketime.so.1 /out/libfaketime.so.1 + +FROM ${ENGINE_IMAGE} +COPY --from=libfaketime /out/libfaketime.so.1 /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1 +COPY sim_config.py /tmp/sim_config.py +RUN rm -rf /app/config && cp -r /app/config_template /app/config \ + && python3 /tmp/sim_config.py /app/config && rm /tmp/sim_config.py diff --git a/sim/engine/sim_config.py b/sim/engine/sim_config.py new file mode 100644 index 000000000..ada5782ba --- /dev/null +++ b/sim/engine/sim_config.py @@ -0,0 +1,40 @@ +"""Turn a copy of trade-ngin's config_template into the simulator's /app/config. + +* defaults.json: the database block points at the sim postgres (the C++ engine reads its + database settings only from here; the Python desk service reads DB_* first). +* portfolios/*/email.json: SMTP is Mailpit. The C++ mailer always uses STARTTLS and logs in, + so use_tls stays true and a dummy login is set; Mailpit accepts any login. +""" + +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) + +defaults_path = root / "defaults.json" +defaults = json.loads(defaults_path.read_text(encoding="utf-8")) +defaults["database"] = { + "host": "postgres", + "port": "5432", + "username": "postgres", + "password": "sim", + "name": "new_algo_data", + "num_connections": 5, +} +defaults_path.write_text(json.dumps(defaults, indent=2) + "\n", encoding="utf-8") + +email = { + "_description": "Simulator: every e-mail goes to Mailpit (http://localhost:8025)", + "smtp_host": "mailpit", + "smtp_port": 1025, + "username": "sim", + "password": "sim", + "from_email": "qt-engine@sim.local", + "to_emails": ["desk@sim.local"], + "to_emails_production": ["desk@sim.local"], + "use_tls": True, +} +for portfolio in sorted(p for p in (root / "portfolios").iterdir() if p.is_dir()): + (portfolio / "email.json").write_text(json.dumps(email, indent=2) + "\n", encoding="utf-8") + print(f"sim config: {portfolio.name}") diff --git a/sim/postgres/Dockerfile b/sim/postgres/Dockerfile new file mode 100644 index 000000000..67795df97 --- /dev/null +++ b/sim/postgres/Dockerfile @@ -0,0 +1,7 @@ +# postgres with libfaketime: every process reads the sim clock (sim/clock/faketime.rc). +ARG PG_IMAGE=postgres:16 +FROM ${PG_IMAGE} +RUN apt-get update \ + && apt-get install -y --no-install-recommends libfaketime \ + && rm -rf /var/lib/apt/lists/* \ + && ls -l /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1 diff --git a/sim/seed/build-synthetic.sh b/sim/seed/build-synthetic.sh new file mode 100644 index 000000000..c65303d1e --- /dev/null +++ b/sim/seed/build-synthetic.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Rebuild sim/seed/synthetic.sql: the synthetic fixture `simctl load synthetic` restores. +# +# synthetic-base.sql the tables the QT migrations build on, and auth.users +# trade-ngin migrations 014, 017 and 021-029, read from the engine image +# (ENGINE_IMAGE), so they match the engine under test +# algolens-api/migrations/003 screen access (auth.user_screen_access, investor) +# synthetic-day.sql one QT day +# +# trade-ngin's other migrations need tables the fixture does not have (backtest.*, signals). +# +# bash sim/seed/build-synthetic.sh +set -euo pipefail +export MSYS_NO_PATHCONV=1 + +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/../.." && pwd)" +image="${ENGINE_IMAGE:-trade-ngin:rpc-0b9dbf6e}" +migrations=(014_contract_metadata_fee_per_contract 017_live_results_roll_costs 021_qt_books + 022_strategy_config 023_qt_command_log 024_drop_retired_qt_tables + 025_qt_command_log_hardening 027_qt_daily_cutoff 028_ops_data_ready 029_qt_rescind) + +out="$here/synthetic.sql" +{ + echo "-- GENERATED by sim/seed/build-synthetic.sh from $image; do not edit." + echo "-- Restored into an empty new_algo_data by \`simctl load synthetic\`." + echo "\\set ON_ERROR_STOP on" + echo + cat "$here/synthetic-base.sql" + for m in "${migrations[@]}"; do + echo + echo "-- ===== trade-ngin migrations/$m.sql ($image)" + docker run --rm --entrypoint cat "$image" "/app/migrations/$m.sql" + done + echo + echo "-- ===== algolens-api/migrations/003_screen_access.sql" + cat "$repo/algolens-api/migrations/003_screen_access.sql" + echo + echo "-- ===== sim/seed/synthetic-day.sql" + cat "$here/synthetic-day.sql" +} | sed 's/\r$//' > "$out" +echo "wrote $out ($(wc -l < "$out") lines)" diff --git a/sim/seed/synthetic-base.sql b/sim/seed/synthetic-base.sql new file mode 100644 index 000000000..618a5c8c6 --- /dev/null +++ b/sim/seed/synthetic-base.sql @@ -0,0 +1,157 @@ +-- The synthetic fixture's base: the tables trade-ngin's QT migrations (021 on) build on, in the +-- shape new_algo_data had after trade-ngin 001 and 016, plus AlgoLens' auth.users. +-- +-- Copied from algolens-api/tests/qt_fixture.py (BASE_SCHEMA) and +-- algolens-api/tests/test_screen_access_integration.py (AUTH_SCHEMA), without their test rows +-- and roles. build-synthetic.sh puts trade-ngin's migrations and AlgoLens' 003 after it. + +DROP SCHEMA IF EXISTS trading CASCADE; +DROP SCHEMA IF EXISTS metadata CASCADE; +DROP SCHEMA IF EXISTS futures_data CASCADE; +CREATE SCHEMA trading; +CREATE SCHEMA metadata; +CREATE SCHEMA futures_data; + +CREATE TABLE trading.positions ( + symbol VARCHAR NOT NULL, + quantity NUMERIC NOT NULL, + average_price NUMERIC NOT NULL, + daily_unrealized_pnl NUMERIC NOT NULL, + daily_realized_pnl NUMERIC NOT NULL, + last_update TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + strategy_id VARCHAR NOT NULL, + strategy_name VARCHAR NOT NULL, + date DATE NOT NULL, + portfolio_id VARCHAR NOT NULL, + portfolio_type TEXT NOT NULL DEFAULT 'system', + instrument_id TEXT, + CONSTRAINT positions_portfolio_type_check CHECK (portfolio_type IN ('system', 'qt')), + CONSTRAINT positions_pkey PRIMARY KEY + (portfolio_id, strategy_id, strategy_name, date, symbol, portfolio_type) +); + +CREATE TABLE trading.equity_curve ( + id SERIAL PRIMARY KEY, + strategy_id VARCHAR NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + equity DOUBLE PRECISION NOT NULL, + portfolio_id VARCHAR, + portfolio_type TEXT NOT NULL DEFAULT 'system', + CONSTRAINT equity_curve_portfolio_type_check CHECK (portfolio_type IN ('system', 'qt')), + CONSTRAINT trading_equity_curve_unique + UNIQUE (portfolio_id, strategy_id, "timestamp", portfolio_type) +); + +CREATE TABLE trading.executions ( + portfolio_id VARCHAR NOT NULL, + strategy_id VARCHAR NOT NULL, + strategy_name VARCHAR NOT NULL, + date DATE NOT NULL, + exec_id TEXT NOT NULL, + symbol TEXT, + side TEXT, + quantity NUMERIC, + price NUMERIC, + execution_time TIMESTAMPTZ, + commissions_fees NUMERIC, + CONSTRAINT executions_pkey PRIMARY KEY (portfolio_id, strategy_id, strategy_name, date, exec_id) +); + +CREATE TABLE trading.live_results ( + id SERIAL PRIMARY KEY, + strategy_id VARCHAR(100) NOT NULL, + portfolio_id VARCHAR(100) NOT NULL, + date DATE NOT NULL, + config JSONB, + current_portfolio_value NUMERIC, + total_annualized_return NUMERIC, + total_cumulative_return NUMERIC, + volatility NUMERIC, + daily_return NUMERIC, + gross_leverage NUMERIC, + net_leverage NUMERIC, + portfolio_leverage NUMERIC, + margin_posted NUMERIC, + equity_to_margin_ratio NUMERIC, + margin_cushion NUMERIC, + gross_notional NUMERIC, + total_unrealized_pnl NUMERIC, + total_realized_pnl NUMERIC, + total_transaction_costs NUMERIC, + cash_available NUMERIC, + risk_scale NUMERIC, + risk_detail JSONB, + -- Not in the AlgoLens fixture; the scheduler reads it (book_written_at), prod has it. + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT live_results_key UNIQUE (portfolio_id, strategy_id, date) +); + +CREATE TABLE trading.live_run_metadata ( + id SERIAL PRIMARY KEY, + date DATE NOT NULL, + strategy_id TEXT NOT NULL, + portfolio_id TEXT NOT NULL, + strategy_allocations JSONB, + portfolio_config JSONB, + strategy_configs JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT live_run_metadata_key UNIQUE (date, strategy_id, portfolio_id) +); + +-- AlgoLens migrations 001 + 002. +CREATE TABLE trading.strategy_registry ( + id TEXT PRIMARY KEY, + strategy_type TEXT NOT NULL, + portfolio_id TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + initial_equity NUMERIC NOT NULL DEFAULT 500000, + managers JSONB NOT NULL DEFAULT '["AlgoLens System"]'::jsonb, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + sort_order INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + lifecycle TEXT NOT NULL DEFAULT 'live' CHECK (lifecycle IN ('incubating', 'live', 'retired')), + incubation_started_at TIMESTAMPTZ, + mock_capital NUMERIC +); +INSERT INTO trading.strategy_registry (id, strategy_type, portfolio_id, name) +VALUES ('trendfollowing', 'LIVE_TREND_FOLLOWING', 'CONSERVATIVE_PORTFOLIO', 'Trend Following'), + ('equity_mr', 'LIVE_EQUITY_MEAN_REVERSION', 'EQUITY_MR_PORTFOLIO', 'Equity MR'); + +CREATE TABLE metadata.contract_metadata ( + "Databento Symbol" TEXT NOT NULL, + "IB Symbol" TEXT NOT NULL, + "Name" TEXT NOT NULL, + "Exchange" TEXT NOT NULL, + "Asset Type" TEXT NOT NULL, + "Sector" TEXT NOT NULL, + "Contract Size" TEXT NOT NULL, + "Fee Per Contract" TEXT NOT NULL DEFAULT '1.50' +); + +-- futures_data.ohlcv_1d has no key; one (symbol, time) may be stored twice. +CREATE TABLE futures_data.ohlcv_1d ( + time TIMESTAMPTZ NOT NULL, + symbol TEXT NOT NULL, + open DOUBLE PRECISION, + high DOUBLE PRECISION, + low DOUBLE PRECISION, + close DOUBLE PRECISION, + volume DOUBLE PRECISION +); + +DROP SCHEMA IF EXISTS investor CASCADE; +DROP SCHEMA IF EXISTS auth CASCADE; +CREATE SCHEMA auth; +CREATE TYPE auth.user_role AS ENUM ('general_member', 'team_lead', 'exec_board'); +CREATE TABLE auth.users ( + id SERIAL PRIMARY KEY, + email VARCHAR(120) NOT NULL UNIQUE, + password_hash VARCHAR(256) NOT NULL DEFAULT '', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + role auth.user_role NOT NULL, + first_name VARCHAR(120) NOT NULL DEFAULT '', + last_name VARCHAR(120) NOT NULL DEFAULT '' +); diff --git a/sim/seed/synthetic-day.sql b/sim/seed/synthetic-day.sql new file mode 100644 index 000000000..f8caa9b6b --- /dev/null +++ b/sim/seed/synthetic-day.sql @@ -0,0 +1,83 @@ +-- One QT day for the synthetic fixture: enough for the desk panel to render and for the +-- clock rules (09:30 send, 10:00 cutoff) to be exercised. Not market data: no engine run can +-- succeed on it (that needs the prod snapshot). +-- +-- 2026-10-14 previous day: three books, published by the 10:00 fallback and sent +-- 2026-10-15 the sim day: the model run's three books (qt_proposal and qt = system), +-- not published +-- +-- Symbols: ZT, ZN, GC, CL, 6E (continuous .v.0), bars for 2026-10-12..14. + +BEGIN; + +INSERT INTO metadata.contract_metadata + ("Databento Symbol", "IB Symbol", "Name", "Exchange", "Asset Type", "Sector", "Contract Size") +VALUES + ('ZT', 'ZT', '2-Year T-Note', 'CBOT', 'Futures', 'Rates', '200000'), + ('ZN', 'ZN', '10-Year T-Note', 'CBOT', 'Futures', 'Rates', '100000'), + ('GC', 'GC', 'Gold', 'COMEX', 'Futures', 'Metals', '100'), + ('CL', 'CL', 'Crude Oil', 'NYMEX', 'Futures', 'Energy', '1000'), + ('6E', 'EUR', 'Euro FX', 'CME', 'Futures', 'FX', '125000'); + +INSERT INTO futures_data.ohlcv_1d (time, symbol, open, high, low, close, volume) +SELECT d::timestamptz, s.symbol, s.px, s.px * 1.002, s.px * 0.998, s.px * (1 + 0.001 * i), 10000 +FROM (VALUES ('ZT.v.0', 103.5), ('ZN.v.0', 111.2), ('GC.v.0', 2650.0), + ('CL.v.0', 71.4), ('6E.v.0', 1.09)) AS s(symbol, px) +CROSS JOIN LATERAL ( + SELECT g::date AS d, row_number() OVER () AS i + FROM generate_series('2026-10-12'::date, '2026-10-14'::date, '1 day') g +) days; + +-- Three books a day for the desk portfolio, the system book for its model twin. +INSERT INTO trading.positions + (symbol, quantity, average_price, daily_unrealized_pnl, daily_realized_pnl, last_update, + strategy_id, strategy_name, date, portfolio_id, portfolio_type) +SELECT p.symbol, p.qty, p.px, 0, 0, (b.day + time '07:00') AT TIME ZONE 'America/New_York', + 'TREND_FOLLOWING', 'TREND_FOLLOWING', b.day, b.portfolio_id, b.book +FROM (VALUES ('ZT.v.0', 8, 103.6), ('ZN.v.0', -3, 111.3), ('GC.v.0', 2, 2652.0), + ('CL.v.0', 0, 71.5), ('6E.v.0', -4, 1.091)) AS p(symbol, qty, px) +CROSS JOIN (VALUES + (date '2026-10-14', 'QT_CONSERVATIVE_PORTFOLIO', 'system'), + (date '2026-10-14', 'QT_CONSERVATIVE_PORTFOLIO', 'qt_proposal'), + (date '2026-10-14', 'QT_CONSERVATIVE_PORTFOLIO', 'qt'), + (date '2026-10-15', 'QT_CONSERVATIVE_PORTFOLIO', 'system'), + (date '2026-10-15', 'QT_CONSERVATIVE_PORTFOLIO', 'qt_proposal'), + (date '2026-10-15', 'QT_CONSERVATIVE_PORTFOLIO', 'qt'), + (date '2026-10-14', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system'), + (date '2026-10-15', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system') +) AS b(day, portfolio_id, book); + +INSERT INTO trading.equity_curve (strategy_id, timestamp, equity, portfolio_id, portfolio_type) +SELECT 'TREND_FOLLOWING', (d + time '16:00') AT TIME ZONE 'America/New_York', + 500000 + 1500 * row_number() OVER (PARTITION BY pid, book ORDER BY d), pid, book +FROM generate_series('2026-10-12'::date, '2026-10-14'::date, '1 day') d +CROSS JOIN (VALUES ('QT_CONSERVATIVE_PORTFOLIO', 'system'), ('QT_CONSERVATIVE_PORTFOLIO', 'qt'), + ('QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system')) AS x(pid, book); + +INSERT INTO trading.live_results + (strategy_id, portfolio_id, date, current_portfolio_value, daily_return, gross_leverage, + net_leverage, portfolio_type, created_at) +SELECT 'TREND_FOLLOWING', pid, d, 504500, 0.003, 1.2, 0.4, book, + (d + time '07:00') AT TIME ZONE 'America/New_York' +FROM (VALUES (date '2026-10-14'), (date '2026-10-15')) AS days(d) +CROSS JOIN (VALUES ('QT_CONSERVATIVE_PORTFOLIO', 'system'), ('QT_CONSERVATIVE_PORTFOLIO', 'qt'), + ('QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system')) AS x(pid, book); + +-- 2026-10-14 published by the 10:00 fallback and sent; 2026-10-15 built at 07:00, unpublished. +INSERT INTO trading.live_run_metadata + (date, strategy_id, portfolio_id, created_at, published_by, published_at, publish_source, + sent_at) +VALUES + ('2026-10-14', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_PORTFOLIO', + '2026-10-14 07:00 America/New_York', 'system:fallback-10am', + '2026-10-14 10:00 America/New_York', 'fallback', '2026-10-14 10:00 America/New_York'), + ('2026-10-15', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_PORTFOLIO', + '2026-10-15 07:00 America/New_York', NULL, NULL, NULL, NULL), + ('2026-10-14', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', + '2026-10-14 07:00 America/New_York', 'system:model-only', + '2026-10-14 07:00 America/New_York', 'model-only', '2026-10-14 09:30 America/New_York'), + ('2026-10-15', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', + '2026-10-15 07:00 America/New_York', 'system:model-only', + '2026-10-15 07:00 America/New_York', 'model-only', NULL); + +COMMIT; diff --git a/sim/seed/synthetic.sql b/sim/seed/synthetic.sql new file mode 100644 index 000000000..2ec635d42 --- /dev/null +++ b/sim/seed/synthetic.sql @@ -0,0 +1,1311 @@ +-- GENERATED by sim/seed/build-synthetic.sh from trade-ngin:rpc-0b9dbf6e; do not edit. +-- Restored into an empty new_algo_data by `simctl load synthetic`. +\set ON_ERROR_STOP on + +-- The synthetic fixture's base: the tables trade-ngin's QT migrations (021 on) build on, in the +-- shape new_algo_data had after trade-ngin 001 and 016, plus AlgoLens' auth.users. +-- +-- Copied from algolens-api/tests/qt_fixture.py (BASE_SCHEMA) and +-- algolens-api/tests/test_screen_access_integration.py (AUTH_SCHEMA), without their test rows +-- and roles. build-synthetic.sh puts trade-ngin's migrations and AlgoLens' 003 after it. + +DROP SCHEMA IF EXISTS trading CASCADE; +DROP SCHEMA IF EXISTS metadata CASCADE; +DROP SCHEMA IF EXISTS futures_data CASCADE; +CREATE SCHEMA trading; +CREATE SCHEMA metadata; +CREATE SCHEMA futures_data; + +CREATE TABLE trading.positions ( + symbol VARCHAR NOT NULL, + quantity NUMERIC NOT NULL, + average_price NUMERIC NOT NULL, + daily_unrealized_pnl NUMERIC NOT NULL, + daily_realized_pnl NUMERIC NOT NULL, + last_update TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + strategy_id VARCHAR NOT NULL, + strategy_name VARCHAR NOT NULL, + date DATE NOT NULL, + portfolio_id VARCHAR NOT NULL, + portfolio_type TEXT NOT NULL DEFAULT 'system', + instrument_id TEXT, + CONSTRAINT positions_portfolio_type_check CHECK (portfolio_type IN ('system', 'qt')), + CONSTRAINT positions_pkey PRIMARY KEY + (portfolio_id, strategy_id, strategy_name, date, symbol, portfolio_type) +); + +CREATE TABLE trading.equity_curve ( + id SERIAL PRIMARY KEY, + strategy_id VARCHAR NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + equity DOUBLE PRECISION NOT NULL, + portfolio_id VARCHAR, + portfolio_type TEXT NOT NULL DEFAULT 'system', + CONSTRAINT equity_curve_portfolio_type_check CHECK (portfolio_type IN ('system', 'qt')), + CONSTRAINT trading_equity_curve_unique + UNIQUE (portfolio_id, strategy_id, "timestamp", portfolio_type) +); + +CREATE TABLE trading.executions ( + portfolio_id VARCHAR NOT NULL, + strategy_id VARCHAR NOT NULL, + strategy_name VARCHAR NOT NULL, + date DATE NOT NULL, + exec_id TEXT NOT NULL, + symbol TEXT, + side TEXT, + quantity NUMERIC, + price NUMERIC, + execution_time TIMESTAMPTZ, + commissions_fees NUMERIC, + CONSTRAINT executions_pkey PRIMARY KEY (portfolio_id, strategy_id, strategy_name, date, exec_id) +); + +CREATE TABLE trading.live_results ( + id SERIAL PRIMARY KEY, + strategy_id VARCHAR(100) NOT NULL, + portfolio_id VARCHAR(100) NOT NULL, + date DATE NOT NULL, + config JSONB, + current_portfolio_value NUMERIC, + total_annualized_return NUMERIC, + total_cumulative_return NUMERIC, + volatility NUMERIC, + daily_return NUMERIC, + gross_leverage NUMERIC, + net_leverage NUMERIC, + portfolio_leverage NUMERIC, + margin_posted NUMERIC, + equity_to_margin_ratio NUMERIC, + margin_cushion NUMERIC, + gross_notional NUMERIC, + total_unrealized_pnl NUMERIC, + total_realized_pnl NUMERIC, + total_transaction_costs NUMERIC, + cash_available NUMERIC, + risk_scale NUMERIC, + risk_detail JSONB, + -- Not in the AlgoLens fixture; the scheduler reads it (book_written_at), prod has it. + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT live_results_key UNIQUE (portfolio_id, strategy_id, date) +); + +CREATE TABLE trading.live_run_metadata ( + id SERIAL PRIMARY KEY, + date DATE NOT NULL, + strategy_id TEXT NOT NULL, + portfolio_id TEXT NOT NULL, + strategy_allocations JSONB, + portfolio_config JSONB, + strategy_configs JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT live_run_metadata_key UNIQUE (date, strategy_id, portfolio_id) +); + +-- AlgoLens migrations 001 + 002. +CREATE TABLE trading.strategy_registry ( + id TEXT PRIMARY KEY, + strategy_type TEXT NOT NULL, + portfolio_id TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + initial_equity NUMERIC NOT NULL DEFAULT 500000, + managers JSONB NOT NULL DEFAULT '["AlgoLens System"]'::jsonb, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + sort_order INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + lifecycle TEXT NOT NULL DEFAULT 'live' CHECK (lifecycle IN ('incubating', 'live', 'retired')), + incubation_started_at TIMESTAMPTZ, + mock_capital NUMERIC +); +INSERT INTO trading.strategy_registry (id, strategy_type, portfolio_id, name) +VALUES ('trendfollowing', 'LIVE_TREND_FOLLOWING', 'CONSERVATIVE_PORTFOLIO', 'Trend Following'), + ('equity_mr', 'LIVE_EQUITY_MEAN_REVERSION', 'EQUITY_MR_PORTFOLIO', 'Equity MR'); + +CREATE TABLE metadata.contract_metadata ( + "Databento Symbol" TEXT NOT NULL, + "IB Symbol" TEXT NOT NULL, + "Name" TEXT NOT NULL, + "Exchange" TEXT NOT NULL, + "Asset Type" TEXT NOT NULL, + "Sector" TEXT NOT NULL, + "Contract Size" TEXT NOT NULL, + "Fee Per Contract" TEXT NOT NULL DEFAULT '1.50' +); + +-- futures_data.ohlcv_1d has no key; one (symbol, time) may be stored twice. +CREATE TABLE futures_data.ohlcv_1d ( + time TIMESTAMPTZ NOT NULL, + symbol TEXT NOT NULL, + open DOUBLE PRECISION, + high DOUBLE PRECISION, + low DOUBLE PRECISION, + close DOUBLE PRECISION, + volume DOUBLE PRECISION +); + +DROP SCHEMA IF EXISTS investor CASCADE; +DROP SCHEMA IF EXISTS auth CASCADE; +CREATE SCHEMA auth; +CREATE TYPE auth.user_role AS ENUM ('general_member', 'team_lead', 'exec_board'); +CREATE TABLE auth.users ( + id SERIAL PRIMARY KEY, + email VARCHAR(120) NOT NULL UNIQUE, + password_hash VARCHAR(256) NOT NULL DEFAULT '', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + role auth.user_role NOT NULL, + first_name VARCHAR(120) NOT NULL DEFAULT '', + last_name VARCHAR(120) NOT NULL DEFAULT '' +); + +-- ===== trade-ngin migrations/014_contract_metadata_fee_per_contract.sql (trade-ngin:rpc-0b9dbf6e) +-- 014_contract_metadata_fee_per_contract.sql +-- +-- Add metadata.contract_metadata."Fee Per Contract": the dollars per contract per side the +-- transaction cost model charges for a futures fill, as a column of the contract's own metadata +-- row (T-7b-2 CM1). +-- +-- WHY +-- +-- The cost model priced every futures fill's explicit fee with a code default of 1.50 per +-- contract (TransactionCostManager::Config, BacktestExecutionConfig, ExecutionConfig), while the +-- contract size the strategy sizes with comes from this table. HD 2026-09-25 (rulings 11 and 14): +-- one source of truth for contract specs, nothing hard-coded, costs from the metadata. After this +-- column exists the instrument registry reads it into each futures contract and the cost model +-- charges it; the code default remains only for a database without the column. +-- +-- Every existing row gets the configured default fee per contract, 1.50, the reference until +-- per-row fees land (HD ruling 33, 2026-09-26), so every fill is priced exactly as before: the +-- column is number-neutral when applied. A micro or mini contract whose broker fee differs is +-- updated row by row by the migration that changes that row's contract, with its source. +-- +-- The column is text, like every other column of the table; the loader converts it to a double +-- by name (PostgresDatabase::convert_metadata_to_arrow), so the loader's positional indices of +-- the older columns do not move. +-- +-- SAFETY +-- +-- * ADD COLUMN with a constant DEFAULT is metadata-only on PostgreSQL 11 and later: no table +-- rewrite, every existing row reads the default. +-- * NOT NULL holds from the start (every row reads '1.50'); a later INSERT that omits the +-- column gets '1.50'. +-- * No key, index, constraint or view is touched; no other column changes. +-- * Type-guarded: an existing "Fee Per Contract" of another type makes this refuse. +-- * Transactional and idempotent (ADD COLUMN IF NOT EXISTS; COMMENT repeats). +-- +-- ORDER AND NUMBERING +-- +-- Independent of 001-013; 014 is the next free number after 013 (T-7b-2 8b). Like 012 and 013, +-- applied to the stage-3 session clones in-session and to production only at the end-of-stage +-- merge on HD's go. +-- +-- REACHABILITY +-- +-- Every futures runner and backtest loads this table through InstrumentRegistry::load_instruments +-- (SELECT *). With the column present each futures contract carries its fee, and all three cost +-- managers (the backtest fills, the live fills, the PortfolioManager's optimizer cost vector) +-- charge it. The equity cost configs charge per share and never read it. + +BEGIN; + +DO $$ +DECLARE + t text; +BEGIN + IF to_regclass('metadata.contract_metadata') IS NULL THEN + RAISE EXCEPTION 'metadata.contract_metadata does not exist; this migration adds a column ' + 'to it, it does not create the table'; + END IF; + SELECT data_type INTO t + FROM information_schema.columns + WHERE table_schema = 'metadata' AND table_name = 'contract_metadata' + AND column_name = 'Fee Per Contract'; + IF t IS NOT NULL AND t <> 'text' THEN + RAISE EXCEPTION 'metadata.contract_metadata."Fee Per Contract" already exists as %, not ' + 'text: refusing', t; + END IF; +END $$; + +ALTER TABLE metadata.contract_metadata + ADD COLUMN IF NOT EXISTS "Fee Per Contract" TEXT NOT NULL DEFAULT '1.50'; + +COMMENT ON COLUMN metadata.contract_metadata."Fee Per Contract" IS + 'Dollars per contract per side that the transaction cost model charges for a fill of this ' + 'contract (explicit fee: broker commission plus exchange and clearing, all-in). Every row ' + 'starts at the configured default fee per contract, 1.50, the reference until per-row fees ' + 'land (HD ruling 33, 2026-09-26). Read by the instrument registry by name; a database without ' + 'this column prices every fill at the code default 1.50 (migration 014).'; + +COMMIT; + +-- ===== trade-ngin migrations/017_live_results_roll_costs.sql (trade-ngin:rpc-0b9dbf6e) +-- 017_live_results_roll_costs.sql +-- +-- Add trading.live_results.daily_roll_costs (numeric NOT NULL DEFAULT 0) and total_roll_costs +-- (numeric(20,8) NOT NULL DEFAULT 0): the ROLL subset of the day's and the cumulative +-- transaction costs (LOOP_SPEC v6.1 sections 6.5 and 7; T-ROLLX commit 3). The pair is the +-- SUPERSET form: daily_transaction_costs / total_transaction_costs keep their meaning (every +-- fill's cost, STRATEGY + ROLL + BORROW) and the roll columns are the part of them paid on ROLL +-- legs; the strategy-only cost is the difference. No existing read moves. The types mirror +-- daily_transaction_costs (numeric) and total_transaction_costs (numeric(20,8)). +-- +-- SAFETY: ADD COLUMN with a constant DEFAULT is metadata-only; every existing row reads 0 (no +-- roll was booked before this column); no key, index or view is touched; type-guarded; +-- transactional and idempotent. + +BEGIN; +DO $$ +DECLARE t text; +BEGIN + IF to_regclass('trading.live_results') IS NULL THEN RAISE EXCEPTION 'trading.live_results does not exist'; END IF; + FOR t IN SELECT data_type FROM information_schema.columns WHERE table_schema = 'trading' AND table_name = 'live_results' + AND column_name IN ('daily_roll_costs', 'total_roll_costs') AND data_type <> 'numeric' LOOP + RAISE EXCEPTION 'a live_results roll cost column exists with type %, not numeric', t; + END LOOP; +END $$; +ALTER TABLE trading.live_results ADD COLUMN IF NOT EXISTS daily_roll_costs numeric NOT NULL DEFAULT 0; +ALTER TABLE trading.live_results ADD COLUMN IF NOT EXISTS total_roll_costs numeric(20,8) NOT NULL DEFAULT 0; +COMMENT ON COLUMN trading.live_results.daily_roll_costs IS + 'The part of daily_transaction_costs paid on ROLL legs (execution_type ROLL) on this date; an upper bound (rolls trade as spreads, modelled as two outright legs). daily_transaction_costs stays the superset. LOOP_SPEC v6.1 section 6.5; migration 017.'; +COMMENT ON COLUMN trading.live_results.total_roll_costs IS + 'The cumulative daily_roll_costs (the previous row''s total plus this date''s); the part of total_transaction_costs paid on ROLL legs. Migration 017.'; +COMMIT; + +-- ===== trade-ngin migrations/021_qt_books.sql (trade-ngin:rpc-0b9dbf6e) +-- 021_qt_books.sql +-- +-- Three books side by side in every live trading table (QT platform, PR E1; master document +-- rulings 5, 9, 12, 17 and "final trading schema changes"): +-- +-- system the model's answer, written by the model run. Every row stored to date. +-- qt_proposal what the desk asked for: seeded from system by the model run, edited in AlgoLens. +-- qt what the engine gave back: what goes out, and what tomorrow's run starts from. +-- +-- One column tells the books apart, `portfolio_type`, the column migration 001 put on +-- trading.positions and trading.equity_curve. This migration: +-- +-- 1. widens the portfolio_type CHECK on trading.positions and trading.equity_curve from +-- ('system','qt') to ('system','qt_proposal','qt'); +-- 2. adds portfolio_type TEXT NOT NULL DEFAULT 'system', with the same CHECK, to +-- trading.executions and trading.live_results, and puts it last in their keys: +-- executions_pkey (portfolio_id, strategy_id, strategy_name, +-- date, exec_id) + portfolio_type +-- the live_results UNIQUE key (portfolio_id, strategy_id, date) +-- + portfolio_type +-- The key constraints are found by their columns, not their names (the stage-3 scratch and +-- the test fixtures name the live_results key differently), and are re-created under the +-- name they had; +-- 3. adds trading.positions.moved_by TEXT NULL: the one-pass step that moved the symbol +-- between the desk's request (qt_proposal) and what the engine gave back (qt) (ruling 12). +-- A CHECK keeps it NULL on every book but qt. +-- +-- WHY THE KEYS. A qt row and a system row for the same portfolio, sleeve, date and exec id (or +-- the same portfolio, strategy and date in live_results) collide on the old keys, so the second +-- book's write fails or, worse, the first book's re-run DELETE takes the other's rows. With the +-- book last in the key the two coexist, and the engine's DELETEs are scoped by book (same PR). +-- Order ids are NOT changed: system ids stay byte-identical, and the book in the key is what +-- keeps two books' executions apart. +-- +-- SAFETY +-- +-- * ADD COLUMN with a constant DEFAULT is metadata-only on PostgreSQL 11+; every existing row +-- reads 'system', which is what every existing row is. No backfill. +-- * Adding a column to a key only makes it more permissive: no existing row can conflict. +-- The key rebuild re-indexes the table (executions and live_results are small: thousands of +-- rows), inside this transaction. +-- * NULL-able moved_by with no default is metadata-only; every existing row reads NULL. +-- * Guarded: refuses unless migration 001 is applied (positions/equity_curve.portfolio_type +-- exist); refuses an existing portfolio_type or moved_by of another type than text; refuses +-- when the executions primary key or the live_results unique key is not found in either its +-- pre-021 or its post-021 shape. +-- * Transactional and idempotent: a second apply changes nothing. +-- +-- DEPLOY ORDERING: ship with the binary of the same commit. That binary names portfolio_type in +-- every read, write and delete of these four tables, so it fails (refuses) against a database +-- without 021; the old binary keeps working against 021 (it writes the default 'system' and its +-- DELETE predicates only ever matched system rows, the only rows a database without a desk has). +-- +-- Rollback: migrations/021_qt_books_rollback.sql + +BEGIN; + +DO $$ +DECLARE + t text; + tbl text; +BEGIN + FOREACH tbl IN ARRAY ARRAY['positions', 'equity_curve', 'executions', 'live_results'] LOOP + IF to_regclass('trading.' || tbl) IS NULL THEN + RAISE EXCEPTION 'trading.% does not exist; migration 021 changes it', tbl; + END IF; + END LOOP; + FOREACH tbl IN ARRAY ARRAY['positions', 'equity_curve'] LOOP + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'trading' + AND table_name = tbl AND column_name = 'portfolio_type') THEN + RAISE EXCEPTION 'trading.%.portfolio_type does not exist: apply migration 001 first', tbl; + END IF; + END LOOP; + FOREACH tbl IN ARRAY ARRAY['executions', 'live_results'] LOOP + SELECT data_type INTO t FROM information_schema.columns + WHERE table_schema = 'trading' AND table_name = tbl AND column_name = 'portfolio_type'; + IF t IS NOT NULL AND t <> 'text' THEN + RAISE EXCEPTION 'trading.%.portfolio_type exists with type %, not text', tbl, t; + END IF; + END LOOP; + SELECT data_type INTO t FROM information_schema.columns + WHERE table_schema = 'trading' AND table_name = 'positions' AND column_name = 'moved_by'; + IF t IS NOT NULL AND t <> 'text' THEN + RAISE EXCEPTION 'trading.positions.moved_by exists with type %, not text', t; + END IF; +END $$; + +-- The key of `tbl` whose column set is exactly `cols` (contype 'p' or 'u'), or NULL. +CREATE OR REPLACE FUNCTION pg_temp.qt021_key(tbl text, cols text[]) RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT c.conname::text + FROM pg_constraint c + WHERE c.conrelid = ('trading.' || tbl)::regclass AND c.contype IN ('p', 'u') + AND (SELECT array_agg(a.attname::text ORDER BY a.attname) + FROM pg_attribute a + WHERE a.attrelid = c.conrelid AND a.attnum = ANY (c.conkey)) + = (SELECT array_agg(x ORDER BY x) FROM unnest(cols) x) + LIMIT 1 +$$; + +DO $$ +BEGIN + IF pg_temp.qt021_key('executions', ARRAY['portfolio_id','strategy_id','strategy_name','date','exec_id']) IS NULL + AND pg_temp.qt021_key('executions', ARRAY['portfolio_id','strategy_id','strategy_name','date','exec_id','portfolio_type']) IS NULL THEN + RAISE EXCEPTION 'trading.executions has no key on (portfolio_id, strategy_id, strategy_name, date, exec_id[, portfolio_type]); refusing'; + END IF; + IF pg_temp.qt021_key('live_results', ARRAY['portfolio_id','strategy_id','date']) IS NULL + AND pg_temp.qt021_key('live_results', ARRAY['portfolio_id','strategy_id','date','portfolio_type']) IS NULL THEN + RAISE EXCEPTION 'trading.live_results has no key on (portfolio_id, strategy_id, date[, portfolio_type]); refusing'; + END IF; +END $$; + +-- 1. positions and equity_curve: widen the CHECK. +ALTER TABLE trading.positions DROP CONSTRAINT IF EXISTS positions_portfolio_type_check; +ALTER TABLE trading.positions ADD CONSTRAINT positions_portfolio_type_check + CHECK (portfolio_type IN ('system', 'qt_proposal', 'qt')); +ALTER TABLE trading.equity_curve DROP CONSTRAINT IF EXISTS equity_curve_portfolio_type_check; +ALTER TABLE trading.equity_curve ADD CONSTRAINT equity_curve_portfolio_type_check + CHECK (portfolio_type IN ('system', 'qt_proposal', 'qt')); + +-- 2. executions and live_results: the column, its CHECK, and the key. +ALTER TABLE trading.executions ADD COLUMN IF NOT EXISTS portfolio_type TEXT NOT NULL DEFAULT 'system'; +ALTER TABLE trading.executions DROP CONSTRAINT IF EXISTS executions_portfolio_type_check; +ALTER TABLE trading.executions ADD CONSTRAINT executions_portfolio_type_check + CHECK (portfolio_type IN ('system', 'qt_proposal', 'qt')); +ALTER TABLE trading.live_results ADD COLUMN IF NOT EXISTS portfolio_type TEXT NOT NULL DEFAULT 'system'; +ALTER TABLE trading.live_results DROP CONSTRAINT IF EXISTS live_results_portfolio_type_check; +ALTER TABLE trading.live_results ADD CONSTRAINT live_results_portfolio_type_check + CHECK (portfolio_type IN ('system', 'qt_proposal', 'qt')); + +DO $$ +DECLARE k text; +BEGIN + k := pg_temp.qt021_key('executions', ARRAY['portfolio_id','strategy_id','strategy_name','date','exec_id']); + IF k IS NOT NULL THEN + EXECUTE format('ALTER TABLE trading.executions DROP CONSTRAINT %I', k); + EXECUTE format('ALTER TABLE trading.executions ADD CONSTRAINT %I PRIMARY KEY ' + '(portfolio_id, strategy_id, strategy_name, date, exec_id, portfolio_type)', k); + END IF; + k := pg_temp.qt021_key('live_results', ARRAY['portfolio_id','strategy_id','date']); + IF k IS NOT NULL THEN + EXECUTE format('ALTER TABLE trading.live_results DROP CONSTRAINT %I', k); + EXECUTE format('ALTER TABLE trading.live_results ADD CONSTRAINT %I UNIQUE ' + '(portfolio_id, strategy_id, date, portfolio_type)', k); + END IF; +END $$; + +-- 3. positions.moved_by. +ALTER TABLE trading.positions ADD COLUMN IF NOT EXISTS moved_by TEXT; +ALTER TABLE trading.positions DROP CONSTRAINT IF EXISTS positions_moved_by_qt_only; +ALTER TABLE trading.positions ADD CONSTRAINT positions_moved_by_qt_only + CHECK (moved_by IS NULL OR portfolio_type = 'qt'); + +COMMENT ON COLUMN trading.executions.portfolio_type IS + 'The book the fill belongs to: system (the model run), qt_proposal (the desk''s request) or qt (what the engine gave back and what goes out). Part of executions_pkey. Every row before migration 021 is system.'; +COMMENT ON COLUMN trading.live_results.portfolio_type IS + 'The book the row belongs to: system, qt_proposal or qt (as trading.positions.portfolio_type). Part of the (portfolio_id, strategy_id, date, portfolio_type) key. Every row before migration 021 is system.'; +COMMENT ON COLUMN trading.positions.moved_by IS + 'qt rows only: the one-pass step that moved the symbol from the desk''s request (the qt_proposal row) to what the engine gave back (this row); NULL when nothing moved it. NULL on every system and qt_proposal row (CHECK positions_moved_by_qt_only). Ruling 12; migration 021.'; + +COMMIT; + +-- ===== trade-ngin migrations/022_strategy_config.sql (trade-ngin:rpc-0b9dbf6e) +-- 022_strategy_config.sql +-- +-- Desk settings from the database (QT plan E2; rulings 2, 3, 4, 24, 26). +-- +-- 1. trading.strategy_config: the ONE new table of the settings lane. One row per desk change to +-- a portfolio's settings, versioned and append-only. `overrides` is a JSON object in the shape +-- of the portfolio's merged config files (defaults.json + portfolio.json, with risk.json under +-- "risk"), deep-merged on top of the files at the start of a LIVE run only (ruling 4: backtests +-- keep the file config). At most one row per portfolio is active (partial unique index); with +-- no active row the run uses its files unchanged. A change states its reason and its author. +-- The engine refuses a run whose active row cannot be applied (ruling 24) and never falls back +-- to the files; the checks it makes (no null, no credential/database/email key, no key the +-- files do not hold) are in ConfigLoader, not here. +-- +-- 2. trading.live_run_metadata gains: +-- settings_used jsonb the credential-free effective config the run traded on, as +-- {"strategy_config_version": , "config": {...}}; +-- written by every live run of a binary from this migration on; +-- published_by text, published_at timestamptz the desk publish record (QT plan E8, ruling +-- 18); added here so the table changes once. NULL until E8 writes them. +-- Rows written before this migration read NULL in all three. +-- +-- Ported from PR #60's migration 008 (strategy_config DDL only; its config_manifest table is NOT +-- created: ruling 26 puts the settings used on live_run_metadata). created_by is text here (#60 +-- had integer): AlgoLens records the author's login. +-- +-- SAFETY: CREATE TABLE / ADD COLUMN of NULL-able columns with no default are metadata-only; no +-- existing row, key, index or view of live_run_metadata is touched; type-guarded (an existing +-- strategy_config or column of another shape makes 022 refuse); transactional and idempotent. + +BEGIN; +DO $$ +DECLARE bad text; +BEGIN + IF to_regclass('trading.live_run_metadata') IS NULL THEN + RAISE EXCEPTION 'trading.live_run_metadata does not exist'; + END IF; + -- A strategy_config left by another branch (PR #60's 008 had created_by integer and no + -- object check) must not be taken for this one. + IF to_regclass('trading.strategy_config') IS NOT NULL THEN + FOR bad IN + SELECT e.col || ' expected ' || e.typ || ' got ' || coalesce(c.data_type, 'nothing') + FROM (VALUES ('id', 'bigint'), ('portfolio_id', 'text'), ('version', 'integer'), + ('overrides', 'jsonb'), ('reason', 'text'), ('created_by', 'text'), + ('created_at', 'timestamp with time zone'), ('is_active', 'boolean')) e(col, typ) + LEFT JOIN information_schema.columns c + ON c.table_schema = 'trading' AND c.table_name = 'strategy_config' AND c.column_name = e.col + WHERE c.data_type IS DISTINCT FROM e.typ + LOOP + RAISE EXCEPTION 'trading.strategy_config exists with another shape: %', bad; + END LOOP; + END IF; + FOR bad IN + SELECT column_name || ' ' || data_type FROM information_schema.columns + WHERE table_schema = 'trading' AND table_name = 'live_run_metadata' + AND ((column_name = 'settings_used' AND data_type <> 'jsonb') + OR (column_name = 'published_by' AND data_type <> 'text') + OR (column_name = 'published_at' AND data_type <> 'timestamp with time zone')) + LOOP + RAISE EXCEPTION 'trading.live_run_metadata has a column of another type: %', bad; + END LOOP; +END $$; + +CREATE TABLE IF NOT EXISTS trading.strategy_config ( + id bigserial PRIMARY KEY, + portfolio_id text NOT NULL, + version integer NOT NULL, + overrides jsonb NOT NULL, + reason text NOT NULL, + created_by text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + is_active boolean NOT NULL DEFAULT false, + CONSTRAINT strategy_config_portfolio_version_key UNIQUE (portfolio_id, version), + CONSTRAINT strategy_config_version_positive CHECK (version > 0), + CONSTRAINT strategy_config_overrides_object CHECK (jsonb_typeof(overrides) = 'object'), + CONSTRAINT strategy_config_reason_not_empty CHECK (length(btrim(reason)) > 0), + CONSTRAINT strategy_config_created_by_not_empty CHECK (length(btrim(created_by)) > 0) +); + +-- At most one active version per portfolio: two active rows would be worse than no versioning. +CREATE UNIQUE INDEX IF NOT EXISTS strategy_config_one_active_per_portfolio + ON trading.strategy_config (portfolio_id) WHERE is_active; + +COMMENT ON TABLE trading.strategy_config IS + 'Desk settings changes, one versioned row per change, at most one active per portfolio. overrides is deep-merged over the portfolio''s config files at the start of a live run only; a row that cannot be applied refuses the run (ruling 24). QT plan E2; migration 022.'; +COMMENT ON COLUMN trading.strategy_config.overrides IS + 'A JSON object in the shape of the merged config files (defaults.json + portfolio.json, risk.json under "risk"). Every key must exist in the files; no null; no database, email, portfolio_id or credential key. Arrays replace the file''s array whole.'; + +ALTER TABLE trading.live_run_metadata ADD COLUMN IF NOT EXISTS settings_used jsonb; +ALTER TABLE trading.live_run_metadata ADD COLUMN IF NOT EXISTS published_by text; +ALTER TABLE trading.live_run_metadata ADD COLUMN IF NOT EXISTS published_at timestamptz; + +COMMENT ON COLUMN trading.live_run_metadata.settings_used IS + 'The credential-free effective config the run traded on: {"strategy_config_version": n or null (files only), "config": the merged files plus the active strategy_config overrides, without the database and email sections}. NULL on rows written before migration 022''s binary. QT plan E2; migration 022.'; +COMMENT ON COLUMN trading.live_run_metadata.published_by IS + 'Who published the day''s desk book (QT plan E8). NULL until published. Migration 022.'; +COMMENT ON COLUMN trading.live_run_metadata.published_at IS + 'When the day''s desk book was published (QT plan E8). NULL until published. Migration 022.'; +COMMIT; + +-- ===== trade-ngin migrations/023_qt_command_log.sql (trade-ngin:rpc-0b9dbf6e) +-- 023_qt_command_log.sql +-- +-- The QT platform's command log and the remaining schema of docs/design/qt-contract.md (sections 2, +-- 4 and 5). Applied after 021 (books) and 022 (strategy_config). +-- +-- 1. trading.position_overrides is rebuilt as the desk's command log: one row per save, override +-- request, override decision and publish, keyed by portfolio and book date. AlgoLens inserts +-- rows; the engine (desk-agent) moves status/result. Rows are never deleted, and only the +-- engine-owned columns may change (trigger). The old #55 shape had no portfolio or date and an +-- UPDATE ... DO INSTEAD NOTHING rule that made recording a decision impossible; it holds 0 rows +-- on new_algo_data (checked 2026-10-09) and this migration refuses if any row exists. +-- 2. trading.live_results.book_source: which path produced a qt row (model, desk, override). +-- Master doc ruling 9: each day's record says which happened. +-- 3. trading.strategy_registry.portfolio_group and desk_editable (ruling 20 and section 2 of the +-- contract), and the two QT portfolios. +-- +-- The ruling-28 drops (risk_limits, portfolios, strategy_book_memberships, portfolio_assignments) +-- are in 024, applied separately. +-- +-- SAFETY: transactional; refuses on a non-empty position_overrides; otherwise additive. + +BEGIN; + +DO $$ +BEGIN + IF to_regclass('trading.live_results') IS NULL THEN RAISE EXCEPTION 'trading.live_results does not exist'; END IF; + IF to_regclass('trading.strategy_registry') IS NULL THEN RAISE EXCEPTION 'trading.strategy_registry does not exist'; END IF; + IF to_regclass('trading.position_overrides') IS NOT NULL THEN + IF EXISTS (SELECT 1 FROM trading.position_overrides) THEN + RAISE EXCEPTION 'trading.position_overrides holds rows; 023 rebuilds it and refuses to discard them'; + END IF; + END IF; +END $$; + +-- 1. The command log. +DROP TABLE IF EXISTS trading.position_overrides CASCADE; + +CREATE TABLE trading.position_overrides ( + id bigserial PRIMARY KEY, + portfolio_id text NOT NULL, + date date NOT NULL, + kind text NOT NULL CHECK (kind IN ('save', 'override_request', 'override_decision', 'publish')), + status text NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'running', 'done', 'refused', 'failed')), + requested_by text NOT NULL CHECK (length(btrim(requested_by)) > 0), + reason text, + payload jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(payload) = 'object'), + parent_id bigint REFERENCES trading.position_overrides (id), + approver_role text CHECK (approver_role IN ('vp', 'president')), + token_hash text, + token_expires_at timestamptz, + result jsonb, + message text, + created_at timestamptz NOT NULL DEFAULT now(), + started_at timestamptz, + finished_at timestamptz, + CONSTRAINT position_overrides_reason_required + CHECK (kind NOT IN ('save', 'override_request') OR length(btrim(coalesce(reason, ''))) > 0), + CONSTRAINT position_overrides_decision_shape + CHECK (kind <> 'override_decision' OR (parent_id IS NOT NULL AND approver_role IS NOT NULL)) +); + +CREATE INDEX position_overrides_portfolio_date_kind ON trading.position_overrides (portfolio_id, date, kind); +CREATE INDEX position_overrides_pending ON trading.position_overrides (status) WHERE status IN ('pending', 'running'); +CREATE UNIQUE INDEX position_overrides_one_publish + ON trading.position_overrides (portfolio_id, date) WHERE kind = 'publish' AND status = 'done'; +CREATE UNIQUE INDEX position_overrides_token ON trading.position_overrides (token_hash) WHERE token_hash IS NOT NULL; + +COMMENT ON TABLE trading.position_overrides IS + 'QT desk command log (docs/design/qt-contract.md section 4): one row per save, override request, override decision and publish. AlgoLens inserts; the engine moves status, started_at, finished_at, result and message. Never deleted. Migration 023.'; + +CREATE OR REPLACE FUNCTION trading.position_overrides_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'trading.position_overrides rows are never deleted'; + END IF; + -- Only the engine-owned columns (and token_hash being cleared once used) may change. + IF (NEW.id, NEW.portfolio_id, NEW.date, NEW.kind, NEW.requested_by, NEW.reason, NEW.payload, + NEW.parent_id, NEW.approver_role, NEW.created_at) + IS DISTINCT FROM + (OLD.id, OLD.portfolio_id, OLD.date, OLD.kind, OLD.requested_by, OLD.reason, OLD.payload, + OLD.parent_id, OLD.approver_role, OLD.created_at) THEN + RAISE EXCEPTION 'trading.position_overrides: only status, result, message, started_at, finished_at and the token may change'; + END IF; + RETURN NEW; +END $$; + +CREATE TRIGGER position_overrides_guard + BEFORE UPDATE OR DELETE ON trading.position_overrides + FOR EACH ROW EXECUTE FUNCTION trading.position_overrides_guard(); + +-- 2. Which path produced a book's day. +ALTER TABLE trading.live_results ADD COLUMN IF NOT EXISTS book_source text; +ALTER TABLE trading.live_results DROP CONSTRAINT IF EXISTS live_results_book_source_check; +ALTER TABLE trading.live_results ADD CONSTRAINT live_results_book_source_check + CHECK (book_source IS NULL OR book_source IN ('model', 'desk', 'override')); +COMMENT ON COLUMN trading.live_results.book_source IS + 'For a qt row: model (no desk save that day, qt = the model result), desk (the one pass ran on the desk proposal) or override (an approved override booked the proposal exactly). NULL on system rows. Migration 023.'; + +-- 3. Registry grouping and desk editing, and the two QT portfolios. +ALTER TABLE trading.strategy_registry ADD COLUMN IF NOT EXISTS portfolio_group text; +ALTER TABLE trading.strategy_registry ADD COLUMN IF NOT EXISTS desk_editable boolean NOT NULL DEFAULT false; + +INSERT INTO trading.strategy_registry + (id, strategy_type, portfolio_id, name, description, initial_equity, managers, is_active, sort_order, + lifecycle, portfolio_group, desk_editable) +VALUES + ('qt_conservative', 'LIVE_TREND_FOLLOWING', 'QT_CONSERVATIVE_PORTFOLIO', 'QT Trend Following', + 'Conservative trend following, desk-edited by the QT desk', 500000, '["QT desk"]'::jsonb, true, 10, + 'live', 'qt_conservative', true), + ('qt_conservative_model', 'LIVE_TREND_FOLLOWING', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', 'QT Trend Following (model)', + 'The same strategy and capital with desk editing off: the untouched model book', 500000, '["AlgoLens System"]'::jsonb, true, 11, + 'live', 'qt_conservative', false) +ON CONFLICT (id) DO NOTHING; + + +COMMIT; + +-- ===== trade-ngin migrations/024_drop_retired_qt_tables.sql (trade-ngin:rpc-0b9dbf6e) +-- 024_drop_retired_qt_tables.sql +-- +-- Ruling 28 (QT master document, 2026-10-08): drop the tables added to production outside merged +-- PRs that the design does not use: trading.risk_limits (#56's published limits; its only reader, +-- the AlgoLens pre-check, is gone), and the preview books registry trading.portfolios, +-- trading.strategy_book_memberships and trading.portfolio_assignments (strategy_registry carries +-- the grouping instead, migration 023). Nothing on main in trade-ngin or AlgoLens reads them, and no +-- foreign key points at them (checked on new_algo_data 2026-10-09: rows 0, 0, 4, 0). +-- +-- Irreversible: there is no rollback file. The pre-QT backup on the trade-ngin host, +-- /home/ubuntu/backups/new_algo_data_trading_metadata_backtest_pre_qt_20261009T0225Z.dump, +-- holds their definitions and data (pg_restore -t ). + +BEGIN; +DROP TABLE IF EXISTS trading.portfolio_assignments; +DROP TABLE IF EXISTS trading.strategy_book_memberships; +DROP TABLE IF EXISTS trading.portfolios; +DROP TABLE IF EXISTS trading.risk_limits; +COMMIT; + +-- ===== trade-ngin migrations/025_qt_command_log_hardening.sql (trade-ngin:rpc-0b9dbf6e) +-- 025_qt_command_log_hardening.sql +-- +-- QT hardening (2026-10-09; docs/design/qt-contract.md C2 and C4, and the grants review). Applied +-- after 023 (and 024). Transactional; refuses when existing rows would break the new indexes. +-- +-- 1. One decision per request, one open publish and one open override request per day (C2): +-- unique partial indexes on trading.position_overrides. +-- - override_decision: one per parent_id while pending, running or done (a refused or +-- failed decision does not count, so the request can still be decided). +-- - publish: one per (portfolio_id, date) while pending or running (023 already allows +-- one done publish per day). +-- - override_request: one per (portfolio_id, date) while pending or running. "Done, not +-- yet decided and not expired" depends on other rows and on now(), which no index can +-- express: AlgoLens enforces that part under its lock. +-- 2. The command row life cycle (C4), by trigger: +-- - an INSERT must be 'pending' with started_at, finished_at, result, message, token_hash +-- and token_expires_at NULL; +-- - status moves pending -> running, running -> done | refused | failed, and +-- running -> pending only inside a transaction that set algogators.recovery = 'on' +-- (the desk service's recovery of a stale or orphaned row, command_store.requeue); +-- - a terminal row (done, refused, failed) is final: any UPDATE of it is refused; +-- - TRUNCATE is refused (statement trigger), as DELETE already is (023). +-- The 023 guard function is replaced in place (same trigger), keeping its column rule. +-- 3. Grants. +-- - trading.live_run_metadata and its id sequence: PUBLIC had every privilege +-- (arwdDxt on the table, rU on the sequence). The roles that use the table keep +-- explicit grants (granted here first, idempotently, where the role exists); then +-- PUBLIC loses everything. Checked on new_algo_data 2026-10-09: fund_member, +-- quant_dev, quant_research_ro, leadership_ro and investor_relations_ro keep SELECT; +-- quant_dev_rw, quant_trading_rw (svc_trade_ngin, qt_engine_app), leadership_rw and +-- svc_algolens (qt_algolens_app) keep SELECT, INSERT, UPDATE, DELETE and the sequence. +-- svc_airflow and svc_data_ngin have no USAGE on schema trading, so PUBLIC gave them +-- nothing. postgres (owner, superuser) is unaffected. +-- - trading.position_overrides: svc_algolens loses UPDATE, DELETE and TRUNCATE; it keeps +-- SELECT, INSERT and USAGE on the id sequence (granted explicitly). AlgoLens never +-- updates a command row and, since AlgoLens #116, takes no FOR UPDATE lock on one. +-- svc_trade_ngin (the engine and the desk service) gets SELECT, INSERT, UPDATE and +-- DELETE explicitly (it already has them through quant_trading_rw; DELETE stays +-- refused by the 023 trigger). +-- Compatible with AlgoLens' emulation (algolens-api/tests/fixtures/sql/025_emulated.sql): +-- same index names and predicates, same insert rule, same transitions (the recovery move +-- additionally needs algogators.recovery = 'on'), same truncate block. +-- +-- Not done here, on purpose: SET NOT NULL on position_overrides or live_run_metadata columns +-- that hold NULLs today (the spec leaves them; see migrations/README.md). + +BEGIN; + +DO $$ +DECLARE + dup text; +BEGIN + IF to_regclass('trading.position_overrides') IS NULL THEN + RAISE EXCEPTION 'trading.position_overrides does not exist (apply 023 first)'; + END IF; + IF to_regprocedure('trading.position_overrides_guard()') IS NULL THEN + RAISE EXCEPTION 'trading.position_overrides_guard() does not exist (apply 023 first)'; + END IF; + IF to_regclass('trading.live_run_metadata') IS NULL THEN + RAISE EXCEPTION 'trading.live_run_metadata does not exist'; + END IF; + + SELECT string_agg(parent_id::text, ', ') INTO dup FROM ( + SELECT parent_id FROM trading.position_overrides + WHERE kind = 'override_decision' AND status IN ('pending', 'running', 'done') + GROUP BY parent_id HAVING count(*) > 1) d; + IF dup IS NOT NULL THEN + RAISE EXCEPTION 'override requests with more than one live or done decision: %; resolve them first', dup; + END IF; + SELECT string_agg(portfolio_id || ' ' || date, ', ') INTO dup FROM ( + SELECT portfolio_id, date FROM trading.position_overrides + WHERE kind = 'publish' AND status IN ('pending', 'running') + GROUP BY portfolio_id, date HAVING count(*) > 1) d; + IF dup IS NOT NULL THEN + RAISE EXCEPTION 'days with more than one open publish: %; resolve them first', dup; + END IF; + SELECT string_agg(portfolio_id || ' ' || date, ', ') INTO dup FROM ( + SELECT portfolio_id, date FROM trading.position_overrides + WHERE kind = 'override_request' AND status IN ('pending', 'running') + GROUP BY portfolio_id, date HAVING count(*) > 1) d; + IF dup IS NOT NULL THEN + RAISE EXCEPTION 'days with more than one open override request: %; resolve them first', dup; + END IF; +END $$; + +-- 1. C2 indexes. +CREATE UNIQUE INDEX IF NOT EXISTS position_overrides_one_decision + ON trading.position_overrides (parent_id) + WHERE kind = 'override_decision' AND status IN ('pending', 'running', 'done'); +CREATE UNIQUE INDEX IF NOT EXISTS position_overrides_one_open_publish + ON trading.position_overrides (portfolio_id, date) + WHERE kind = 'publish' AND status IN ('pending', 'running'); +CREATE UNIQUE INDEX IF NOT EXISTS position_overrides_one_open_request + ON trading.position_overrides (portfolio_id, date) + WHERE kind = 'override_request' AND status IN ('pending', 'running'); + +-- 2. C4 life cycle. +CREATE OR REPLACE FUNCTION trading.position_overrides_insert_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.status IS DISTINCT FROM 'pending' + OR NEW.started_at IS NOT NULL OR NEW.finished_at IS NOT NULL + OR NEW.result IS NOT NULL OR NEW.message IS NOT NULL + OR NEW.token_hash IS NOT NULL OR NEW.token_expires_at IS NOT NULL THEN + RAISE EXCEPTION 'trading.position_overrides: a new row must be pending, with started_at, finished_at, result, message and the token NULL'; + END IF; + RETURN NEW; +END $$; + +DROP TRIGGER IF EXISTS position_overrides_insert_guard ON trading.position_overrides; +CREATE TRIGGER position_overrides_insert_guard + BEFORE INSERT ON trading.position_overrides + FOR EACH ROW EXECUTE FUNCTION trading.position_overrides_insert_guard(); + +CREATE OR REPLACE FUNCTION trading.position_overrides_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'trading.position_overrides rows are never deleted'; + END IF; + -- 023: only the engine-owned columns (and token_hash being cleared once used) may change. + IF (NEW.id, NEW.portfolio_id, NEW.date, NEW.kind, NEW.requested_by, NEW.reason, NEW.payload, + NEW.parent_id, NEW.approver_role, NEW.created_at) + IS DISTINCT FROM + (OLD.id, OLD.portfolio_id, OLD.date, OLD.kind, OLD.requested_by, OLD.reason, OLD.payload, + OLD.parent_id, OLD.approver_role, OLD.created_at) THEN + RAISE EXCEPTION 'trading.position_overrides: only status, result, message, started_at, finished_at and the token may change'; + END IF; + -- 025 (C4): terminal rows are final; status moves only forward, or back to pending by recovery. + IF OLD.status IN ('done', 'refused', 'failed') THEN + RAISE EXCEPTION 'trading.position_overrides row % is % and final; insert a new row to retry', OLD.id, OLD.status; + END IF; + IF NEW.status IS DISTINCT FROM OLD.status THEN + IF OLD.status = 'pending' AND NEW.status = 'running' THEN + NULL; + ELSIF OLD.status = 'running' AND NEW.status IN ('done', 'refused', 'failed') THEN + NULL; + ELSIF OLD.status = 'running' AND NEW.status = 'pending' + AND current_setting('algogators.recovery', true) = 'on' THEN + NULL; + ELSE + RAISE EXCEPTION 'trading.position_overrides row %: % -> % is not an allowed status change', OLD.id, OLD.status, NEW.status; + END IF; + END IF; + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION trading.position_overrides_no_truncate() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'trading.position_overrides is never truncated'; +END $$; + +DROP TRIGGER IF EXISTS position_overrides_no_truncate ON trading.position_overrides; +CREATE TRIGGER position_overrides_no_truncate + BEFORE TRUNCATE ON trading.position_overrides + FOR EACH STATEMENT EXECUTE FUNCTION trading.position_overrides_no_truncate(); + +-- 3. Grants. Explicit grants first, for every role that exists, then PUBLIC goes. +DO $$ +DECLARE + r text; +BEGIN + FOREACH r IN ARRAY ARRAY['fund_member', 'quant_dev', 'quant_research_ro', 'leadership_ro', + 'investor_relations_ro'] LOOP + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN + EXECUTE format('GRANT SELECT ON trading.live_run_metadata TO %I', r); + END IF; + END LOOP; + FOREACH r IN ARRAY ARRAY['quant_dev_rw', 'quant_trading_rw', 'leadership_rw', 'svc_algolens', + 'svc_trade_ngin'] LOOP + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON trading.live_run_metadata TO %I', r); + EXECUTE format('GRANT USAGE, SELECT ON SEQUENCE trading.live_run_metadata_id_seq TO %I', r); + END IF; + END LOOP; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'svc_algolens') THEN + GRANT SELECT, INSERT ON trading.position_overrides TO svc_algolens; + GRANT USAGE, SELECT ON SEQUENCE trading.position_overrides_id_seq TO svc_algolens; + REVOKE UPDATE, DELETE, TRUNCATE ON trading.position_overrides FROM svc_algolens; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'svc_trade_ngin') THEN + GRANT SELECT, INSERT, UPDATE, DELETE ON trading.position_overrides TO svc_trade_ngin; + GRANT USAGE, SELECT ON SEQUENCE trading.position_overrides_id_seq TO svc_trade_ngin; + END IF; +END $$; + +REVOKE ALL ON trading.live_run_metadata FROM PUBLIC; +REVOKE ALL ON SEQUENCE trading.live_run_metadata_id_seq FROM PUBLIC; + +COMMENT ON TABLE trading.position_overrides IS + 'QT desk command log (docs/design/qt-contract.md section 4): one row per save, override request, override decision and publish. AlgoLens inserts pending rows; the engine moves status, started_at, finished_at, result and message (pending -> running -> done/refused/failed; running -> pending only by the agent''s recovery). Terminal rows are final. Never deleted or truncated. Migrations 023, 025.'; + +COMMIT; + +-- ===== trade-ngin migrations/027_qt_daily_cutoff.sql (trade-ngin:rpc-0b9dbf6e) +-- 027_qt_daily_cutoff.sql +-- +-- The QT daily cutoff (decided 2026-10-09; docs/design/qt-contract.md C7, amending rulings 18 +-- and 29). Applied after 022 and 025. Additive and transactional; re-applying it is a no-op. +-- +-- 1. trading.live_run_metadata gains the day's approval and send state: +-- - publish_source text NULL, CHECK (publish_source IN ('desk', 'fallback', 'model-only')): +-- who published the day. 'desk' = the desk's approval (the Publish command, "Approve" +-- in AlgoLens); 'fallback' = the 10:00 fallback or the catch-up of a missed past day +-- (qt reset to the model's book); 'model-only' = the untouched model twin. +-- - sent_at timestamptz NULL: when the day's e-mail went out (set once, by the engine). +-- published_by and published_at (022) are kept. Rows published before 027 keep NULL in +-- both columns: their source is not known, and an e-mail they sent is still found in the +-- publish row's result.email_sent_at (the engine checks both and never sends twice). +-- 2. Grants, as 025 set them: svc_trade_ngin (the engine and the desk service) reads and writes +-- the two columns (it already holds SELECT, INSERT, UPDATE, DELETE on the table; granted +-- again here, idempotently). svc_algolens (qt_algolens_app) only reads them: its table-wide +-- INSERT and UPDATE become column grants on every OTHER column, so everything it could +-- write before it still can, and it cannot write publish_source or sent_at. Other roles are +-- unchanged (the table-wide grants 025 made cover the new columns: readers read, the *_rw +-- roles write). + +BEGIN; + +DO $$ +DECLARE + bad text; +BEGIN + IF to_regclass('trading.live_run_metadata') IS NULL THEN + RAISE EXCEPTION 'trading.live_run_metadata does not exist'; + END IF; + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'trading' + AND table_name = 'live_run_metadata' AND column_name = 'published_at') THEN + RAISE EXCEPTION 'trading.live_run_metadata has no published_at (apply 022 first)'; + END IF; + SELECT string_agg(column_name || ' ' || data_type, ', ') INTO bad + FROM information_schema.columns + WHERE table_schema = 'trading' AND table_name = 'live_run_metadata' + AND ((column_name = 'publish_source' AND data_type <> 'text') + OR (column_name = 'sent_at' AND data_type <> 'timestamp with time zone')); + IF bad IS NOT NULL THEN + RAISE EXCEPTION 'trading.live_run_metadata has a column of another type: %', bad; + END IF; +END $$; + +-- 1. The columns. +ALTER TABLE trading.live_run_metadata ADD COLUMN IF NOT EXISTS publish_source text; +ALTER TABLE trading.live_run_metadata ADD COLUMN IF NOT EXISTS sent_at timestamptz; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'live_run_metadata_publish_source_check' + AND conrelid = 'trading.live_run_metadata'::regclass) THEN + ALTER TABLE trading.live_run_metadata ADD CONSTRAINT live_run_metadata_publish_source_check + CHECK (publish_source IN ('desk', 'fallback', 'model-only')); + END IF; +END $$; + +COMMENT ON COLUMN trading.live_run_metadata.publish_source IS + 'QT contract C7: who published the day: desk (the desk''s approval), fallback (the 10:00 fallback, or the catch-up of a missed past day: the model''s book), model-only (the model twin). NULL: unpublished, or published before migration 027.'; +COMMENT ON COLUMN trading.live_run_metadata.sent_at IS + 'QT contract C7: when the day''s daily e-mail and CSV went out (set once by the engine). NULL: not sent (not yet, a past day caught up, or e-mail disabled).'; + +-- 2. Grants. +DO $$ +DECLARE + others text; + priv text; +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'svc_trade_ngin') THEN + GRANT SELECT, INSERT, UPDATE ON trading.live_run_metadata TO svc_trade_ngin; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'svc_algolens') THEN + SELECT string_agg(quote_ident(column_name), ', ' ORDER BY ordinal_position) INTO others + FROM information_schema.columns + WHERE table_schema = 'trading' AND table_name = 'live_run_metadata' + AND column_name NOT IN ('publish_source', 'sent_at'); + GRANT SELECT ON trading.live_run_metadata TO svc_algolens; + FOREACH priv IN ARRAY ARRAY['INSERT', 'UPDATE'] LOOP + -- Only a table-wide grant held directly is narrowed (re-applying finds none). A + -- table-wide REVOKE also drops the role's column grants, so it goes first. + IF EXISTS (SELECT 1 FROM information_schema.role_table_grants + WHERE grantee = 'svc_algolens' AND table_schema = 'trading' + AND table_name = 'live_run_metadata' AND privilege_type = priv) THEN + EXECUTE format('REVOKE %s ON trading.live_run_metadata FROM svc_algolens', priv); + EXECUTE format('GRANT %s (%s) ON trading.live_run_metadata TO svc_algolens', + priv, others); + END IF; + END LOOP; + END IF; +END $$; + +COMMIT; + +-- ===== trade-ngin migrations/028_ops_data_ready.sql (trade-ngin:rpc-0b9dbf6e) +-- 028_ops_data_ready.sql +-- +-- Data readiness for the QT data-driven run (interim design of +-- https://github.com/AlgoGators/trade-ngin/issues/191; #192 later carries the same rows over +-- NATS). Additive, transactional and idempotent: re-applying it is a no-op. +-- +-- 1. Schema ops and table ops.data_ready: one row per (dataset, bar_date), written by data-ngin +-- when its pull of that dataset for that bar date finishes (04:30 ET: the vendor's +-- historical data is ~8 h delayed, so bar D is pulled early on D+1). An upsert: a re-pull +-- rewrites the row and moves finished_at, which makes the engine-rpc scheduler rebuild the +-- next trading day's books once more. status: complete (every expected symbol loaded), partial +-- (missing names the symbols, as futures_data.ohlcv_1d.symbol spells them), failed. +-- The scheduler reads dataset 'futures_ohlcv_1d'. +-- 2. Grants: +-- - qt_engine_app and svc_trade_ngin (the engine and its desk service): USAGE on ops, +-- SELECT on ops.data_ready. They never write it. +-- - postgres (data-ngin connects as postgres today): SELECT, INSERT, UPDATE. A superuser +-- needs none of this; it is written down so the grant moves with data-ngin when it gets +-- its own role (svc_data_ngin: grant the same three then). +-- Roles that do not exist on the server are skipped. +-- +-- Apply (as postgres, on new_algo_data): +-- psql -X -v ON_ERROR_STOP=1 -d new_algo_data -f migrations/028_ops_data_ready.sql + +BEGIN; + +DO $$ +DECLARE + bad text; +BEGIN + IF to_regclass('ops.data_ready') IS NULL THEN + RETURN; + END IF; + -- Re-applied over a table of another shape: refuse instead of silently keeping it. + SELECT string_agg(c, ', ') INTO bad + FROM unnest(ARRAY['dataset', 'bar_date', 'expected', 'loaded', 'missing', 'status', 'run_id', + 'finished_at']) AS c + WHERE NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_schema = 'ops' AND table_name = 'data_ready' + AND column_name = c); + IF bad IS NOT NULL THEN + RAISE EXCEPTION 'ops.data_ready exists without the column(s): %', bad; + END IF; +END $$; + +CREATE SCHEMA IF NOT EXISTS ops; + +CREATE TABLE IF NOT EXISTS ops.data_ready ( + dataset text NOT NULL, + bar_date date NOT NULL, + expected int NOT NULL, + loaded int NOT NULL, + missing text[] NOT NULL DEFAULT '{}', + status text NOT NULL CHECK (status IN ('complete', 'partial', 'failed')), + run_id text, + finished_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (dataset, bar_date) +); + +COMMENT ON SCHEMA ops IS + 'Operational state shared between the AlgoGators services (migration 028).'; +COMMENT ON TABLE ops.data_ready IS + 'Migration 028 (issue #191): data-ngin''s pull of a dataset for a bar date is done. Upserted by data-ngin; read by the engine-rpc scheduler, which builds the next trading day''s QT books once per finished_at.'; +COMMENT ON COLUMN ops.data_ready.missing IS + 'Symbols expected but not loaded (futures_data.ohlcv_1d.symbol spelling); empty when complete.'; +COMMENT ON COLUMN ops.data_ready.finished_at IS + 'When the pull finished. A re-pull moves it, and the scheduler rebuilds the unpublished target day.'; + +DO $$ +DECLARE + r text; +BEGIN + FOREACH r IN ARRAY ARRAY['qt_engine_app', 'svc_trade_ngin'] LOOP + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN + EXECUTE format('GRANT USAGE ON SCHEMA ops TO %I', r); + EXECUTE format('GRANT SELECT ON ops.data_ready TO %I', r); + END IF; + END LOOP; + -- data-ngin (connects as postgres today). + GRANT USAGE ON SCHEMA ops TO postgres; + GRANT SELECT, INSERT, UPDATE ON ops.data_ready TO postgres; +END $$; + +COMMIT; + +-- ===== trade-ngin migrations/029_qt_rescind.sql (trade-ngin:rpc-0b9dbf6e) +-- 029_qt_rescind.sql +-- +-- Rescind (decided by Dom 2026-10-13; docs/design/qt-contract.md C8): the desk takes back its +-- own approval before the 09:30 send. Applied after 023, 025 and 027. Transactional; re-applying +-- it is a no-op. +-- +-- 1. trading.position_overrides.kind gains 'rescind': the command AlgoLens writes for "Rescind +-- approval" (desk.proto 1.2.0 Rescind). The engine clears published_by, published_at and +-- publish_source on the day's live_run_metadata rows and records the approval it took back +-- in the row's result. +-- 2. position_overrides_one_publish (023: at most one done publish row per portfolio and day) +-- is dropped. A rescinded approval leaves its publish row done (C4: terminal rows are +-- final), so the day's next approval is a second done publish row. live_run_metadata +-- published_at, checked under the day lock by AlgoLens and by the engine, is what keeps a +-- day from being approved twice (C3), as it already was. The open-publish index +-- (position_overrides_one_open_publish, 025) stays. +-- 3. position_overrides_one_open_rescind: at most one pending or running rescind per +-- portfolio and day, as for publish and override requests (C2). +-- +-- No grant changes: svc_algolens already inserts command rows and svc_trade_ngin already writes +-- live_run_metadata. + +BEGIN; + +DO $$ +BEGIN + IF to_regclass('trading.position_overrides') IS NULL THEN + RAISE EXCEPTION 'trading.position_overrides does not exist (apply 023 first)'; + END IF; + IF to_regclass('trading.position_overrides_one_open_publish') IS NULL THEN + RAISE EXCEPTION 'position_overrides_one_open_publish does not exist (apply 025 first)'; + END IF; + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'trading' + AND table_name = 'live_run_metadata' AND column_name = 'publish_source') THEN + RAISE EXCEPTION 'trading.live_run_metadata has no publish_source (apply 027 first)'; + END IF; +END $$; + +-- 1. The kind. +DO $$ +DECLARE + c record; +BEGIN + -- 023 declared the CHECK inline, so its name is the server's choice: drop every CHECK on + -- kind alone, then add the named one. + FOR c IN + SELECT con.conname FROM pg_constraint con + WHERE con.conrelid = 'trading.position_overrides'::regclass AND con.contype = 'c' + AND con.conname <> 'position_overrides_kind' + AND pg_get_constraintdef(con.oid) ~ '^CHECK \(\(kind = ANY' + LOOP + EXECUTE format('ALTER TABLE trading.position_overrides DROP CONSTRAINT %I', c.conname); + END LOOP; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'position_overrides_kind' + AND conrelid = 'trading.position_overrides'::regclass) THEN + ALTER TABLE trading.position_overrides ADD CONSTRAINT position_overrides_kind + CHECK (kind IN ('save', 'override_request', 'override_decision', 'publish', 'rescind')); + END IF; +END $$; + +-- 2. A rescinded day may be approved again. +DROP INDEX IF EXISTS trading.position_overrides_one_publish; + +-- 3. One open rescind per day. +CREATE UNIQUE INDEX IF NOT EXISTS position_overrides_one_open_rescind + ON trading.position_overrides (portfolio_id, date) + WHERE kind = 'rescind' AND status IN ('pending', 'running'); + +COMMENT ON TABLE trading.position_overrides IS + 'QT desk command log (docs/design/qt-contract.md section 4): one row per save, override request, override decision, publish and rescind. AlgoLens inserts; the engine moves status, started_at, finished_at, result and message. Never deleted. Migrations 023, 025, 029.'; + +COMMIT; + +-- ===== algolens-api/migrations/003_screen_access.sql +-- Screen access and investor reports (Dom, 2026-10-11). +-- +-- AlgoLens has three screens: member, investor and qt. An admin (role +-- exec_board) assigns each user any of them; the roles themselves are +-- unchanged. The backend reads auth.user_screen_access on every request, so a +-- grant or revoke takes effect at once. Every change is logged, append-only, +-- in auth.screen_access_log. +-- +-- investor.reports holds the PDF reports and letters an admin uploads for the +-- investor screen. Reports are never deleted, only unpublished. +-- +-- Backfill (keeps today's access): every general_member, team_lead and +-- exec_board user gets member; exec_board also gets qt (the desk). New +-- sign-ups get nothing until an admin grants a screen. +-- +-- Grants go to svc_algolens (qt_algolens_app inherits it), when that role +-- exists. Transactional; safe to run more than once. Rollback: +-- 003_screen_access_rollback.sql. + +BEGIN; + +CREATE TABLE IF NOT EXISTS auth.user_screen_access ( + user_id INTEGER NOT NULL REFERENCES auth.users(id) ON DELETE CASCADE, + screen TEXT NOT NULL CHECK (screen IN ('member', 'investor', 'qt')), + granted_by TEXT NOT NULL, + granted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, screen) +); + +-- No foreign key: the log outlives a deleted user. +CREATE TABLE IF NOT EXISTS auth.screen_access_log ( + id BIGSERIAL PRIMARY KEY, + user_id INTEGER NOT NULL, + screen TEXT NOT NULL CHECK (screen IN ('member', 'investor', 'qt')), + action TEXT NOT NULL CHECK (action IN ('grant', 'revoke')), + by_email TEXT NOT NULL, + at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_screen_access_log_user + ON auth.screen_access_log (user_id, at DESC); + +CREATE SCHEMA IF NOT EXISTS investor; + +CREATE TABLE IF NOT EXISTS investor.reports ( + id BIGSERIAL PRIMARY KEY, + title TEXT NOT NULL CHECK (length(btrim(title)) BETWEEN 1 AND 200), + kind TEXT NOT NULL CHECK (kind IN ('monthly', 'quarterly', 'letter', 'other')), + period_label TEXT NULL CHECK (period_label IS NULL OR length(period_label) <= 40), + filename TEXT NOT NULL, + content_type TEXT NOT NULL CHECK (content_type = 'application/pdf'), + size_bytes INTEGER NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760), + content BYTEA NOT NULL CHECK (octet_length(content) = size_bytes), + sha256 TEXT NOT NULL, + uploaded_by TEXT NOT NULL, + uploaded_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published BOOLEAN NOT NULL DEFAULT TRUE +); +CREATE INDEX IF NOT EXISTS idx_investor_reports_uploaded + ON investor.reports (uploaded_at DESC); + +-- Backfill. role is the auth.user_role enum; compare as text. +INSERT INTO auth.user_screen_access (user_id, screen, granted_by) +SELECT id, 'member', 'migration:003' FROM auth.users +WHERE role::text IN ('general_member', 'team_lead', 'exec_board') +ON CONFLICT DO NOTHING; + +INSERT INTO auth.user_screen_access (user_id, screen, granted_by) +SELECT id, 'qt', 'migration:003' FROM auth.users +WHERE role::text = 'exec_board' +ON CONFLICT DO NOTHING; + +INSERT INTO auth.screen_access_log (user_id, screen, action, by_email) +SELECT a.user_id, a.screen, 'grant', 'migration:003' +FROM auth.user_screen_access a +WHERE a.granted_by = 'migration:003' + AND NOT EXISTS ( + SELECT 1 FROM auth.screen_access_log l + WHERE l.user_id = a.user_id AND l.screen = a.screen AND l.by_email = 'migration:003' + ); + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'svc_algolens') THEN + GRANT USAGE ON SCHEMA investor TO svc_algolens; + GRANT SELECT, INSERT, DELETE ON auth.user_screen_access TO svc_algolens; + GRANT SELECT, INSERT ON auth.screen_access_log TO svc_algolens; + GRANT USAGE ON SEQUENCE auth.screen_access_log_id_seq TO svc_algolens; + GRANT SELECT, INSERT, UPDATE ON investor.reports TO svc_algolens; + GRANT USAGE ON SEQUENCE investor.reports_id_seq TO svc_algolens; + END IF; +END $$; + +COMMIT; + +-- ===== sim/seed/synthetic-day.sql +-- One QT day for the synthetic fixture: enough for the desk panel to render and for the +-- clock rules (09:30 send, 10:00 cutoff) to be exercised. Not market data: no engine run can +-- succeed on it (that needs the prod snapshot). +-- +-- 2026-10-14 previous day: three books, published by the 10:00 fallback and sent +-- 2026-10-15 the sim day: the model run's three books (qt_proposal and qt = system), +-- not published +-- +-- Symbols: ZT, ZN, GC, CL, 6E (continuous .v.0), bars for 2026-10-12..14. + +BEGIN; + +INSERT INTO metadata.contract_metadata + ("Databento Symbol", "IB Symbol", "Name", "Exchange", "Asset Type", "Sector", "Contract Size") +VALUES + ('ZT', 'ZT', '2-Year T-Note', 'CBOT', 'Futures', 'Rates', '200000'), + ('ZN', 'ZN', '10-Year T-Note', 'CBOT', 'Futures', 'Rates', '100000'), + ('GC', 'GC', 'Gold', 'COMEX', 'Futures', 'Metals', '100'), + ('CL', 'CL', 'Crude Oil', 'NYMEX', 'Futures', 'Energy', '1000'), + ('6E', 'EUR', 'Euro FX', 'CME', 'Futures', 'FX', '125000'); + +INSERT INTO futures_data.ohlcv_1d (time, symbol, open, high, low, close, volume) +SELECT d::timestamptz, s.symbol, s.px, s.px * 1.002, s.px * 0.998, s.px * (1 + 0.001 * i), 10000 +FROM (VALUES ('ZT.v.0', 103.5), ('ZN.v.0', 111.2), ('GC.v.0', 2650.0), + ('CL.v.0', 71.4), ('6E.v.0', 1.09)) AS s(symbol, px) +CROSS JOIN LATERAL ( + SELECT g::date AS d, row_number() OVER () AS i + FROM generate_series('2026-10-12'::date, '2026-10-14'::date, '1 day') g +) days; + +-- Three books a day for the desk portfolio, the system book for its model twin. +INSERT INTO trading.positions + (symbol, quantity, average_price, daily_unrealized_pnl, daily_realized_pnl, last_update, + strategy_id, strategy_name, date, portfolio_id, portfolio_type) +SELECT p.symbol, p.qty, p.px, 0, 0, (b.day + time '07:00') AT TIME ZONE 'America/New_York', + 'TREND_FOLLOWING', 'TREND_FOLLOWING', b.day, b.portfolio_id, b.book +FROM (VALUES ('ZT.v.0', 8, 103.6), ('ZN.v.0', -3, 111.3), ('GC.v.0', 2, 2652.0), + ('CL.v.0', 0, 71.5), ('6E.v.0', -4, 1.091)) AS p(symbol, qty, px) +CROSS JOIN (VALUES + (date '2026-10-14', 'QT_CONSERVATIVE_PORTFOLIO', 'system'), + (date '2026-10-14', 'QT_CONSERVATIVE_PORTFOLIO', 'qt_proposal'), + (date '2026-10-14', 'QT_CONSERVATIVE_PORTFOLIO', 'qt'), + (date '2026-10-15', 'QT_CONSERVATIVE_PORTFOLIO', 'system'), + (date '2026-10-15', 'QT_CONSERVATIVE_PORTFOLIO', 'qt_proposal'), + (date '2026-10-15', 'QT_CONSERVATIVE_PORTFOLIO', 'qt'), + (date '2026-10-14', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system'), + (date '2026-10-15', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system') +) AS b(day, portfolio_id, book); + +INSERT INTO trading.equity_curve (strategy_id, timestamp, equity, portfolio_id, portfolio_type) +SELECT 'TREND_FOLLOWING', (d + time '16:00') AT TIME ZONE 'America/New_York', + 500000 + 1500 * row_number() OVER (PARTITION BY pid, book ORDER BY d), pid, book +FROM generate_series('2026-10-12'::date, '2026-10-14'::date, '1 day') d +CROSS JOIN (VALUES ('QT_CONSERVATIVE_PORTFOLIO', 'system'), ('QT_CONSERVATIVE_PORTFOLIO', 'qt'), + ('QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system')) AS x(pid, book); + +INSERT INTO trading.live_results + (strategy_id, portfolio_id, date, current_portfolio_value, daily_return, gross_leverage, + net_leverage, portfolio_type, created_at) +SELECT 'TREND_FOLLOWING', pid, d, 504500, 0.003, 1.2, 0.4, book, + (d + time '07:00') AT TIME ZONE 'America/New_York' +FROM (VALUES (date '2026-10-14'), (date '2026-10-15')) AS days(d) +CROSS JOIN (VALUES ('QT_CONSERVATIVE_PORTFOLIO', 'system'), ('QT_CONSERVATIVE_PORTFOLIO', 'qt'), + ('QT_CONSERVATIVE_MODEL_PORTFOLIO', 'system')) AS x(pid, book); + +-- 2026-10-14 published by the 10:00 fallback and sent; 2026-10-15 built at 07:00, unpublished. +INSERT INTO trading.live_run_metadata + (date, strategy_id, portfolio_id, created_at, published_by, published_at, publish_source, + sent_at) +VALUES + ('2026-10-14', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_PORTFOLIO', + '2026-10-14 07:00 America/New_York', 'system:fallback-10am', + '2026-10-14 10:00 America/New_York', 'fallback', '2026-10-14 10:00 America/New_York'), + ('2026-10-15', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_PORTFOLIO', + '2026-10-15 07:00 America/New_York', NULL, NULL, NULL, NULL), + ('2026-10-14', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', + '2026-10-14 07:00 America/New_York', 'system:model-only', + '2026-10-14 07:00 America/New_York', 'model-only', '2026-10-14 09:30 America/New_York'), + ('2026-10-15', 'TREND_FOLLOWING', 'QT_CONSERVATIVE_MODEL_PORTFOLIO', + '2026-10-15 07:00 America/New_York', 'system:model-only', + '2026-10-15 07:00 America/New_York', 'model-only', NULL); + +COMMIT; diff --git a/sim/seed/users.sql b/sim/seed/users.sql new file mode 100644 index 000000000..3bd0575b0 --- /dev/null +++ b/sim/seed/users.sql @@ -0,0 +1,34 @@ +-- The simulator's three users, applied by `simctl load` after the snapshot (or the synthetic +-- fixture). The snapshot carries no auth.users rows on purpose. +-- +-- qt@sim.local the desk (QT trader) +-- vp@sim.local VP, approver (QT_APPROVERS vp=) +-- president@sim.local President, approver (QT_APPROVERS president=) +-- +-- Password for all three: sim. The hash is werkzeug's generate_password_hash('sim') (scrypt), +-- the format AlgoLens checks with check_password_hash (auth.users.password_hash). +-- Screens (AlgoLens migration 003): member and qt for each. + +BEGIN; + +INSERT INTO auth.users (email, password_hash, role, first_name, last_name) +VALUES + ('qt@sim.local', + 'scrypt:32768:8:1$wGNaZmcVB0vQDcoM$92e271223334ccb1e61316a42ba3b0f53030b259afad72b0ae388434c104c5a19c1b801150a0973903a3593c92a81742c6c1679797f08271aba6343c86ad1d07', + 'general_member', 'QT', 'Desk'), + ('vp@sim.local', + 'scrypt:32768:8:1$wGNaZmcVB0vQDcoM$92e271223334ccb1e61316a42ba3b0f53030b259afad72b0ae388434c104c5a19c1b801150a0973903a3593c92a81742c6c1679797f08271aba6343c86ad1d07', + 'general_member', 'Sim', 'VP'), + ('president@sim.local', + 'scrypt:32768:8:1$wGNaZmcVB0vQDcoM$92e271223334ccb1e61316a42ba3b0f53030b259afad72b0ae388434c104c5a19c1b801150a0973903a3593c92a81742c6c1679797f08271aba6343c86ad1d07', + 'general_member', 'Sim', 'President') +ON CONFLICT (email) DO UPDATE SET password_hash = EXCLUDED.password_hash; + +INSERT INTO auth.user_screen_access (user_id, screen, granted_by) +SELECT u.id, s.screen, 'sim:seed' +FROM auth.users u +CROSS JOIN (VALUES ('member'), ('qt')) AS s(screen) +WHERE u.email IN ('qt@sim.local', 'vp@sim.local', 'president@sim.local') +ON CONFLICT DO NOTHING; + +COMMIT; diff --git a/sim/simctl b/sim/simctl new file mode 100644 index 000000000..9eeb77813 --- /dev/null +++ b/sim/simctl @@ -0,0 +1,414 @@ +#!/usr/bin/env bash +# simctl: drive the QT desk simulator (sim/README.md). Git Bash on Windows, or any bash. +# +# simctl build build the images (AlgoLens from the checked-out branch) +# simctl up | down | destroy start, stop (keep data), stop and delete every volume +# simctl load [--day YYYY-MM-DD | --keep] +# restore a snapshot (or the synthetic fixture) + the sim users, +# then save it as the template `reset` returns to +# simctl reset back to the template; clock to 08:00 of the sim day; Mailpit +# emptied; engine-rpc restarted (its scheduler state cleared) +# simctl save make the current database the template +# simctl clock HH:MM[:SS] set the clock to that New York time on the sim day +# simctl clock +5m | -1h | +30s +# simctl day YYYY-MM-DD set the sim day (keeps the New York time of day) +# simctl model-run [dir] the engine's model run of the sim day (dir: qt_conservative) +# simctl pass one scheduler pass now (what the 1-minute tick would do) +# simctl status clocks, the day's publish state, the latest desk commands +# simctl mail the Mailpit URL +# simctl logs [service...] follow logs (default: engine-rpc backend) +# simctl psql [args] psql on the sim database +# +# The clock: sim/clock/faketime.rc holds an offset in seconds from real time. libfaketime in +# postgres, engine-rpc and backend reads it, so all three move together and keep ticking. +set -euo pipefail +export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' + +SIM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# docker.exe needs a Windows path under Git Bash (path conversion is off, see above). +SIM_DIR_HOST="$(cygpath -m "$SIM_DIR" 2>/dev/null || echo "$SIM_DIR")" +# sim/.env (optional; sim/.env.example): read by docker compose, and here for PG_IMAGE. +if [ -f "$SIM_DIR/.env" ]; then set -a; . "$SIM_DIR/.env"; set +a; fi +CLOCK_FILE="$SIM_DIR/clock/faketime.rc" +DAY_FILE="$SIM_DIR/clock/day" +DB=new_algo_data +TEMPLATE=sim_template +TZ_NY=America/New_York + +compose() { docker compose -f "$SIM_DIR_HOST/docker-compose.sim.yml" --project-directory "$SIM_DIR_HOST" "$@"; } +die() { echo "simctl: $*" >&2; exit 1; } +say() { echo "simctl: $*"; } + +running() { [ -n "$(compose ps -q --status running "$1" 2>/dev/null)" ]; } + +psql_db() { # psql_db [psql args...] + local db="$1"; shift + compose exec -T -e PGOPTIONS="-c client_min_messages=warning" postgres \ + psql -X -q -v ON_ERROR_STOP=1 -U postgres -d "$db" "$@" +} + +wait_postgres() { + compose up -d postgres >/dev/null + for _ in $(seq 1 60); do + if compose exec -T postgres pg_isready -U postgres -q 2>/dev/null; then return 0; fi + sleep 1 + done + die "postgres did not become ready" +} + +# Real (un-faked) shell in a container with tzdata: the running postgres with LD_PRELOAD +# cleared, else a throwaway postgres container. +real_sh() { + if running postgres; then + compose exec -T -e LD_PRELOAD= postgres sh -c "$1" + else + docker run --rm --entrypoint sh "${PG_IMAGE:-postgres:16}" -c "$1" + fi +} + +ensure_clock() { + mkdir -p "$SIM_DIR/clock" "$SIM_DIR/data" + [ -f "$CLOCK_FILE" ] || printf '+0\n' > "$CLOCK_FILE" +} + +sim_day() { + if [ -f "$DAY_FILE" ]; then tr -d '\r\n ' < "$DAY_FILE"; else date -u +%Y-%m-%d; fi +} + +current_offset() { + ensure_clock + local raw; raw="$(tr -d '\r\n ' < "$CLOCK_FILE")" + raw="${raw#+}" + [[ "$raw" =~ ^-?[0-9]+$ ]] || die "unreadable clock file: $CLOCK_FILE" + echo "$raw" +} + +write_offset() { + ensure_clock + printf '%+d\n' "$1" > "$CLOCK_FILE.tmp" + # In place (not a rename): the file is bind-mounted into three containers. + cat "$CLOCK_FILE.tmp" > "$CLOCK_FILE" + rm -f "$CLOCK_FILE.tmp" +} + +# Seconds from now (real) until New York. +offset_to() { + real_sh "echo \$(( \$(TZ=$TZ_NY date -d '$1 $2' +%s) - \$(date +%s) ))" +} + +show_clock() { + local off; off="$(current_offset)" + real_sh "TZ=$TZ_NY date -d @\$(( \$(date +%s) + $off )) '+sim clock: %Y-%m-%d %H:%M:%S %Z (New York)'; date -u -d @\$(( \$(date +%s) + $off )) '+ %Y-%m-%d %H:%M:%S UTC'" +} + +set_clock_hms() { # HH:MM[:SS] on the sim day + local t="$1" + [[ "$t" =~ ^[0-9]{1,2}:[0-9]{2}(:[0-9]{2})?$ ]] || die "clock: expected HH:MM, got $t" + [[ "$t" =~ ^[0-9]{1,2}:[0-9]{2}$ ]] && t="$t:00" + local off; off="$(offset_to "$(sim_day)" "$t")" + write_offset "$off" + show_clock +} + +cmd_clock() { + [ $# -ge 1 ] || { show_clock; return; } + local arg="$1" + if [[ "$arg" =~ ^([+-])([0-9]+)([smhd]?)$ ]]; then + local n="${BASH_REMATCH[2]}" unit="${BASH_REMATCH[3]:-s}" mult=1 + case "$unit" in m) mult=60 ;; h) mult=3600 ;; d) mult=86400 ;; esac + local delta=$(( n * mult )) + [ "${BASH_REMATCH[1]}" = "-" ] && delta=$(( -delta )) + write_offset $(( $(current_offset) + delta )) + show_clock + else + set_clock_hms "$arg" + fi +} + +cmd_day() { + local d="${1:-}" + [[ "$d" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || die "day: expected YYYY-MM-DD" + local hms; hms="$(real_sh "TZ=$TZ_NY date -d @\$(( \$(date +%s) + $(current_offset) )) +%H:%M:%S")" + echo "$d" > "$DAY_FILE" + say "sim day $d" + set_clock_hms "$hms" +} + +cmd_build() { + ensure_clock + compose --profile build build backend-base + compose build "$@" +} + +cmd_up() { + ensure_clock + if [ -z "$(docker images -q algolens-sim/backend-base:local 2>/dev/null)" ]; then + compose --profile build build backend-base + fi + compose up -d --build + say "AlgoLens http://localhost:8080 (qt@sim.local / vp@sim.local / president@sim.local, password sim)" + say "Mailpit http://localhost:8025" + show_clock +} + +stop_apps() { compose stop engine-rpc backend >/dev/null 2>&1 || true; } + +clear_engine_state() { + # The scheduler's alert and review records and stale flocks live in volumes. + compose run --rm --no-deps -T --entrypoint sh engine-rpc \ + -c 'rm -rf /var/log/qt-engine/* /tmp/qt-locks/*' >/dev/null 2>&1 || true +} + +clear_mail() { + if running mailpit; then + curl -fsS -X DELETE http://localhost:8025/api/v1/messages >/dev/null 2>&1 \ + && say "Mailpit emptied" || true + fi +} + +fix_sequences() { + psql_db "$DB" <<'SQL' +DO $$ +DECLARE r record; m bigint; +BEGIN + FOR r IN + SELECT s.oid::regclass AS seq, c.oid::regclass AS tbl, a.attname AS col + FROM pg_class s + JOIN pg_depend d ON d.objid = s.oid AND d.deptype IN ('a', 'i') + JOIN pg_class c ON c.oid = d.refobjid + JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = d.refobjsubid + WHERE s.relkind = 'S' + LOOP + EXECUTE format('SELECT max(%I)::bigint FROM %s', r.col, r.tbl) INTO m; + PERFORM setval(r.seq, COALESCE(m, 0) + 1, false); + END LOOP; +END $$; +SQL +} + +newest_qt_day() { + psql_db "$DB" -At -c "SELECT COALESCE(max(date)::text, '') FROM trading.positions WHERE portfolio_id = 'QT_CONSERVATIVE_PORTFOLIO'" +} + +# Remove everything of the QT portfolios on and after , so the real engine rebuilds it. +drop_from_day() { + psql_db "$DB" -c "SELECT set_config('sim.day', '$1', false)" -f - >/dev/null <<'SQL' +SET session_replication_role = replica; -- position_overrides' guard triggers +DO $$ +DECLARE t text; col text; +BEGIN + FOREACH t IN ARRAY ARRAY['positions', 'executions', 'live_results', 'live_run_metadata', + 'position_overrides', 'signals', 'equity_curve'] + LOOP + IF to_regclass('trading.' || t) IS NULL THEN CONTINUE; END IF; + col := CASE WHEN t = 'equity_curve' + THEN '("timestamp" AT TIME ZONE ''America/New_York'')::date' ELSE 'date' END; + EXECUTE format( + 'DELETE FROM trading.%I WHERE portfolio_id IN (''QT_CONSERVATIVE_PORTFOLIO'', ' + '''QT_CONSERVATIVE_MODEL_PORTFOLIO'') AND %s >= %L::date', + t, col, current_setting('sim.day')); + END LOOP; +END $$; +SQL +} + +cmd_load() { + local src="${1:-}"; shift || true + [ -n "$src" ] || die "load: give a snapshot directory (sim/data/...) or 'synthetic'" + local day="" keep=no + while [ $# -gt 0 ]; do + case "$1" in + --day) day="${2:-}"; shift 2 ;; + --keep) keep=yes; shift ;; + *) die "load: unknown option $1" ;; + esac + done + ensure_clock + + local snap="" + if [ "$src" != synthetic ]; then + local abs; abs="$(cd "$src" 2>/dev/null && pwd)" || die "load: no directory $src" + case "$abs" in + "$SIM_DIR/data/"*) snap="${abs#"$SIM_DIR/data/"}" ;; + *) die "load: the snapshot must be under sim/data/ (mounted at /sim-data)" ;; + esac + [ -f "$abs/schema.sql.gz" ] || die "load: $abs has no schema.sql.gz" + if [ -f "$abs/server_version" ]; then + local major; major="$(cut -d. -f1 < "$abs/server_version" | tr -d '\r ')" + local want="${PG_IMAGE:-postgres:16}"; want="${want##*:}"; want="${want%%.*}" + [ "$major" = "$want" ] || say "WARNING: snapshot from Postgres $major, sim runs $want (set PG_IMAGE=postgres:$major and simctl build)" + fi + fi + + local apps_were_up=no + running backend && apps_were_up=yes + stop_apps + wait_postgres + + say "recreating $DB" + psql_db postgres -c "DROP DATABASE IF EXISTS $TEMPLATE WITH (FORCE)" \ + -c "DROP DATABASE IF EXISTS $DB WITH (FORCE)" -c "CREATE DATABASE $DB" + + if [ "$src" = synthetic ]; then + say "restoring the synthetic fixture (sim/seed/synthetic.sql)" + psql_db "$DB" -f /sim-seed/synthetic.sql >/dev/null + else + local d="/sim-data/$snap" errs=0 + say "restoring the schema of $snap" + # Not ON_ERROR_STOP: objects the sim cannot create (an extension prod has) are reported + # and skipped. + compose exec -T postgres sh -c "gunzip -c $d/schema.sql.gz | psql -X -q -U postgres -d $DB" \ + > /dev/null 2> "$SIM_DIR/data/$snap.load.log" || true + errs=$(grep -c ERROR "$SIM_DIR/data/$snap.load.log" || true) + say "restoring market data and metadata (data.sql.gz)" + compose exec -T postgres sh -c "gunzip -c $d/data.sql.gz | psql -X -q -U postgres -d $DB" \ + > /dev/null 2>> "$SIM_DIR/data/$snap.load.log" || true + local f t + for f in "$SIM_DIR/data/$snap"/trading.*.csv.gz; do + [ -e "$f" ] || continue + t="$(basename "$f" .csv.gz)" + say "restoring $t (QT portfolios)" + compose exec -T postgres psql -X -q -U postgres -d "$DB" \ + -c "SET session_replication_role = replica" \ + -c "\\copy $t FROM PROGRAM 'gunzip -c $d/$t.csv.gz' WITH (FORMAT csv, HEADER true)" \ + >/dev/null 2>> "$SIM_DIR/data/$snap.load.log" || true + done + errs=$(grep -c ERROR "$SIM_DIR/data/$snap.load.log" || true) + [ "$errs" = 0 ] || say "WARNING: $errs errors while restoring; see sim/data/$snap.load.log" + fix_sequences + fi + + say "seeding the sim users (sim/seed/users.sql)" + psql_db "$DB" -f /sim-seed/users.sql >/dev/null + + local newest; newest="$(newest_qt_day)" + # The sim day: --day, else the newest QT day of the data. From a snapshot (not --keep) that + # day's rows are removed so the real engine rebuilds it (simctl model-run, or the scheduler). + [ -n "$day" ] || day="$newest" + [ -n "$day" ] || day="$(date -u +%Y-%m-%d)" + if [ "$src" != synthetic ] && [ "$keep" = no ]; then + say "removing the QT portfolios' rows on and after $day (the engine rebuilds that day)" + drop_from_day "$day" + fi + echo "$day" > "$DAY_FILE" + say "sim day $day" + set_clock_hms 08:00 + + psql_db postgres -c "CREATE DATABASE $TEMPLATE TEMPLATE $DB" + say "saved as the template ($TEMPLATE); simctl reset returns here" + clear_engine_state + if [ "$apps_were_up" = yes ]; then compose up -d >/dev/null; fi +} + +cmd_save() { + wait_postgres + local was_up=no; running engine-rpc && was_up=yes + stop_apps + psql_db postgres -c "DROP DATABASE IF EXISTS $TEMPLATE WITH (FORCE)" \ + -c "CREATE DATABASE $TEMPLATE TEMPLATE $DB" + say "the current database is the template now" + [ "$was_up" = no ] || compose up -d >/dev/null +} + +cmd_reset() { + wait_postgres + [ "$(psql_db postgres -At -c "SELECT count(*) FROM pg_database WHERE datname = '$TEMPLATE'")" = 1 ] \ + || die "reset: no template; run simctl load first" + local was_up=no; running engine-rpc && was_up=yes + stop_apps + # Before the engine starts again: no pass may run at the old clock on the fresh database. + set_clock_hms 08:00 + psql_db postgres -c "DROP DATABASE IF EXISTS $DB WITH (FORCE)" \ + -c "CREATE DATABASE $DB TEMPLATE $TEMPLATE" + clear_engine_state + clear_mail + say "reset to the template" + [ "$was_up" = no ] || compose up -d >/dev/null +} + +portfolio_id_of() { # qt_conservative -> QT_CONSERVATIVE_PORTFOLIO + echo "$(echo "$1" | tr '[:lower:]' '[:upper:]')_PORTFOLIO" +} + +cmd_model_run() { + local dir="${1:-qt_conservative}" day; day="$(sim_day)" + local pid; pid="$(portfolio_id_of "$dir")" + running engine-rpc || die "engine-rpc is not running (simctl up)" + say "model run of $pid ($dir) for $day, under its flock, as the scheduler runs it" + compose exec -T -w /app engine-rpc flock "/tmp/qt-locks/$pid.lock" \ + /app/build/bin/Release/live_portfolio_conservative --portfolio-config "$dir" --date "$day" +} + +cmd_pass() { + running engine-rpc || die "engine-rpc is not running (simctl up)" + compose exec -T -w /opt/rpc/app engine-rpc /opt/rpc/venv/bin/python \ + -m algogators_rpc.services.desk.catchup run-once || true +} + +cmd_status() { + show_clock + say "sim day $(sim_day)" + echo "--- clocks inside the containers" + local s + for s in postgres engine-rpc backend; do + if running "$s"; then + printf '%-11s %s\n' "$s" "$(compose exec -T "$s" date -u '+%Y-%m-%d %H:%M:%S UTC')" + else + printf '%-11s (not running)\n' "$s" + fi + done + if running postgres; then + printf '%-11s %s\n' "now()" "$(psql_db "$DB" -At -c "SELECT to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS') || ' UTC = ' || to_char(now() AT TIME ZONE 'America/New_York', 'HH24:MI:SS') || ' New York'" 2>/dev/null || echo '?')" + echo "--- publish state of $(sim_day)" + psql_db "$DB" -P pager=off -c " + SELECT portfolio_id, date, published_by, + to_char(published_at AT TIME ZONE 'America/New_York', 'HH24:MI:SS') AS published_ny, + publish_source, + to_char(sent_at AT TIME ZONE 'America/New_York', 'HH24:MI:SS') AS sent_ny, + to_char(created_at AT TIME ZONE 'America/New_York', 'HH24:MI:SS') AS built_ny + FROM trading.live_run_metadata + WHERE portfolio_id LIKE 'QT\\_%' AND date = '$(sim_day)' + ORDER BY portfolio_id, id" 2>/dev/null || true + echo "--- latest desk commands (trading.position_overrides)" + psql_db "$DB" -P pager=off -c " + SELECT id, date, kind, status, requested_by, approver_role, + to_char(created_at AT TIME ZONE 'America/New_York', 'MM-DD HH24:MI:SS') AS created_ny, + to_char(finished_at AT TIME ZONE 'America/New_York', 'HH24:MI:SS') AS finished_ny, + left(coalesce(message, ''), 60) AS message + FROM trading.position_overrides + WHERE portfolio_id LIKE 'QT\\_%' + ORDER BY id DESC LIMIT 10" 2>/dev/null || true + fi + echo "--- mail: http://localhost:8025" +} + +cmd_logs() { + if [ $# -eq 0 ]; then set -- engine-rpc backend; fi + compose logs -f --tail=100 "$@" +} + +main() { + local cmd="${1:-help}"; shift || true + case "$cmd" in + build) cmd_build "$@" ;; + up) cmd_up ;; + down) compose down ;; + destroy) compose down -v ;; + load) cmd_load "$@" ;; + reset) cmd_reset ;; + save) cmd_save ;; + clock) cmd_clock "$@" ;; + day) cmd_day "$@" ;; + model-run) cmd_model_run "$@" ;; + pass) cmd_pass ;; + status) cmd_status ;; + mail) echo "Mailpit: http://localhost:8025" ;; + logs) cmd_logs "$@" ;; + psql) compose exec postgres psql -U postgres -d "$DB" "$@" ;; + compose) compose "$@" ;; + help|-h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//' ;; + *) die "unknown command $cmd (simctl help)" ;; + esac +} + +main "$@" diff --git a/sim/simctl.ps1 b/sim/simctl.ps1 new file mode 100644 index 000000000..8596d55d1 --- /dev/null +++ b/sim/simctl.ps1 @@ -0,0 +1,7 @@ +# PowerShell wrapper: runs sim/simctl with Git Bash. Same commands and arguments. +# .\sim\simctl.ps1 clock 09:29 +$ErrorActionPreference = 'Stop' +$bash = Join-Path $env:ProgramFiles 'Git\bin\bash.exe' +if (-not (Test-Path $bash)) { $bash = 'bash' } +& $bash (Join-Path $PSScriptRoot 'simctl') @args +exit $LASTEXITCODE diff --git a/sim/snapshot.sh b/sim/snapshot.sh new file mode 100644 index 000000000..33852cc01 --- /dev/null +++ b/sim/snapshot.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Take a READ-ONLY snapshot of production new_algo_data for the simulator. +# +# Dom runs this by hand (it is never run by tooling). It only reads: pg_dump and +# `\copy (SELECT ...) TO STDOUT`, streamed back over ssh. Nothing is written on the server, +# no file, no table. +# +# bash sim/snapshot.sh # writes sim/data/snapshot-/ +# bash sim/snapshot.sh my-name # writes sim/data/my-name/ +# +# Then: bash sim/simctl load sim/data/ +# +# What it takes (all gzip-compressed): +# schema.sql.gz schema only of trading, metadata, futures_data, auth, ops, investor +# (the ones present), --no-owner --no-privileges +# data.sql.gz every row of metadata.contract_metadata, futures_data.ohlcv_1d (the +# engine's market data, ~3 years), ops.data_ready and +# trading.strategy_registry +# trading..csv.gz rows of the QT portfolios only (QT_CONSERVATIVE_PORTFOLIO, +# QT_CONSERVATIVE_MODEL_PORTFOLIO) of positions, executions, +# live_results, live_run_metadata, equity_curve, position_overrides, +# strategy_config and signals (the ones present) +# server_version the server's version (restore into the same major version: PG_IMAGE) +# Not taken: auth.users rows and any other user data. The simulator seeds its own users. +# +# Database login: the read role in ~/.config/algogators/db-app-roles.env on the host, when that +# file sets PGUSER and PGPASSWORD; otherwise you are asked here and the answers are sent to the +# server over the ssh connection's stdin (never on a command line). +set -euo pipefail + +SSH_HOST="${SNAPSHOT_SSH_HOST:-ubuntu@ec2-18-118-225-224.us-east-2.compute.amazonaws.com}" +SSH_KEY="${SNAPSHOT_SSH_KEY:-C:/Users/domdd/Documents/GitHub/prod-deploy.pem}" +DB_NAME="new_algo_data" +ROLES_FILE='~/.config/algogators/db-app-roles.env' +QT_PORTFOLIOS="'QT_CONSERVATIVE_PORTFOLIO','QT_CONSERVATIVE_MODEL_PORTFOLIO'" +SCHEMAS=(trading metadata futures_data auth ops investor) +FULL_TABLES=(metadata.contract_metadata futures_data.ohlcv_1d ops.data_ready trading.strategy_registry) +QT_TABLES=(positions executions live_results live_run_metadata equity_curve position_overrides + strategy_config signals) + +here="$(cd "$(dirname "$0")" && pwd)" +name="${1:-snapshot-$(date -u +%Y%m%d-%H%M%S)}" +out="$here/data/$name" +if [ -e "$out" ]; then + echo "snapshot: $out already exists" >&2 + exit 1 +fi + +ssh_q() { ssh -i "$SSH_KEY" -o BatchMode=yes -o ConnectTimeout=15 "$SSH_HOST" "$@"; } + +# Whether the host's roles file provides the login. +creds_from_file=no +if ssh_q "f=$ROLES_FILE; [ -f \"\$f\" ] && (set -a; . \"\$f\"; [ -n \"\${PGUSER:-}\" ] && [ -n \"\${PGPASSWORD:-}\" ])" "$out/server_version" +echo " server_version $(cat "$out/server_version")" + +present_schemas=$(remote "psql -XAtc \"SELECT string_agg(nspname, ' ') FROM pg_namespace WHERE nspname IN ($(printf "'%s'," "${SCHEMAS[@]}" | sed 's/,$//'))\"") +echo " schemas: $present_schemas" +schema_args="" +for s in $present_schemas; do schema_args+=" -n $s"; done + +echo " schema.sql.gz" +remote "pg_dump --schema-only --no-owner --no-privileges $schema_args" | gzip > "$out/schema.sql.gz" + +table_present() { + remote "psql -XAtc \"SELECT to_regclass('$1') IS NOT NULL\"" +} + +full_args="" +for t in "${FULL_TABLES[@]}"; do + if [ "$(table_present "$t")" = t ]; then full_args+=" -t $t"; else echo " (no $t)"; fi +done +echo " data.sql.gz:$full_args" +remote "pg_dump --data-only --no-owner --no-privileges --disable-triggers $full_args" | gzip > "$out/data.sql.gz" + +for t in "${QT_TABLES[@]}"; do + if [ "$(table_present "trading.$t")" != t ]; then + echo " (no trading.$t)" + continue + fi + echo " trading.$t.csv.gz" + remote "psql -Xc \"\\copy (SELECT * FROM trading.$t WHERE portfolio_id IN ($QT_PORTFOLIOS)) TO STDOUT WITH (FORMAT csv, HEADER true)\"" \ + | gzip > "$out/trading.$t.csv.gz" +done + +du -sh "$out" | sed 's/^/snapshot: done, /' +echo "next: bash sim/simctl load $out" From 2eebcf60bd7017a842b27446862d6432b95d37c0 Mon Sep 17 00:00:00 2001 From: BonelessWater Date: Wed, 14 Oct 2026 16:40:17 -0400 Subject: [PATCH 2/3] sim: mark the scripts executable Claude-Session: https://claude.ai/code/session_01VCt4NHq2wcnzSiVtdabnEJ --- sim/seed/build-synthetic.sh | 0 sim/simctl | 0 sim/snapshot.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sim/seed/build-synthetic.sh mode change 100644 => 100755 sim/simctl mode change 100644 => 100755 sim/snapshot.sh diff --git a/sim/seed/build-synthetic.sh b/sim/seed/build-synthetic.sh old mode 100644 new mode 100755 diff --git a/sim/simctl b/sim/simctl old mode 100644 new mode 100755 diff --git a/sim/snapshot.sh b/sim/snapshot.sh old mode 100644 new mode 100755 From 3448bd02affd3bbd9adfbc33fef1a924356ea05a Mon Sep 17 00:00:00 2001 From: BonelessWater Date: Wed, 14 Oct 2026 18:12:15 -0400 Subject: [PATCH 3/3] sim: snapshot through the hypertables and the app roles; restore and seed fixes - snapshot: the AlgoLens app role from the host's roles file; only the schemas and tables that login can read (pg_dump locks every table); the market data copied with \copy (SELECT *) as the engine's login, since TimescaleDB hypertables dump no rows from their parent; the definition of tables the first login cannot read (ohlcv_1d_raw). - load: restores schema-extra and every .
.csv.gz. - users seed: prod's auth.users has no unique email; replace the three sim users. - drop_from_day: tables without a date column use their timestamp in New York. --- sim/seed/users.sql | 8 +++++++- sim/simctl | 18 ++++++++++++----- sim/snapshot.sh | 48 ++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/sim/seed/users.sql b/sim/seed/users.sql index 3bd0575b0..d8ef28f0f 100644 --- a/sim/seed/users.sql +++ b/sim/seed/users.sql @@ -11,6 +11,12 @@ BEGIN; +-- Prod's auth.users has no unique constraint on email: replace the three rows instead of +-- upserting them (this database is the sim's own). +DELETE FROM auth.user_screen_access WHERE user_id IN ( + SELECT id FROM auth.users WHERE email IN ('qt@sim.local', 'vp@sim.local', 'president@sim.local')); +DELETE FROM auth.users WHERE email IN ('qt@sim.local', 'vp@sim.local', 'president@sim.local'); + INSERT INTO auth.users (email, password_hash, role, first_name, last_name) VALUES ('qt@sim.local', @@ -22,7 +28,7 @@ VALUES ('president@sim.local', 'scrypt:32768:8:1$wGNaZmcVB0vQDcoM$92e271223334ccb1e61316a42ba3b0f53030b259afad72b0ae388434c104c5a19c1b801150a0973903a3593c92a81742c6c1679797f08271aba6343c86ad1d07', 'general_member', 'Sim', 'President') -ON CONFLICT (email) DO UPDATE SET password_hash = EXCLUDED.password_hash; +; INSERT INTO auth.user_screen_access (user_id, screen, granted_by) SELECT u.id, s.screen, 'sim:seed' diff --git a/sim/simctl b/sim/simctl index 9eeb77813..5a55495ab 100755 --- a/sim/simctl +++ b/sim/simctl @@ -201,8 +201,12 @@ BEGIN 'position_overrides', 'signals', 'equity_curve'] LOOP IF to_regclass('trading.' || t) IS NULL THEN CONTINUE; END IF; - col := CASE WHEN t = 'equity_curve' - THEN '("timestamp" AT TIME ZONE ''America/New_York'')::date' ELSE 'date' END; + -- The day column: date where the table has one, else its "timestamp" in New York. + col := CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_schema = 'trading' AND table_name = t + AND column_name = 'date') + THEN 'date' + ELSE '("timestamp" AT TIME ZONE ''America/New_York'')::date' END; EXECUTE format( 'DELETE FROM trading.%I WHERE portfolio_id IN (''QT_CONSERVATIVE_PORTFOLIO'', ' '''QT_CONSERVATIVE_MODEL_PORTFOLIO'') AND %s >= %L::date', @@ -259,15 +263,19 @@ cmd_load() { # and skipped. compose exec -T postgres sh -c "gunzip -c $d/schema.sql.gz | psql -X -q -U postgres -d $DB" \ > /dev/null 2> "$SIM_DIR/data/$snap.load.log" || true + if [ -f "$SIM_DIR/data/$snap/schema-extra.sql.gz" ]; then + compose exec -T postgres sh -c "gunzip -c $d/schema-extra.sql.gz | psql -X -q -U postgres -d $DB" > /dev/null 2>> "$SIM_DIR/data/$snap.load.log" || true + fi errs=$(grep -c ERROR "$SIM_DIR/data/$snap.load.log" || true) - say "restoring market data and metadata (data.sql.gz)" + say "restoring metadata (data.sql.gz)" compose exec -T postgres sh -c "gunzip -c $d/data.sql.gz | psql -X -q -U postgres -d $DB" \ > /dev/null 2>> "$SIM_DIR/data/$snap.load.log" || true local f t - for f in "$SIM_DIR/data/$snap"/trading.*.csv.gz; do + # .
.csv.gz: the QT portfolios' trading rows and the market data. + for f in "$SIM_DIR/data/$snap"/*.csv.gz; do [ -e "$f" ] || continue t="$(basename "$f" .csv.gz)" - say "restoring $t (QT portfolios)" + say "restoring $t" compose exec -T postgres psql -X -q -U postgres -d "$DB" \ -c "SET session_replication_role = replica" \ -c "\\copy $t FROM PROGRAM 'gunzip -c $d/$t.csv.gz' WITH (FORMAT csv, HEADER true)" \ diff --git a/sim/snapshot.sh b/sim/snapshot.sh index 33852cc01..aeadf5ebe 100755 --- a/sim/snapshot.sh +++ b/sim/snapshot.sh @@ -34,7 +34,11 @@ DB_NAME="new_algo_data" ROLES_FILE='~/.config/algogators/db-app-roles.env' QT_PORTFOLIOS="'QT_CONSERVATIVE_PORTFOLIO','QT_CONSERVATIVE_MODEL_PORTFOLIO'" SCHEMAS=(trading metadata futures_data auth ops investor) -FULL_TABLES=(metadata.contract_metadata futures_data.ohlcv_1d ops.data_ready trading.strategy_registry) +FULL_TABLES=(metadata.contract_metadata ops.data_ready trading.strategy_registry) +# The engine's market data. TimescaleDB hypertables on prod: pg_dump of the parent carries no rows +# (they live in _timescaledb_internal chunks), so each is copied with \copy (SELECT *), as the +# engine's login (QT_ENGINE_DB_*), which reads both. +MARKET_TABLES=(futures_data.ohlcv_1d futures_data.ohlcv_1d_raw) QT_TABLES=(positions executions live_results live_run_metadata equity_curve position_overrides strategy_config signals) @@ -50,7 +54,10 @@ ssh_q() { ssh -i "$SSH_KEY" -o BatchMode=yes -o ConnectTimeout=15 "$SSH_HOST" "$ # Whether the host's roles file provides the login. creds_from_file=no -if ssh_q "f=$ROLES_FILE; [ -f \"\$f\" ] && (set -a; . \"\$f\"; [ -n \"\${PGUSER:-}\" ] && [ -n \"\${PGPASSWORD:-}\" ])" "$out/server_version" echo " server_version $(cat "$out/server_version")" -present_schemas=$(remote "psql -XAtc \"SELECT string_agg(nspname, ' ') FROM pg_namespace WHERE nspname IN ($(printf "'%s'," "${SCHEMAS[@]}" | sed 's/,$//'))\"") +# Only the schemas this login may use (the AlgoLens role has no USAGE on ops, say). +present_schemas=$(remote "psql -XAtc \"SELECT string_agg(nspname, ' ') FROM pg_namespace WHERE nspname IN ($(printf "'%s'," "${SCHEMAS[@]}" | sed 's/,$//')) AND has_schema_privilege(nspname, 'USAGE')\"") echo " schemas: $present_schemas" schema_args="" for s in $present_schemas; do schema_args+=" -n $s"; done +# pg_dump locks every table it dumps, even schema only: leave out the tables this login cannot +# read (backups and staging copies the apps never use). +schema_list=$(printf "'%s'," $present_schemas | sed 's/,$//') +unreadable=$(remote "psql -XAtc \"SELECT quote_ident(schemaname) || '.' || quote_ident(tablename) FROM pg_tables WHERE schemaname IN ($schema_list) AND NOT has_table_privilege(quote_ident(schemaname) || '.' || quote_ident(tablename), 'SELECT') ORDER BY 1\"") +for t in $unreadable; do schema_args+=" --exclude-table=$t"; done +[ -n "$unreadable" ] && echo " left out (not readable by this login): $(echo $unreadable)" echo " schema.sql.gz" remote "pg_dump --schema-only --no-owner --no-privileges $schema_args" | gzip > "$out/schema.sql.gz" -table_present() { - remote "psql -XAtc \"SELECT to_regclass('$1') IS NOT NULL\"" +table_present() { # present AND readable by this login + remote "psql -XAtc \"SELECT CASE WHEN NOT has_schema_privilege(split_part('$1', '.', 1), 'USAGE') THEN false WHEN to_regclass('$1') IS NULL THEN false ELSE has_table_privilege('$1', 'SELECT') END\"" } full_args="" @@ -110,5 +130,21 @@ for t in "${QT_TABLES[@]}"; do | gzip > "$out/trading.$t.csv.gz" done +# The market data the engine reads (see MARKET_TABLES), and the definition of any of them the +# first login could not dump. +extra="" +for t in "${MARKET_TABLES[@]}"; do + case " $(echo $unreadable) " in *" $t "*) extra+=" -t $t" ;; esac +done +if [ -n "$extra" ]; then + echo " schema-extra.sql.gz:$extra (as the engine's login)" + remote_engine "pg_dump --schema-only --no-owner --no-privileges$extra" | gzip > "$out/schema-extra.sql.gz" +fi +for t in "${MARKET_TABLES[@]}"; do + echo " $t.csv.gz (as the engine's login)" + remote_engine "psql -Xc \"\copy (SELECT * FROM $t) TO STDOUT WITH (FORMAT csv, HEADER true)\"" | gzip > "$out/$t.csv.gz" + echo " $(( $(gunzip -c "$out/$t.csv.gz" | wc -l) - 1 )) rows" +done + du -sh "$out" | sed 's/^/snapshot: done, /' echo "next: bash sim/simctl load $out"