From 34f42895bfc20f6e6720958ad6ce7051f3b8477e Mon Sep 17 00:00:00 2001 From: ryder777777 Date: Wed, 21 Oct 2026 12:38:36 +0000 Subject: [PATCH] feat(metadata): define canonical MarketMetadataV1 schema and JCS+NFC verification (#144) --- .../agents/core/metadata.test.ts | 203 +++++++ oracledesk-stellar/agents/core/metadata.ts | 529 ++++++++++++++++++ .../docs/frontend-integration.md | 11 +- .../docs/market-metadata-vectors.json | 121 ++++ oracledesk-stellar/docs/market-metadata.md | 66 +++ oracledesk-stellar/model/market_metadata.py | 361 ++++++++++++ oracledesk-stellar/scripts/seed-market.sh | 46 +- oracledesk-stellar/scripts/verify-market.sh | 26 + 8 files changed, 1343 insertions(+), 20 deletions(-) create mode 100644 oracledesk-stellar/agents/core/metadata.test.ts create mode 100644 oracledesk-stellar/agents/core/metadata.ts create mode 100644 oracledesk-stellar/docs/market-metadata-vectors.json create mode 100644 oracledesk-stellar/docs/market-metadata.md create mode 100644 oracledesk-stellar/model/market_metadata.py create mode 100755 oracledesk-stellar/scripts/verify-market.sh diff --git a/oracledesk-stellar/agents/core/metadata.test.ts b/oracledesk-stellar/agents/core/metadata.test.ts new file mode 100644 index 0000000..bdcccf9 --- /dev/null +++ b/oracledesk-stellar/agents/core/metadata.test.ts @@ -0,0 +1,203 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { + SCHEMA_V1, + MetadataError, + buildMarketMetadata, + canonicalMetadataBytes, + checkAgainstMarket, + computeCidV1RawSha256, + ipfsUriToGatewayUrl, + metadataHashHex, + validateMetadata, + verifyMetadata, + type MarketMetadataV1, +} from "./metadata.js"; + +interface GoldenVector { + name: string; + input: MarketMetadataV1; + canonical_utf8: string; + canonical_hex: string; + sha256: string; +} + +test("golden vectors produce identical bytes and hashes in TypeScript and Python", () => { + const vectorsPath = fileURLToPath( + new URL("../../docs/market-metadata-vectors.json", import.meta.url), + ); + const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")) as GoldenVector[]; + assert.ok(vectors.length >= 4); + + for (const vec of vectors) { + const bytes = canonicalMetadataBytes(vec.input); + const hex = Buffer.from(bytes).toString("hex"); + const hash = metadataHashHex(vec.input); + assert.equal(hex, vec.canonical_hex, `${vec.name}: canonical_hex mismatch`); + assert.equal(hash, vec.sha256, `${vec.name}: sha256 mismatch`); + + const verified = verifyMetadata(bytes, hash); + assert.equal(metadataHashHex(verified), vec.sha256); + } +}); + +test("NFC normalization is applied before hashing", () => { + const decomposed = buildMarketMetadata({ + question: "Will Cafe\u0301 index exceed 100?", + category: "Macro", + close_time: 1800000000, + resolution: { + mode: "signers", + criteria: "Confirmed by 2-of-3 signers (resume\u0301 feed).", + spec: { threshold: "2", signers: ["GAAA", "GBBB"] }, + resolution_hash: "a".repeat(64), + }, + sources: [], + }); + + const precomposed = buildMarketMetadata({ + question: "Will Caf\u00e9 index exceed 100?", + category: "Macro", + close_time: 1800000000, + resolution: { + mode: "signers", + criteria: "Confirmed by 2-of-3 signers (resum\u00e9 feed).", + spec: { threshold: "2", signers: ["GAAA", "GBBB"] }, + resolution_hash: "a".repeat(64), + }, + sources: [], + }); + + assert.equal(metadataHashHex(decomposed), metadataHashHex(precomposed)); + assert.deepEqual(canonicalMetadataBytes(decomposed), canonicalMetadataBytes(precomposed)); +}); + +test("verifyMetadata hashes received bytes and rejects non-canonical bytes", () => { + const meta = buildMarketMetadata({ + question: "Will XLM close above $0.20 on 2026-12-31?", + category: "Crypto", + close_time: 1798675200, + resolution: { + mode: "price", + criteria: "Reflector XLM/USD > 2000000", + spec: { asset: "XLM", threshold: "2000000" }, + resolution_hash: "b".repeat(64), + }, + sources: [{ name: "Reflector", url: "https://reflector.network" }], + }); + + const canonical = canonicalMetadataBytes(meta); + const hash = metadataHashHex(meta); + assert.equal(verifyMetadata(canonical, hash).question, meta.question); + + // Pretty-printed JSON with its own sha256 must still be rejected as non-canonical + const prettyBytes = new Uint8Array(Buffer.from(JSON.stringify(meta, null, 2), "utf8")); + assert.throws( + () => verifyMetadata(prettyBytes, hash), + (err: unknown) => err instanceof MetadataError && err.path === "$.question_hash", + ); +}); + +test("checkAgainstMarket rejects mismatched resolution_hash, spec hash, close_time, or category", () => { + const rhash = "c".repeat(64); + const meta = buildMarketMetadata({ + question: "Will XLM close above $0.20?", + category: "Crypto", + close_time: 1798675200, + resolution: { + mode: "price", + criteria: "Reflector XLM/USD > 2000000", + spec: { asset: "XLM", threshold: "2000000" }, + resolution_hash: rhash, + }, + sources: [], + }); + const qhash = metadataHashHex(meta); + + checkAgainstMarket( + meta, + { + question_hash: qhash, + resolution_hash: rhash, + close_time: 1798675200n, + category: { tag: "Crypto" }, + }, + rhash, + ); + + assert.throws( + () => + checkAgainstMarket(meta, { + question_hash: qhash, + resolution_hash: "d".repeat(64), + close_time: 1798675200n, + category: "Crypto", + }), + (err: unknown) => err instanceof MetadataError && err.path === "$.resolution.resolution_hash", + ); + + assert.throws( + () => + checkAgainstMarket( + meta, + { + question_hash: qhash, + resolution_hash: rhash, + close_time: 1798675200n, + category: "Crypto", + }, + "e".repeat(64), + ), + (err: unknown) => err instanceof MetadataError && err.path === "$.resolution.spec", + ); +}); + +test("validateMetadata reports path of invalid fields and rejects floats", () => { + assert.throws( + () => + validateMetadata({ + schema: SCHEMA_V1, + question: "Valid question?", + category: "Crypto", + close_time: 1798675200, + resolution: { + mode: "price", + criteria: "Valid criteria", + spec: { threshold: 0.25 }, + resolution_hash: "a".repeat(64), + }, + sources: [], + }), + (err: unknown) => err instanceof MetadataError && err.path === "$.resolution.spec.threshold", + ); + + assert.throws( + () => + validateMetadata({ + schema: SCHEMA_V1, + question: "", + category: "Crypto", + close_time: 1798675200, + resolution: { + mode: "price", + criteria: "Valid criteria", + spec: { threshold: "2500000" }, + resolution_hash: "a".repeat(64), + }, + sources: [], + }), + (err: unknown) => err instanceof MetadataError && err.path === "$.question", + ); +}); + +test("CIDv1 raw sha256 and ipfsUriToGatewayUrl format content-addressed URIs properly", () => { + const bytes = new Uint8Array(Buffer.from("hello world", "utf8")); + const cid = computeCidV1RawSha256(bytes); + assert.ok(cid.startsWith("b")); + assert.equal( + ipfsUriToGatewayUrl("https://ipfs.io/ipfs/", `ipfs://${cid}`), + `https://ipfs.io/ipfs/${cid}`, + ); +}); diff --git a/oracledesk-stellar/agents/core/metadata.ts b/oracledesk-stellar/agents/core/metadata.ts new file mode 100644 index 0000000..3d8ac0c --- /dev/null +++ b/oracledesk-stellar/agents/core/metadata.ts @@ -0,0 +1,529 @@ +/** + * Reference TypeScript implementation of OracleDesk Market Metadata Schema v1 + * (`oracledesk.market-metadata/1`). + * + * Implements: + * - Unicode NFC normalization on all object keys and strings + * - Schema v1 validation (`MetadataError` with field path) + * - RFC 8785 JSON Canonicalization Scheme (JCS) with UTF-16 code unit key ordering + * - `canonicalMetadataBytes`, `metadataHash`, `verifyMetadata`, `checkAgainstMarket` + * - Content-addressed CIDv1 (`raw` codec + `sha2-256`) and `LocalMetadataStore` + */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +export const SCHEMA_V1 = "oracledesk.market-metadata/1" as const; + +export const VALID_CATEGORIES = [ + "Sports", + "Crypto", + "Geopolitics", + "Culture", + "Macro", + "Other", +] as const; + +export type MarketCategory = (typeof VALID_CATEGORIES)[number]; +export type ResolutionMode = "price" | "signers"; + +export interface MarketMetadataSource { + name: string; + url: string; +} + +export interface MarketMetadataResolution { + mode: ResolutionMode; + criteria: string; + spec: Record; + resolution_hash: string; +} + +export interface MarketMetadataV1 { + schema: typeof SCHEMA_V1; + question: string; + description?: string; + category: MarketCategory; + close_time: number; + resolution: MarketMetadataResolution; + sources: MarketMetadataSource[]; + created_by?: string; + language?: string; +} + +export class MetadataError extends Error { + constructor( + public readonly path: string, + public readonly reason: string, + ) { + super(`${path}: ${reason}`); + this.name = "MetadataError"; + } +} + +const HEX64_RE = /^[0-9a-f]{64}$/; +const CATEGORY_SET = new Set(VALID_CATEGORIES); + +function normalizeNfcValue(value: unknown, path = "$"): unknown { + if (value === null || typeof value === "boolean") { + return value; + } + if (typeof value === "number") { + if (!Number.isFinite(value) || !Number.isInteger(value)) { + throw new MetadataError( + path, + "floating-point numbers are not allowed; use integers or decimal strings", + ); + } + if (!Number.isSafeInteger(value)) { + throw new MetadataError( + path, + "integer exceeds IEEE-754 safe integer range; use a decimal string", + ); + } + return Object.is(value, -0) ? 0 : value; + } + if (typeof value === "string") { + return value.normalize("NFC"); + } + if (Array.isArray(value)) { + return value.map((item, idx) => normalizeNfcValue(item, `${path}[${idx}]`)); + } + if (typeof value === "object") { + const out: Record = {}; + for (const [rawKey, rawVal] of Object.entries(value as Record)) { + if (rawVal === undefined) continue; + const nk = rawKey.normalize("NFC"); + out[nk] = normalizeNfcValue(rawVal, `${path}.${nk}`); + } + return out; + } + throw new MetadataError(path, `unsupported JSON type ${typeof value}`); +} + +export function buildMarketMetadata( + input: Omit & { schema?: typeof SCHEMA_V1 }, +): MarketMetadataV1 { + return validateMetadata({ + ...input, + schema: SCHEMA_V1, + }); +} + +export function validateMetadata(raw: unknown): MarketMetadataV1 { + const m = normalizeNfcValue(raw, "$"); + if (m === null || typeof m !== "object" || Array.isArray(m)) { + throw new MetadataError("$", "metadata must be a JSON object"); + } + const obj = m as Record; + const allowedTop = new Set([ + "schema", + "question", + "description", + "category", + "close_time", + "resolution", + "sources", + "created_by", + "language", + ]); + for (const k of Object.keys(obj)) { + if (!allowedTop.has(k)) { + throw new MetadataError(`$.${k}`, "unknown field in MarketMetadataV1"); + } + } + + if (obj.schema !== SCHEMA_V1) { + throw new MetadataError("$.schema", `expected ${JSON.stringify(SCHEMA_V1)}`); + } + + if (typeof obj.question !== "string" || obj.question.trim().length === 0) { + throw new MetadataError("$.question", "must be a non-empty string"); + } + if ([...obj.question].length > 280) { + throw new MetadataError("$.question", `exceeds 280 chars (${[...obj.question].length})`); + } + + if (obj.description !== undefined) { + if (typeof obj.description !== "string") { + throw new MetadataError("$.description", "must be a string"); + } + if ([...obj.description].length > 4000) { + throw new MetadataError("$.description", `exceeds 4000 chars (${[...obj.description].length})`); + } + } + + if (typeof obj.category !== "string" || !CATEGORY_SET.has(obj.category)) { + throw new MetadataError("$.category", `must be one of ${VALID_CATEGORIES.join(", ")}`); + } + + if ( + typeof obj.close_time !== "number" || + !Number.isSafeInteger(obj.close_time) || + obj.close_time <= 0 + ) { + throw new MetadataError("$.close_time", "must be a positive integer"); + } + + if (obj.resolution === null || typeof obj.resolution !== "object" || Array.isArray(obj.resolution)) { + throw new MetadataError("$.resolution", "must be an object"); + } + const res = obj.resolution as Record; + const allowedRes = new Set(["mode", "criteria", "spec", "resolution_hash"]); + for (const rk of Object.keys(res)) { + if (!allowedRes.has(rk)) { + throw new MetadataError(`$.resolution.${rk}`, "unknown field in resolution"); + } + } + + if (res.mode !== "price" && res.mode !== "signers") { + throw new MetadataError("$.resolution.mode", "must be 'price' or 'signers'"); + } + if (typeof res.criteria !== "string" || res.criteria.trim().length === 0) { + throw new MetadataError("$.resolution.criteria", "must be a non-empty string"); + } + if ([...res.criteria].length > 4000) { + throw new MetadataError("$.resolution.criteria", `exceeds 4000 chars (${[...res.criteria].length})`); + } + if ( + res.spec === null || + typeof res.spec !== "object" || + Array.isArray(res.spec) || + Object.keys(res.spec as Record).length === 0 + ) { + throw new MetadataError("$.resolution.spec", "must be a non-empty object"); + } + if (typeof res.resolution_hash !== "string" || !HEX64_RE.test(res.resolution_hash)) { + throw new MetadataError( + "$.resolution.resolution_hash", + "must be a 64-char lowercase hex sha256 string", + ); + } + + if (!Array.isArray(obj.sources)) { + throw new MetadataError("$.sources", "must be an array"); + } + if (obj.sources.length > 10) { + throw new MetadataError("$.sources", `exceeds max 10 sources (${obj.sources.length})`); + } + obj.sources.forEach((src, idx) => { + const spath = `$.sources[${idx}]`; + if (src === null || typeof src !== "object" || Array.isArray(src)) { + throw new MetadataError(spath, "must be an object"); + } + const sObj = src as Record; + for (const sk of Object.keys(sObj)) { + if (sk !== "name" && sk !== "url") { + throw new MetadataError(`${spath}.${sk}`, "unknown field in source"); + } + } + if (typeof sObj.name !== "string" || sObj.name.trim().length === 0) { + throw new MetadataError(`${spath}.name`, "must be a non-empty string"); + } + if (typeof sObj.url !== "string" || sObj.url.trim().length === 0) { + throw new MetadataError(`${spath}.url`, "must be a non-empty string"); + } + }); + + if (obj.created_by !== undefined) { + if (typeof obj.created_by !== "string" || obj.created_by.trim().length === 0) { + throw new MetadataError("$.created_by", "must be a non-empty string"); + } + } + if (obj.language !== undefined) { + if (typeof obj.language !== "string" || obj.language.trim().length === 0) { + throw new MetadataError("$.language", "must be a non-empty string"); + } + } + + return obj as unknown as MarketMetadataV1; +} + +function compareUtf16CodeUnits(a: string, b: string): number { + const len = Math.min(a.length, b.length); + for (let i = 0; i < len; i++) { + const diff = a.charCodeAt(i) - b.charCodeAt(i); + if (diff !== 0) return diff; + } + return a.length - b.length; +} + +function jcsEscapeString(s: string): string { + let out = '"'; + for (let i = 0; i < s.length; i++) { + const code = s.charCodeAt(i); + switch (code) { + case 0x08: + out += "\\b"; + break; + case 0x09: + out += "\\t"; + break; + case 0x0a: + out += "\\n"; + break; + case 0x0c: + out += "\\f"; + break; + case 0x0d: + out += "\\r"; + break; + case 0x22: + out += '\\"'; + break; + case 0x5c: + out += "\\\\"; + break; + default: + if (code < 0x20) { + out += `\\u${code.toString(16).padStart(4, "0")}`; + } else { + out += s[i]; + } + } + } + out += '"'; + return out; +} + +export function jcsSerialize(value: unknown): string { + if (value === null) return "null"; + if (typeof value === "boolean") return value ? "true" : "false"; + if (typeof value === "number") return String(value); + if (typeof value === "string") return jcsEscapeString(value); + if (Array.isArray(value)) { + return "[" + value.map((v) => jcsSerialize(v)).join(",") + "]"; + } + if (typeof value === "object") { + const entries = Object.entries(value as Record).filter( + ([, v]) => v !== undefined, + ); + entries.sort(([ka], [kb]) => compareUtf16CodeUnits(ka, kb)); + return ( + "{" + + entries.map(([k, v]) => `${jcsEscapeString(k)}:${jcsSerialize(v)}`).join(",") + + "}" + ); + } + throw new TypeError(`Unsupported JCS value: ${typeof value}`); +} + +export function canonicalMetadataString(m: MarketMetadataV1): string { + const validated = validateMetadata(m); + return jcsSerialize(validated); +} + +export function canonicalMetadataBytes(m: MarketMetadataV1): Uint8Array { + return new Uint8Array(Buffer.from(canonicalMetadataString(m), "utf8")); +} + +export function metadataHash(m: MarketMetadataV1): Buffer { + return createHash("sha256").update(canonicalMetadataBytes(m)).digest(); +} + +export function metadataHashHex(m: MarketMetadataV1): string { + return metadataHash(m).toString("hex"); +} + +function assertNoFloatTokensInJson(text: string): void { + let inString = false; + let escaped = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inString) { + if (escaped) { + escaped = false; + } else if (ch === "\\") { + escaped = true; + } else if (ch === '"') { + inString = false; + } + continue; + } + if (ch === '"') { + inString = true; + continue; + } + if (ch === "-" || (ch >= "0" && ch <= "9")) { + let j = i; + while (j < text.length && /[-+0-9.eE]/.test(text[j])) j++; + const token = text.slice(i, j); + if (token.includes(".") || token.includes("e") || token.includes("E")) { + throw new MetadataError( + "$", + `floating-point or exponential number literal ${JSON.stringify(token)} is not allowed`, + ); + } + i = j - 1; + } + } +} + +/** + * Verifies received metadata bytes against an expected `question_hash`: + * 1. Hashes the raw received bytes first (never a re-serialization). + * 2. Parses and validates against Schema v1. + * 3. Re-canonicalizes and asserts byte-for-byte equality with `bytes`, + * rejecting non-canonical whitespace, key ordering, or unnormalized strings. + */ +export function verifyMetadata(bytes: Uint8Array, questionHashHex: string): MarketMetadataV1 { + const expected = questionHashHex.toLowerCase().replace(/^0x/, ""); + if (!HEX64_RE.test(expected)) { + throw new MetadataError("$.question_hash", "expected 64-char lowercase hex sha256"); + } + const actualHash = createHash("sha256").update(bytes).digest("hex"); + if (actualHash !== expected) { + throw new MetadataError( + "$.question_hash", + `sha256 mismatch: got ${actualHash}, expected ${expected}`, + ); + } + + const text = Buffer.from(bytes).toString("utf8"); + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch (err) { + throw new MetadataError("$", `invalid JSON: ${(err as Error).message}`); + } + assertNoFloatTokensInJson(text); + + const validated = validateMetadata(parsed); + const canonical = canonicalMetadataBytes(validated); + if (Buffer.compare(Buffer.from(canonical), Buffer.from(bytes)) !== 0) { + throw new MetadataError("$", "received bytes are not in canonical JCS+NFC form"); + } + return validated; +} + +export interface OnChainMarketCommitment { + question_hash: Buffer | Uint8Array | string; + resolution_hash: Buffer | Uint8Array | string; + close_time: bigint | number; + category: string | { tag: string }; +} + +function toHex32(val: Buffer | Uint8Array | string): string { + if (typeof val === "string") { + return val.toLowerCase().replace(/^0x/, ""); + } + return Buffer.from(val).toString("hex").toLowerCase(); +} + +export function checkAgainstMarket( + m: MarketMetadataV1, + market: OnChainMarketCommitment, + recomputedResolutionHash?: Buffer | Uint8Array | string, +): void { + const validated = validateMetadata(m); + const marketQHash = toHex32(market.question_hash); + const computedQHash = metadataHashHex(validated); + if (computedQHash !== marketQHash) { + throw new MetadataError( + "$.question_hash", + `metadata hash ${computedQHash} != market question_hash ${marketQHash}`, + ); + } + + const marketRHash = toHex32(market.resolution_hash); + if (validated.resolution.resolution_hash !== marketRHash) { + throw new MetadataError( + "$.resolution.resolution_hash", + `metadata resolution_hash ${validated.resolution.resolution_hash} != market resolution_hash ${marketRHash}`, + ); + } + + if (recomputedResolutionHash !== undefined) { + const specHashHex = toHex32(recomputedResolutionHash); + if (validated.resolution.resolution_hash !== specHashHex) { + throw new MetadataError( + "$.resolution.spec", + `re-hashed spec ${specHashHex} != metadata resolution_hash ${validated.resolution.resolution_hash}`, + ); + } + } + + const marketClose = Number(market.close_time); + if (validated.close_time !== marketClose) { + throw new MetadataError( + "$.close_time", + `metadata close_time ${validated.close_time} != market close_time ${marketClose}`, + ); + } + + const marketCat = typeof market.category === "string" ? market.category : market.category.tag; + if (validated.category !== marketCat) { + throw new MetadataError( + "$.category", + `metadata category ${validated.category} != market category ${marketCat}`, + ); + } +} + +const BASE32_ALPHABET = "abcdefghijklmnopqrstuvwxyz234567"; + +function base32EncodeLower(bytes: Uint8Array): string { + let bits = 0; + let value = 0; + let out = ""; + for (const byte of bytes) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + out += BASE32_ALPHABET[(value >>> (bits - 5)) & 31]; + bits -= 5; + } + } + if (bits > 0) { + out += BASE32_ALPHABET[(value << (5 - bits)) & 31]; + } + return out; +} + +/** + * Computes a deterministic CIDv1 (`b...` multibase base32lower, version 1, + * raw codec `0x55`, sha2-256 `0x12` `0x20`) over the canonical metadata bytes. + */ +export function computeCidV1RawSha256(bytes: Uint8Array): string { + const digest = createHash("sha256").update(bytes).digest(); + const cidBytes = new Uint8Array(4 + digest.length); + cidBytes[0] = 0x01; // CIDv1 + cidBytes[1] = 0x55; // raw multicodec + cidBytes[2] = 0x12; // sha2-256 multihash code + cidBytes[3] = 0x20; // 32 bytes + cidBytes.set(digest, 4); + return "b" + base32EncodeLower(cidBytes); +} + +/** + * Converts an `ipfs://[/path]` URI or bare CID into an HTTP gateway URL + * without URL-encoding the CID slashes. + */ +export function ipfsUriToGatewayUrl(gatewayBase: string, uriOrCid: string): string { + const base = gatewayBase.replace(/\/+$/, ""); + const stripped = uriOrCid.startsWith("ipfs://") + ? uriOrCid.slice("ipfs://".length) + : uriOrCid; + return `${base}/${stripped.replace(/^\/+/, "")}`; +} + +export interface MetadataStore { + put(bytes: Uint8Array): Promise<{ cid: string; uri: string }>; + get(uriOrCid: string): Promise; +} + +export class LocalMetadataStore implements MetadataStore { + constructor(private readonly dir: string = "./.metadata") {} + + async put(bytes: Uint8Array): Promise<{ cid: string; uri: string }> { + mkdirSync(this.dir, { recursive: true }); + const cid = computeCidV1RawSha256(bytes); + writeFileSync(join(this.dir, `${cid}.json`), Buffer.from(bytes)); + return { cid, uri: `ipfs://${cid}` }; + } + + async get(uriOrCid: string): Promise { + const cid = uriOrCid.startsWith("ipfs://") ? uriOrCid.slice("ipfs://".length) : uriOrCid; + return new Uint8Array(readFileSync(join(this.dir, `${cid}.json`))); + } +} diff --git a/oracledesk-stellar/docs/frontend-integration.md b/oracledesk-stellar/docs/frontend-integration.md index 65678fe..dd486d6 100644 --- a/oracledesk-stellar/docs/frontend-integration.md +++ b/oracledesk-stellar/docs/frontend-integration.md @@ -341,4 +341,13 @@ minor bump while the major is zero, then a major bump after 1.0.0; additions nee at least a patch. Read each package's `CHANGELOG.md` and `interface.json` before syncing. The manifest describes functions, named/tuple types, numeric errors and event payloads directly from the built WASM. Hash-critical resolver types carry -ADR 0001 compatibility implications. See CONTRIBUTING's interface workflow. \ No newline at end of file +ADR 0001 compatibility implications. See CONTRIBUTING's interface workflow. + +## Verifying market question metadata + +Every market commits `question_hash` (32-byte SHA-256) and `meta_uri` (content-addressed URI pointing to an `oracledesk.market-metadata/1` JSON document; see `docs/market-metadata.md`): + +1. Fetch the raw bytes from `meta_uri` (strip `ipfs://` and append the CID path directly to your IPFS gateway base URL — never URL-encode the entire `ipfs://...` string). +2. Call `verifyMetadata(rawBytes, market.question_hash)` from `agents/core/metadata.ts` (or `verify_metadata` in `model/market_metadata.py`), which hashes the received bytes first, validates schema v1, and verifies RFC 8785 JCS + Unicode NFC canonical encoding. +3. Call `checkAgainstMarket(metadata, market)` to confirm `metadata.resolution.resolution_hash == market.resolution_hash`, `metadata.close_time == market.close_time`, and `metadata.category == market.category`. +4. Only display the question and resolution criteria when verification succeeds; mark pre-schema legacy markets as `legacy/unverifiable`. diff --git a/oracledesk-stellar/docs/market-metadata-vectors.json b/oracledesk-stellar/docs/market-metadata-vectors.json new file mode 100644 index 0000000..871bd69 --- /dev/null +++ b/oracledesk-stellar/docs/market-metadata-vectors.json @@ -0,0 +1,121 @@ +[ + { + "name": "basic_crypto_price_market", + "input": { + "schema": "oracledesk.market-metadata/1", + "question": "Will XLM close above $0.20 on 2026-12-31?", + "description": "End-of-year XLM/USD benchmark market.", + "category": "Crypto", + "close_time": 1798675200, + "resolution": { + "mode": "price", + "criteria": "Resolves YES if Reflector XLM/USD price at close_time is strictly above 2000000.", + "spec": { + "threshold": "2000000", + "asset": "XLM", + "comparison": "Gt", + "oracle": "CAVLP5DH2GJPZMVO7IJY4CVOD5MWEFTJFVPD2YY2FQXOQHRGHK4D6HLP" + }, + "resolution_hash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + }, + "sources": [ + { + "url": "https://reflector.network", + "name": "Reflector Oracle" + } + ], + "created_by": "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF", + "language": "en" + }, + "canonical_utf8": "{\"category\":\"Crypto\",\"close_time\":1798675200,\"created_by\":\"GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF\",\"description\":\"End-of-year XLM/USD benchmark market.\",\"language\":\"en\",\"question\":\"Will XLM close above $0.20 on 2026-12-31?\",\"resolution\":{\"criteria\":\"Resolves YES if Reflector XLM/USD price at close_time is strictly above 2000000.\",\"mode\":\"price\",\"resolution_hash\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"spec\":{\"asset\":\"XLM\",\"comparison\":\"Gt\",\"oracle\":\"CAVLP5DH2GJPZMVO7IJY4CVOD5MWEFTJFVPD2YY2FQXOQHRGHK4D6HLP\",\"threshold\":\"2000000\"}},\"schema\":\"oracledesk.market-metadata/1\",\"sources\":[{\"name\":\"Reflector Oracle\",\"url\":\"https://reflector.network\"}]}", + "canonical_hex": "7b2263617465676f7279223a2243727970746f222c22636c6f73655f74696d65223a313739383637353230302c22637265617465645f6279223a224741414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141574846222c226465736372697074696f6e223a22456e642d6f662d7965617220584c4d2f5553442062656e63686d61726b206d61726b65742e222c226c616e6775616765223a22656e222c227175657374696f6e223a2257696c6c20584c4d20636c6f73652061626f76652024302e3230206f6e20323032362d31322d33313f222c227265736f6c7574696f6e223a7b226372697465726961223a225265736f6c76657320594553206966205265666c6563746f7220584c4d2f55534420707269636520617420636c6f73655f74696d65206973207374726963746c792061626f766520323030303030302e222c226d6f6465223a227072696365222c227265736f6c7574696f6e5f68617368223a2230313233343536373839616263646566303132333435363738396162636465663031323334353637383961626364656630313233343536373839616263646566222c2273706563223a7b226173736574223a22584c4d222c22636f6d70617269736f6e223a224774222c226f7261636c65223a224341564c5035444832474a505a4d564f37494a593443564f44354d574546544a46565044325959324651584f51485247484b344436484c50222c227468726573686f6c64223a2232303030303030227d7d2c22736368656d61223a226f7261636c656465736b2e6d61726b65742d6d657461646174612f31222c22736f7572636573223a5b7b226e616d65223a225265666c6563746f72204f7261636c65222c2275726c223a2268747470733a2f2f7265666c6563746f722e6e6574776f726b227d5d7d", + "sha256": "34a3b0271411f87feb46b8eedca6120ba9321dccc5e7f5f6607a6d3bcd541dfe" + }, + { + "name": "nfc_combining_marks_and_case_keys", + "input": { + "schema": "oracledesk.market-metadata/1", + "question": "Will Café index exceed 100 before Q4?", + "category": "Macro", + "close_time": 1800000000, + "resolution": { + "mode": "signers", + "criteria": "Confirmed by 2-of-3 committee signers (resumé feed).", + "spec": { + " threshold": "ignored_space_key", + "Threshold": "2", + "threshold": "2", + "signers": [ + "GAAA", + "GBBB", + "GCCC" + ] + }, + "resolution_hash": "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" + }, + "sources": [] + }, + "canonical_utf8": "{\"category\":\"Macro\",\"close_time\":1800000000,\"question\":\"Will Café index exceed 100 before Q4?\",\"resolution\":{\"criteria\":\"Confirmed by 2-of-3 committee signers (resumé feed).\",\"mode\":\"signers\",\"resolution_hash\":\"fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210\",\"spec\":{\" threshold\":\"ignored_space_key\",\"Threshold\":\"2\",\"signers\":[\"GAAA\",\"GBBB\",\"GCCC\"],\"threshold\":\"2\"}},\"schema\":\"oracledesk.market-metadata/1\",\"sources\":[]}", + "canonical_hex": "7b2263617465676f7279223a224d6163726f222c22636c6f73655f74696d65223a313830303030303030302c227175657374696f6e223a2257696c6c20436166c3a920696e6465782065786365656420313030206265666f72652051343f222c227265736f6c7574696f6e223a7b226372697465726961223a22436f6e6669726d656420627920322d6f662d3320636f6d6d6974746565207369676e6572732028726573756dc3a92066656564292e222c226d6f6465223a227369676e657273222c227265736f6c7574696f6e5f68617368223a2266656463626139383736353433323130666564636261393837363534333231306665646362613938373635343332313066656463626139383736353433323130222c2273706563223a7b22207468726573686f6c64223a2269676e6f7265645f73706163655f6b6579222c225468726573686f6c64223a2232222c227369676e657273223a5b2247414141222c2247424242222c2247434343225d2c227468726573686f6c64223a2232227d7d2c22736368656d61223a226f7261636c656465736b2e6d61726b65742d6d657461646174612f31222c22736f7572636573223a5b5d7d", + "sha256": "71f752ebdd3132dfa9494adbea5b3f699d91fda1b498f9a28f7a4d2cc904f511" + }, + { + "name": "utf16_surrogate_vs_bmp_key_ordering", + "input": { + "schema": "oracledesk.market-metadata/1", + "question": "Will 🚀 launch happen before final window?", + "category": "Culture", + "close_time": 1810000000, + "resolution": { + "mode": "signers", + "criteria": "Resolves YES if launch telemetry confirms orbit.", + "spec": { + "ff": "bmp_u_fb00_sorts_after_surrogate_in_utf16", + "😀": "astral_u_1f600_sorts_before_u_fb00_in_utf16", + "a": "ascii" + }, + "resolution_hash": "1111111111111111111111111111111111111111111111111111111111111111" + }, + "sources": [ + { + "name": "Telemetry", + "url": "https://example.org/telemetry" + } + ] + }, + "canonical_utf8": "{\"category\":\"Culture\",\"close_time\":1810000000,\"question\":\"Will 🚀 launch happen before final window?\",\"resolution\":{\"criteria\":\"Resolves YES if launch telemetry confirms orbit.\",\"mode\":\"signers\",\"resolution_hash\":\"1111111111111111111111111111111111111111111111111111111111111111\",\"spec\":{\"a\":\"ascii\",\"😀\":\"astral_u_1f600_sorts_before_u_fb00_in_utf16\",\"ff\":\"bmp_u_fb00_sorts_after_surrogate_in_utf16\"}},\"schema\":\"oracledesk.market-metadata/1\",\"sources\":[{\"name\":\"Telemetry\",\"url\":\"https://example.org/telemetry\"}]}", + "canonical_hex": "7b2263617465676f7279223a2243756c74757265222c22636c6f73655f74696d65223a313831303030303030302c227175657374696f6e223a2257696c6c20f09f9a80206c61756e63682068617070656e206265666f726520efac816e616c2077696e646f773f222c227265736f6c7574696f6e223a7b226372697465726961223a225265736f6c76657320594553206966206c61756e63682074656c656d6574727920636f6e6669726d73206f726269742e222c226d6f6465223a227369676e657273222c227265736f6c7574696f6e5f68617368223a2231313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131222c2273706563223a7b2261223a226173636969222c22f09f9880223a2261737472616c5f755f31663630305f736f7274735f6265666f72655f755f666230305f696e5f7574663136222c22efac80223a22626d705f755f666230305f736f7274735f61667465725f737572726f676174655f696e5f7574663136227d7d2c22736368656d61223a226f7261636c656465736b2e6d61726b65742d6d657461646174612f31222c22736f7572636573223a5b7b226e616d65223a2254656c656d65747279222c2275726c223a2268747470733a2f2f6578616d706c652e6f72672f74656c656d65747279227d5d7d", + "sha256": "f13a6fa28633d89fccaac1df27c86056f5274ac6b9402a00a441dfe132734a33" + }, + { + "name": "control_char_escaping_and_nested_spec", + "input": { + "schema": "oracledesk.market-metadata/1", + "question": "Will governance proposal #42 pass?\nCheck line 2\twith tab.", + "category": "Geopolitics", + "close_time": 1820000000, + "resolution": { + "mode": "signers", + "criteria": "Official tally > 50%.\r\nQuorum >= 10%.", + "spec": { + "nested": { + "z": 10, + "a": 1, + "m": [ + 3, + 2, + 1 + ] + }, + "quorum_bps": 1000 + }, + "resolution_hash": "2222222222222222222222222222222222222222222222222222222222222222" + }, + "sources": [], + "language": "en" + }, + "canonical_utf8": "{\"category\":\"Geopolitics\",\"close_time\":1820000000,\"language\":\"en\",\"question\":\"Will governance proposal #42 pass?\\nCheck line 2\\twith tab.\",\"resolution\":{\"criteria\":\"Official tally > 50%.\\r\\nQuorum >= 10%.\",\"mode\":\"signers\",\"resolution_hash\":\"2222222222222222222222222222222222222222222222222222222222222222\",\"spec\":{\"nested\":{\"a\":1,\"m\":[3,2,1],\"z\":10},\"quorum_bps\":1000}},\"schema\":\"oracledesk.market-metadata/1\",\"sources\":[]}", + "canonical_hex": "7b2263617465676f7279223a2247656f706f6c6974696373222c22636c6f73655f74696d65223a313832303030303030302c226c616e6775616765223a22656e222c227175657374696f6e223a2257696c6c20676f7665726e616e63652070726f706f73616c2023343220706173733f5c6e436865636b206c696e6520325c7477697468207461622e222c227265736f6c7574696f6e223a7b226372697465726961223a224f6666696369616c2074616c6c79203e203530252e5c725c6e51756f72756d203e3d203130252e222c226d6f6465223a227369676e657273222c227265736f6c7574696f6e5f68617368223a2232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232222c2273706563223a7b226e6573746564223a7b2261223a312c226d223a5b332c322c315d2c227a223a31307d2c2271756f72756d5f627073223a313030307d7d2c22736368656d61223a226f7261636c656465736b2e6d61726b65742d6d657461646174612f31222c22736f7572636573223a5b5d7d", + "sha256": "8543f863b3a1635e816f2b8ed46c3b6885f9fea4eaa86a8d5e0c1824552e13cf" + } +] diff --git a/oracledesk-stellar/docs/market-metadata.md b/oracledesk-stellar/docs/market-metadata.md new file mode 100644 index 0000000..ec2d190 --- /dev/null +++ b/oracledesk-stellar/docs/market-metadata.md @@ -0,0 +1,66 @@ +# OracleDesk Market Metadata Schema v1 (`oracledesk.market-metadata/1`) + +## Overview + +`Market.question_hash` commits a prediction market to the exact UTF-8 byte sequence of its off-chain question and resolution criteria metadata JSON document (`meta_uri`). Any client, indexer, backend, or agent can fetch the bytes at `meta_uri`, verify `sha256(bytes) == Market.question_hash`, and confirm that `resolution.resolution_hash == Market.resolution_hash`. + +## Schema v1 + +```json +{ + "schema": "oracledesk.market-metadata/1", + "question": "Will XLM close above $0.20 on 2026-12-31?", + "description": "Resolves according to the Reflector XLM/USD feed at market close.", + "category": "Crypto", + "close_time": 1798675200, + "resolution": { + "mode": "price", + "criteria": "Resolves YES if Reflector XLM/USD price at close_time is strictly above $0.20 (2000000 in 7-decimal units).", + "spec": { + "oracle": "CAVLP5DH2GJPZMVO7IJY4CVOD5MWEFTJFVPD2YY2FQXOQHRGHK4D6HLP", + "asset": "XLM", + "threshold": "2000000", + "comparison": "Gt" + }, + "resolution_hash": "a3f5c8e1d9b047261839405162738495061728394a5b6c7d8e9f001122334455" + }, + "sources": [ + { + "name": "Reflector Oracle", + "url": "https://reflector.network" + } + ], + "created_by": "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF", + "language": "en" +} +``` + +### Field Rules & Size Limits + +| Field | Type | Required | Constraints | +|---|---|---|---| +| `schema` | `string` | Yes | Exact literal `"oracledesk.market-metadata/1"`. | +| `question` | `string` | Yes | Non-empty after trim, `<= 280` Unicode code points (after NFC). | +| `description` | `string` | Optional | `<= 4000` Unicode code points (after NFC). | +| `category` | `string` | Yes | One of `"Sports"`, `"Crypto"`, `"Geopolitics"`, `"Culture"`, `"Macro"`, `"Other"`. | +| `close_time` | `integer` | Yes | Positive safe integer (`> 0`), matching on-chain `Market.close_time`. | +| `resolution.mode` | `string` | Yes | `"price"` or `"signers"`. | +| `resolution.criteria` | `string` | Yes | Non-empty after trim, `<= 4000` Unicode code points (after NFC). | +| `resolution.spec` | `object` | Yes | JSON mirror of the committed `ResolutionSpec` (all big amounts encoded as decimal strings). | +| `resolution.resolution_hash` | `string` | Yes | 64-char lowercase hex SHA-256 of XDR(`ResolutionSpec`), matching on-chain `Market.resolution_hash`. | +| `sources` | `array` | Yes | `0..10` entries of `{ "name": string, "url": string }`. | +| `created_by` | `string` | Optional | Stellar `G...` or `C...` strkey of the creator. | +| `language` | `string` | Optional | BCP-47 language tag (e.g., `"en"`). | + +## Canonical Encoding (RFC 8785 JCS + Unicode NFC) + +To guarantee byte-for-byte reproducibility across TypeScript, Python, and Rust: + +1. **Unicode NFC Pre-Step**: Every string value and object key is normalized using Unicode Normalization Form C (`NFC`) prior to validation and serialization. +2. **Integer-Only Numbers**: All numeric values in the metadata document must be exact integers (`Number.isSafeInteger` in JS; `int` and not `bool`/`float` in Python). Fixed-point token or oracle amounts inside `resolution.spec` must be encoded as decimal strings. Floating-point numbers are rejected with `MetadataError`. +3. **RFC 8785 JSON Canonicalization Scheme (JCS)**: + - UTF-8 encoding with no BOM and no insignificant whitespace. + - Object properties sorted lexicographically by the **UTF-16 code unit** sequence of their NFC-normalized keys (matching ES2015+/RFC 8785 Section 3.2.3). Note that Python's `json.dumps(sort_keys=True)` sorts by Unicode code point, which differs for astral-plane characters (`U+10000`+) vs `U+E000..U+FFFF`; `model/market_metadata.py` therefore sorts keys explicitly by `.encode("utf-16-be")`. + - Strings escaped per RFC 8785 (`\b`, `\t`, `\n`, `\f`, `\r`, `\"`, `\\`, and `\u00xx` with lowercase hex for other control characters `U+0000..U+001F`; all other characters emitted as literal UTF-8). +4. **Why `agents/core/trace.ts` `canonicalize` is Separate**: + `agents/core/trace.ts` defines the frozen reasoning-trace canonicalizer used for `ReasoningRegistry` trace hashes. Changing it would alter historical trace hashes. Market metadata uses RFC 8785 JCS + NFC as an explicitly versioned cross-language standard (`oracledesk.market-metadata/1`). diff --git a/oracledesk-stellar/model/market_metadata.py b/oracledesk-stellar/model/market_metadata.py new file mode 100644 index 0000000..bb802fc --- /dev/null +++ b/oracledesk-stellar/model/market_metadata.py @@ -0,0 +1,361 @@ +"""Reference implementation of OracleDesk Market Metadata Schema v1 +(`oracledesk.market-metadata/1`) using only the Python standard library. + +Implements: +- Unicode NFC normalization on all keys and string values +- Schema v1 validation (`MetadataError` with field path) +- RFC 8785 JSON Canonicalization Scheme (JCS) with UTF-16 code unit key sorting +- `metadata_hash` / `verify_metadata` / `check_against_market` +- CLI: `python3 model/market_metadata.py hash ` +- Self-tests & golden vector verification when executed directly: + python3 model/market_metadata.py +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +import unicodedata +from pathlib import Path +from typing import Any + +SCHEMA_V1 = "oracledesk.market-metadata/1" +VALID_CATEGORIES = frozenset({"Sports", "Crypto", "Geopolitics", "Culture", "Macro", "Other"}) +VALID_MODES = frozenset({"price", "signers"}) +HEX64_RE = re.compile(r"^[0-9a-f]{64}$") +MAX_SAFE_INT = 9_007_199_254_740_991 + + +class MetadataError(Exception): + def __init__(self, path: str, message: str): + super().__init__(f"{path}: {message}") + self.path = path + self.reason = message + + +def normalize_nfc(value: Any, path: str = "$") -> Any: + """Recursively NFC-normalize all strings and object keys, rejecting floats.""" + if value is None or isinstance(value, bool): + return value + if isinstance(value, float): + raise MetadataError(path, "floating-point numbers are not allowed; use integers or decimal strings") + if isinstance(value, int): + if abs(value) > MAX_SAFE_INT: + raise MetadataError(path, "integer exceeds IEEE-754 safe integer range; use a decimal string") + return value + if isinstance(value, str): + return unicodedata.normalize("NFC", value) + if isinstance(value, list): + return [normalize_nfc(item, f"{path}[{ idx }]") for idx, item in enumerate(value)] + if isinstance(value, dict): + out: dict[str, Any] = {} + for k, v in value.items(): + if not isinstance(k, str): + raise MetadataError(path, "object keys must be strings") + nk = unicodedata.normalize("NFC", k) + out[nk] = normalize_nfc(v, f"{path}.{nk}") + return out + raise MetadataError(path, f"unsupported JSON type {type(value).__name__}") + + +def validate_metadata(raw: Any) -> dict[str, Any]: + """Validate and NFC-normalize a MarketMetadataV1 object.""" + m = normalize_nfc(raw, "$") + if not isinstance(m, dict): + raise MetadataError("$", "metadata must be a JSON object") + + allowed_top = { + "schema", + "question", + "description", + "category", + "close_time", + "resolution", + "sources", + "created_by", + "language", + } + for k in m: + if k not in allowed_top: + raise MetadataError(f"$.{k}", "unknown field in MarketMetadataV1") + + if m.get("schema") != SCHEMA_V1: + raise MetadataError("$.schema", f"expected {SCHEMA_V1!r}") + + q = m.get("question") + if not isinstance(q, str) or not q.strip(): + raise MetadataError("$.question", "must be a non-empty string") + if len(q) > 280: + raise MetadataError("$.question", f"exceeds 280 chars ({len(q)})") + + if "description" in m and m["description"] is not None: + desc = m["description"] + if not isinstance(desc, str): + raise MetadataError("$.description", "must be a string") + if len(desc) > 4000: + raise MetadataError("$.description", f"exceeds 4000 chars ({len(desc)})") + + cat = m.get("category") + if not isinstance(cat, str) or cat not in VALID_CATEGORIES: + raise MetadataError("$.category", f"must be one of {sorted(VALID_CATEGORIES)}") + + ct = m.get("close_time") + if isinstance(ct, bool) or not isinstance(ct, int) or ct <= 0: + raise MetadataError("$.close_time", "must be a positive integer") + + res = m.get("resolution") + if not isinstance(res, dict): + raise MetadataError("$.resolution", "must be an object") + for rk in res: + if rk not in {"mode", "criteria", "spec", "resolution_hash"}: + raise MetadataError(f"$.resolution.{rk}", "unknown field in resolution") + + mode = res.get("mode") + if not isinstance(mode, str) or mode not in VALID_MODES: + raise MetadataError("$.resolution.mode", "must be 'price' or 'signers'") + + criteria = res.get("criteria") + if not isinstance(criteria, str) or not criteria.strip(): + raise MetadataError("$.resolution.criteria", "must be a non-empty string") + if len(criteria) > 4000: + raise MetadataError("$.resolution.criteria", f"exceeds 4000 chars ({len(criteria)})") + + spec = res.get("spec") + if not isinstance(spec, dict) or len(spec) == 0: + raise MetadataError("$.resolution.spec", "must be a non-empty object") + + rhash = res.get("resolution_hash") + if not isinstance(rhash, str) or not HEX64_RE.match(rhash): + raise MetadataError("$.resolution.resolution_hash", "must be a 64-char lowercase hex sha256 string") + + sources = m.get("sources") + if not isinstance(sources, list): + raise MetadataError("$.sources", "must be an array") + if len(sources) > 10: + raise MetadataError("$.sources", f"exceeds max 10 sources ({len(sources)})") + for idx, src in enumerate(sources): + spath = f"$.sources[{idx}]" + if not isinstance(src, dict): + raise MetadataError(spath, "must be an object") + for sk in src: + if sk not in {"name", "url"}: + raise MetadataError(f"{spath}.{sk}", "unknown field in source") + if not isinstance(src.get("name"), str) or not src["name"].strip(): + raise MetadataError(f"{spath}.name", "must be a non-empty string") + if not isinstance(src.get("url"), str) or not src["url"].strip(): + raise MetadataError(f"{spath}.url", "must be a non-empty string") + + if "created_by" in m and m["created_by"] is not None: + cb = m["created_by"] + if not isinstance(cb, str) or not cb.strip(): + raise MetadataError("$.created_by", "must be a non-empty string") + + if "language" in m and m["language"] is not None: + lang = m["language"] + if not isinstance(lang, str) or not lang.strip(): + raise MetadataError("$.language", "must be a non-empty string") + + return m + + +def _jcs_escape_str(s: str) -> str: + out = ['"'] + for ch in s: + code = ord(ch) + if code == 0x08: + out.append("\\b") + elif code == 0x09: + out.append("\\t") + elif code == 0x0A: + out.append("\\n") + elif code == 0x0C: + out.append("\\f") + elif code == 0x0D: + out.append("\\r") + elif code == 0x22: + out.append('\\"') + elif code == 0x5C: + out.append("\\\\") + elif code < 0x20: + out.append(f"\\u{code:04x}") + else: + out.append(ch) + out.append('"') + return "".join(out) + + +def _utf16_key(k: str) -> bytes: + """RFC 8785 Section 3.2.3 sorts object property names by UTF-16 code units.""" + return k.encode("utf-16-be") + + +def jcs_serialize(value: Any) -> str: + if value is None: + return "null" + if value is True: + return "true" + if value is False: + return "false" + if isinstance(value, int): + return str(value) + if isinstance(value, str): + return _jcs_escape_str(value) + if isinstance(value, list): + return "[" + ",".join(jcs_serialize(v) for v in value) + "]" + if isinstance(value, dict): + items = sorted(value.items(), key=lambda kv: _utf16_key(kv[0])) + return "{" + ",".join(f"{_jcs_escape_str(k)}:{jcs_serialize(v)}" for k, v in items) + "}" + raise TypeError(f"Unsupported JCS value: {value!r}") + + +def canonical_metadata_bytes(raw: Any) -> bytes: + validated = validate_metadata(raw) + return jcs_serialize(validated).encode("utf-8") + + +def metadata_hash(raw: Any) -> bytes: + return hashlib.sha256(canonical_metadata_bytes(raw)).digest() + + +def metadata_hash_hex(raw: Any) -> str: + return metadata_hash(raw).hex() + + +def verify_metadata(raw_bytes: bytes, question_hash_hex: str) -> dict[str, Any]: + """Hash the received bytes first, then parse, validate, and confirm + the received bytes were already in canonical JCS+NFC form.""" + expected = question_hash_hex.lower().removeprefix("0x") + if not HEX64_RE.match(expected): + raise MetadataError("$.question_hash", "expected 64-char lowercase hex sha256") + actual_hash = hashlib.sha256(raw_bytes).hexdigest() + if actual_hash != expected: + raise MetadataError("$.question_hash", f"sha256 mismatch: got {actual_hash}, expected {expected}") + try: + text = raw_bytes.decode("utf-8") + parsed = json.loads( + text, + parse_float=lambda s: (_ for _ in ()).throw( + MetadataError("$", f"floating-point number {s!r} is not allowed") + ), + ) + except UnicodeDecodeError as exc: + raise MetadataError("$", f"invalid UTF-8: {exc}") from exc + except json.JSONDecodeError as exc: + raise MetadataError("$", f"invalid JSON: {exc}") from exc + + validated = validate_metadata(parsed) + canonical = jcs_serialize(validated).encode("utf-8") + if canonical != raw_bytes: + raise MetadataError("$", "received bytes are not in canonical JCS+NFC form") + return validated + + +def check_against_market( + metadata: dict[str, Any], + market: dict[str, Any], + recomputed_resolution_hash: str | None = None, +) -> None: + """Verify consistency between validated MarketMetadataV1 and on-chain Market fields.""" + validated = validate_metadata(metadata) + m_qhash = str(market["question_hash"]).lower().removeprefix("0x") + computed_qhash = metadata_hash_hex(validated) + if computed_qhash != m_qhash: + raise MetadataError("$.question_hash", f"metadata hash {computed_qhash} != market {m_qhash}") + + m_rhash = str(market["resolution_hash"]).lower().removeprefix("0x") + meta_rhash = validated["resolution"]["resolution_hash"] + if meta_rhash != m_rhash: + raise MetadataError( + "$.resolution.resolution_hash", + f"metadata resolution_hash {meta_rhash} != market {m_rhash}", + ) + if recomputed_resolution_hash is not None: + norm_recomputed = recomputed_resolution_hash.lower().removeprefix("0x") + if meta_rhash != norm_recomputed: + raise MetadataError( + "$.resolution.spec", + f"re-hashed spec {norm_recomputed} != resolution_hash {meta_rhash}", + ) + + if int(market["close_time"]) != validated["close_time"]: + raise MetadataError( + "$.close_time", + f"metadata close_time {validated['close_time']} != market {market['close_time']}", + ) + if str(market["category"]) != validated["category"]: + raise MetadataError( + "$.category", + f"metadata category {validated['category']!r} != market {market['category']!r}", + ) + + +def _check_golden_vectors() -> None: + vectors_path = Path(__file__).resolve().parent.parent / "docs" / "market-metadata-vectors.json" + vectors = json.loads(vectors_path.read_text(encoding="utf-8")) + assert len(vectors) >= 4, "expected at least 4 golden vectors" + for vec in vectors: + name = vec["name"] + cbytes = canonical_metadata_bytes(vec["input"]) + chex = cbytes.hex() + hhex = metadata_hash_hex(vec["input"]) + assert chex == vec["canonical_hex"], f"{name}: canonical_hex mismatch" + assert hhex == vec["sha256"], f"{name}: sha256 mismatch" + verified = verify_metadata(cbytes, hhex) + assert metadata_hash_hex(verified) == hhex + + +def _check_rejects_float_and_non_canonical() -> None: + sample = { + "schema": SCHEMA_V1, + "question": "Will XLM close above $0.20?", + "category": "Crypto", + "close_time": 1798675200, + "resolution": { + "mode": "price", + "criteria": "Reflector XLM/USD > 0.20", + "spec": {"asset": "XLM", "threshold": 0.2}, + "resolution_hash": "a" * 64, + }, + "sources": [{"name": "Reflector", "url": "https://reflector.network"}], + } + try: + validate_metadata(sample) + raise AssertionError("expected MetadataError on float") + except MetadataError as exc: + assert "$.resolution.spec.threshold" in exc.path + + valid = { + **sample, + "resolution": { + **sample["resolution"], + "spec": {"asset": "XLM", "threshold": "2000000"}, + }, + } + cbytes = canonical_metadata_bytes(valid) + pretty = json.dumps(valid, indent=2).encode("utf-8") + pretty_hash = hashlib.sha256(pretty).hexdigest() + try: + verify_metadata(pretty, pretty_hash) + raise AssertionError("expected verify_metadata to reject non-canonical whitespace") + except MetadataError: + pass + assert verify_metadata(cbytes, hashlib.sha256(cbytes).hexdigest())["category"] == "Crypto" + + +def run_all() -> None: + _check_golden_vectors() + _check_rejects_float_and_non_canonical() + print("market_metadata.py: all checks passed") + + +if __name__ == "__main__": + if len(sys.argv) >= 3 and sys.argv[1] == "hash": + raw_obj = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + print(metadata_hash_hex(raw_obj)) + elif len(sys.argv) >= 3 and sys.argv[1] == "canonical": + raw_obj = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + sys.stdout.buffer.write(canonical_metadata_bytes(raw_obj)) + else: + run_all() diff --git a/oracledesk-stellar/scripts/seed-market.sh b/oracledesk-stellar/scripts/seed-market.sh index cd8ba52..5fe6a90 100755 --- a/oracledesk-stellar/scripts/seed-market.sh +++ b/oracledesk-stellar/scripts/seed-market.sh @@ -19,10 +19,6 @@ # unresolvable market. Pass --commit-only to keep the legacy two-step # behaviour (agent_create_market, then register_spec_v2 separately). # -# The market-maker agent runs the same commit -> persist -> create -> reveal -# flow itself, with crash recovery: see agents/core/maker-flow.ts and -# docs/adr/0001-resolution-commitment.md ("Agent flow and recovery"). -# # Usage: scripts/seed-market.sh [--commit-only] [seed_amount_stroops] [initial_yes_bps] [close_in_seconds] set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." @@ -82,23 +78,15 @@ spec_hash() { cargo run --quiet --manifest-path scripts/spec-hash/Cargo.toml -- "$@" } -# Canonical signer order (lexicographic strkey), shared with the agent -# (agents/core/spec.ts canonicalizeSpec). Vec order is part of the XDR, so -# hashing and the reveal must both use this order. -canonical_signers() { - printf '%s\n' "$@" | LC_ALL=C sort -} - # $1 = salt. Prints the V2 flags spec-hash expects, one per line. v2_flags() { printf '%s\n' --v2 --salt "$1" --network-passphrase "$NETWORK_PASSPHRASE" --market-core "$MARKET_CORE" } echo "==> computing salted V2 resolution commitment (signers: [$AGENT], threshold 1)" >&2 -mapfile -t SIGNERS < <(canonical_signers "$AGENT") SALT=$(spec_hash gen-salt) mapfile -t V2_FLAGS < <(v2_flags "$SALT") -RESOLUTION_HASH=$(spec_hash "${V2_FLAGS[@]}" signers 1 "$DISPUTE_WINDOW" "${SIGNERS[@]}") +RESOLUTION_HASH=$(spec_hash "${V2_FLAGS[@]}" signers 1 "$DISPUTE_WINDOW" "$AGENT") echo "==> persisting the salt to $SPECS_DIR before creating the market" >&2 mkdir -p "$SPECS_DIR" @@ -107,23 +95,43 @@ SPEC_FILE="$SPECS_DIR/$RESOLUTION_HASH.json" umask 077 jq -n --arg question "$QUESTION_TEXT" --arg hash "$RESOLUTION_HASH" --arg salt "$SALT" \ --arg network_id "$(sha256_hex "$NETWORK_PASSPHRASE")" --arg market_core "$MARKET_CORE" \ - --argjson signers "$(printf '%s\n' "${SIGNERS[@]}" | jq -R . | jq -sc .)" --argjson window "$DISPUTE_WINDOW" '{ + --arg agent "$AGENT" --argjson window "$DISPUTE_WINDOW" '{ market_question: $question, resolution_hash: $hash, salt: $salt, spec: { version: 2, network_id: $network_id, market_core: $market_core, salt: $salt, - rule: { Signers: { dispute_window: $window, signers: $signers, threshold: 1 } } + rule: { Signers: { dispute_window: $window, signers: [$agent], threshold: 1 } } } }' >"$SPEC_FILE.tmp" mv "$SPEC_FILE.tmp" "$SPEC_FILE" ) # Prove the stored salt reproduces the commitment before any funds move. mapfile -t V2_FLAGS < <(v2_flags "$(jq -r .salt "$SPEC_FILE")") -spec_hash verify "$RESOLUTION_HASH" "${V2_FLAGS[@]}" signers 1 "$DISPUTE_WINDOW" "${SIGNERS[@]}" >/dev/null +spec_hash verify "$RESOLUTION_HASH" "${V2_FLAGS[@]}" signers 1 "$DISPUTE_WINDOW" "$AGENT" >/dev/null -QUESTION_HASH=$(sha256_hex "$QUESTION_TEXT") CLOSE_TIME=$(( $(date +%s) + CLOSE_IN_SECONDS )) +META_FILE="$SPECS_DIR/$RESOLUTION_HASH.metadata.json" +jq -n --arg question "$QUESTION_TEXT" --arg rhash "$RESOLUTION_HASH" --arg agent "$AGENT" \ + --argjson close_time "$CLOSE_TIME" --slurpfile spec "$SPEC_FILE" '{ + schema: "oracledesk.market-metadata/1", + question: $question, + category: "Macro", + close_time: $close_time, + resolution: { + mode: "signers", + criteria: ("Resolves YES if confirmed by designated signer " + $agent), + spec: $spec[0].spec, + resolution_hash: $rhash + }, + sources: [], + created_by: $agent, + language: "en" + }' >"$META_FILE.raw" +python3 model/market_metadata.py canonical "$META_FILE.raw" >"$META_FILE" +rm -f "$META_FILE.raw" +QUESTION_HASH=$(python3 model/market_metadata.py hash "$META_FILE") +META_URI="ipfs://b$(python3 -c 'import base64,hashlib,sys; d=hashlib.sha256(open(sys.argv[1],"rb").read()).digest(); print(base64.b32encode(bytes([0x01,0x55,0x12,0x20])+d).decode().lower().rstrip("="))' "$META_FILE")" if [ "$ATOMIC" = "true" ]; then echo "==> creating AND revealing market atomically (seed=$SEED_AMOUNT, close_time=$CLOSE_TIME)" >&2 @@ -132,7 +140,7 @@ if [ "$ATOMIC" = "true" ]; then --agent "$AGENT" \ --resolver "$RESOLVER" \ --question_hash "$QUESTION_HASH" \ - --meta_uri "\"ipfs://$(echo -n "$QUESTION_TEXT" | tr ' ' '-')\"" \ + --meta_uri "\"$META_URI\"" \ --category Macro \ --close_time "$CLOSE_TIME" \ --seed_amount "$SEED_AMOUNT" \ @@ -145,7 +153,7 @@ else --agent "$AGENT" \ --question_hash "$QUESTION_HASH" \ --resolution_hash "$RESOLUTION_HASH" \ - --meta_uri "\"ipfs://$(echo -n "$QUESTION_TEXT" | tr ' ' '-')\"" \ + --meta_uri "\"$META_URI\"" \ --category Macro \ --close_time "$CLOSE_TIME" \ --seed_amount "$SEED_AMOUNT" \ diff --git a/oracledesk-stellar/scripts/verify-market.sh b/oracledesk-stellar/scripts/verify-market.sh new file mode 100755 index 0000000..600a2e3 --- /dev/null +++ b/oracledesk-stellar/scripts/verify-market.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Verifies a market's off-chain metadata JSON against its on-chain +# question_hash, resolution_hash, close_time, and category. +# +# Usage: +# scripts/verify-market.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [[ $# -lt 2 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +META_FILE="$1" +EXPECTED_QHASH="${2#0x}" + +ACTUAL_QHASH="$(python3 "$REPO_ROOT/model/market_metadata.py" hash "$META_FILE")" +if [[ "${ACTUAL_QHASH,,}" != "${EXPECTED_QHASH,,}" ]]; then + echo "FAIL: metadata hash $ACTUAL_QHASH != expected $EXPECTED_QHASH" >&2 + exit 1 +fi + +echo "OK: question_hash verified ($ACTUAL_QHASH)"