From 415aef6811794dcccdf449d780f8012ce805757f Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 20:46:05 +0200 Subject: [PATCH 01/11] test(gents-cloud): end-to-end suite for the gents-cloud mechanics on Go Vera --- CLAUDE.md | 4 +- Cargo.lock | 1 + Cargo.toml | 5 + backbone.toml | 17 +- crates/defra-harness/src/cluster/builder.rs | 2 + crates/defra-harness/src/node/go_node.rs | 1 + crates/defra-harness/src/node/mod.rs | 14 + crates/defra-harness/src/node/rust_node.rs | 4 + crates/orbis-harness/src/cli/did.rs | 31 + crates/orbis-harness/src/cli/events.rs | 72 +- crates/orbis-harness/src/cli/mod.rs | 2 +- crates/orbis-harness/src/cli/orbis.rs | 123 ++- crates/orbis-harness/src/cli/sourcehub.rs | 374 +++++-- crates/orbis-harness/src/cli/types.rs | 11 +- crates/orbis-harness/src/defradb/identity.rs | 129 ++- crates/orbis-harness/src/fixture.rs | 6 +- crates/orbis-harness/src/ring/builder.rs | 10 +- crates/orbis-harness/src/ring/node.rs | 8 + .../orbis-harness/tests/secret_lifecycle.rs | 4 +- .../tests/three_component_smoke.rs | 5 +- crates/sourcehub-harness/Cargo.toml | 2 +- crates/sourcehub-harness/src/genesis.rs | 10 +- crates/sourcehub-harness/src/identity.rs | 18 +- crates/sourcehub-harness/src/lib.rs | 9 +- crates/sourcehub-harness/src/node.rs | 16 +- crates/test-infra/src/binary.rs | 69 +- crates/test-infra/src/manifest.rs | 3 + tests/full_stack.rs | 3 + tests/gents_cloud/fixture.rs | 991 ++++++++++++++++++ tests/gents_cloud/identity.rs | 336 ++++++ tests/gents_cloud/main.rs | 74 ++ tests/gents_cloud/p2p.rs | 163 +++ tests/gents_cloud/recovery.rs | 83 ++ tests/gents_cloud/revocation.rs | 452 ++++++++ tests/gents_cloud/scale.rs | 275 +++++ tests/gents_cloud/write_path.rs | 365 +++++++ tests/support/full_stack.rs | 47 +- 37 files changed, 3554 insertions(+), 185 deletions(-) create mode 100644 tests/gents_cloud/fixture.rs create mode 100644 tests/gents_cloud/identity.rs create mode 100644 tests/gents_cloud/main.rs create mode 100644 tests/gents_cloud/p2p.rs create mode 100644 tests/gents_cloud/recovery.rs create mode 100644 tests/gents_cloud/revocation.rs create mode 100644 tests/gents_cloud/scale.rs create mode 100644 tests/gents_cloud/write_path.rs diff --git a/CLAUDE.md b/CLAUDE.md index e02bfc9..aeba2c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,7 +46,7 @@ See `docs/architecture.md` for the full security architecture. backbone/ ├── crates/ │ ├── test-infra/ # Shared primitives (ManagedProcess, ports, log tracking, run dirs) -│ ├── sourcehub-harness/ # Go sourcehubd manager (legacy, being replaced by hub-harness) +│ ├── sourcehub-harness/ # Vera (Go verad) devnet manager: the trust plane for gents-cloud tests │ ├── defra-harness/ # DefraDB node manager + CLI client + test fixtures │ ├── hub-harness/ # Hub.rs node manager + cluster builder + observability │ └── orbis-harness/ # Orbis ring builder + DKG fixtures + event subscriptions @@ -84,7 +84,7 @@ cargo fmt --all # Format ## Running the canonical test ```bash -# Requires sourcehubd, defra, and orbis-node binaries on PATH +# Requires hubd, defra-iroh, and orbis-node binaries on PATH (verad for tests/gents_cloud) cargo test --test full_stack -- --ignored --nocapture ``` diff --git a/Cargo.lock b/Cargo.lock index 5c4ef72..dc635c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -909,6 +909,7 @@ dependencies = [ "alloy-signer", "alloy-signer-local", "alloy-sol-types", + "blst", "bs58", "defra-harness", "eyre", diff --git a/Cargo.toml b/Cargo.toml index 3cc4e18..02ef455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,10 @@ path = "tests/full_stack.rs" name = "hub_light_client" path = "tests/hub_light_client.rs" +[[test]] +name = "gents_cloud" +path = "tests/gents_cloud/main.rs" + [dev-dependencies] orbis-harness = { path = "crates/orbis-harness" } defra-harness = { path = "crates/defra-harness" } @@ -68,6 +72,7 @@ hex.workspace = true bs58 = "0.5" eyre.workspace = true sha2 = "0.10" +blst = "0.3" tracing.workspace = true tracing-subscriber = { version = "0.3", features = ["env-filter"] } alloy-primitives.workspace = true diff --git a/backbone.toml b/backbone.toml index 98c1669..a93476f 100644 --- a/backbone.toml +++ b/backbone.toml @@ -3,23 +3,28 @@ # Single source of truth for binary dependency versions. The CI script # ensure-binaries.sh reads refs from this file. To use a different # version, edit the ref here — branches, tags, and commit SHAs all work. +# +# defra and orbis-node are pinned to their Vera-compatibility branches until +# those land on main: Vera (github.com/sourcenetwork/vera, formerly SourceHub) +# renamed its protos to `vera.*` and its bech32 prefix to `vera`, and the +# clients on main still speak `sourcehub.*` / `source`. [components.defra] prefix = "DEFRA" repo = "https://github.com/sourcenetwork/defradb.rs" -ref = "main" +ref = "vclq/vera-compat" cargo_package = "cli" [components.orbis-node] prefix = "ORBIS" repo = "https://github.com/sourcenetwork/orbis-rs" -ref = "jack/integration-testing" +ref = "vclq/vera-compat" cargo_package = "orbis-node" [components.cli-tool] prefix = "ORBIS_CLI" repo = "https://github.com/sourcenetwork/orbis-rs" -ref = "jack/integration-testing" +ref = "vclq/vera-compat" cargo_package = "cli-tool" [components.hubd] @@ -27,3 +32,9 @@ prefix = "HUBD" repo = "https://github.com/sourcenetwork/hub.rs" ref = "main" cargo_package = "hubd" + +[components.verad] +prefix = "VERA" +repo = "https://github.com/sourcenetwork/vera" +ref = "main" +go_package = "./cmd/verad" diff --git a/crates/defra-harness/src/cluster/builder.rs b/crates/defra-harness/src/cluster/builder.rs index 108dd3c..9720a61 100644 --- a/crates/defra-harness/src/cluster/builder.rs +++ b/crates/defra-harness/src/cluster/builder.rs @@ -488,6 +488,7 @@ impl TestClusterBuilder { acp_request_timeout: self.acp_request_timeout, acp_receipt_timeout: self.acp_receipt_timeout, extra_args: self.extra_rust_args.clone(), + extra_envs: Vec::new(), }; let mut attempt = 1; @@ -588,6 +589,7 @@ impl TestClusterBuilder { acp_request_timeout: self.acp_request_timeout, acp_receipt_timeout: self.acp_receipt_timeout, extra_args: Vec::new(), + extra_envs: Vec::new(), }; let mut attempt = 1; diff --git a/crates/defra-harness/src/node/go_node.rs b/crates/defra-harness/src/node/go_node.rs index 502eca2..1abe314 100644 --- a/crates/defra-harness/src/node/go_node.rs +++ b/crates/defra-harness/src/node/go_node.rs @@ -164,6 +164,7 @@ impl DefraNode for GoNode { } args.extend(config.extra_args.iter().cloned()); + envs.extend(config.extra_envs.iter().cloned()); (self.binary_path.clone(), args, envs) } diff --git a/crates/defra-harness/src/node/mod.rs b/crates/defra-harness/src/node/mod.rs index 7c68918..371d2a0 100644 --- a/crates/defra-harness/src/node/mod.rs +++ b/crates/defra-harness/src/node/mod.rs @@ -181,6 +181,13 @@ pub struct OrbisSignerConfig { pub ring_id: String, /// Derivation label (e.g. `"x-archive"`) for derived key signing. pub derivation: String, + /// Hex ed25519 private key (64 bytes) the node authenticates to the ring + /// with, when it differs from `--identity`. + /// + /// The ring accepts EdDSA bearer tokens only, while a node whose document + /// ACP is SourceHub/Vera must hold a secp256k1 identity to sign chain + /// transactions. Set this to keep both. + pub service_identity: Option, } /// Configuration for a single DefraDB node. @@ -217,6 +224,12 @@ pub struct NodeConfig { pub acp_circuit_breaker_reset_timeout: Option, pub acp_request_timeout: Option, pub acp_receipt_timeout: Option, + /// Extra environment variables for the node process, applied after the + /// managed ones. The escape hatch for process-global switches a node reads + /// from the environment rather than a flag -- notably + /// `DEFRA_ALLOW_NON_GO_VERIFIABLE_SIGNING`, which gates emitting BLS + /// (ring-signed) blocks that Go peers cannot verify. + pub extra_envs: Vec<(String, String)>, /// Raw CLI flags appended after every managed flag, so they win under /// clap's last-one-wins parsing. The escape hatch for options the builder /// has no typed method for -- notably enforcement tests, which must set an @@ -258,6 +271,7 @@ impl NodeConfig { acp_circuit_breaker_reset_timeout: None, acp_request_timeout: None, acp_receipt_timeout: None, + extra_envs: Vec::new(), extra_args: Vec::new(), } } diff --git a/crates/defra-harness/src/node/rust_node.rs b/crates/defra-harness/src/node/rust_node.rs index 4b95bcf..5cb3eb0 100644 --- a/crates/defra-harness/src/node/rust_node.rs +++ b/crates/defra-harness/src/node/rust_node.rs @@ -135,6 +135,9 @@ impl DefraNode for RustNode { "--signer-orbis-derivation".into(), signer.derivation.clone(), ]); + if let Some(ref service_identity) = signer.service_identity { + args.extend(["--signer-orbis-identity".into(), service_identity.clone()]); + } } else if !config.signing_enabled { args.push("--no-signing".to_string()); } @@ -220,6 +223,7 @@ impl DefraNode for RustNode { } args.extend(config.extra_args.iter().cloned()); + envs.extend(config.extra_envs.iter().cloned()); (self.binary_path.clone(), args, envs) } diff --git a/crates/orbis-harness/src/cli/did.rs b/crates/orbis-harness/src/cli/did.rs index 84821c0..a1c975f 100644 --- a/crates/orbis-harness/src/cli/did.rs +++ b/crates/orbis-harness/src/cli/did.rs @@ -6,6 +6,25 @@ /// /// We use the `ed25519-dalek` approach via raw bytes: generate the public key /// from the seed, then encode as did:key with multicodec + base58btc. +/// Build the 64-byte ed25519 private key DefraDB expects for `--identity` or +/// `--signer-orbis-identity`, from a 32-byte seed. +/// +/// DefraDB stores an ed25519 private key as seed followed by public key (Go +/// parity) and picks the key type by length, so a 128-character hex string is +/// an ed25519 identity and a 64-character one is secp256k1. The resulting DID +/// is the same one [`signer_did_for_pk`] derives from the seed. +pub fn ed25519_identity_hex(seed_hex: &str) -> String { + let seed_bytes = hex::decode(seed_hex).expect("seed must be valid hex"); + let signing_key = ed25519_dalek::SigningKey::from_bytes( + &seed_bytes[..32] + .try_into() + .expect("seed must be at least 32 bytes"), + ); + let mut key = signing_key.to_bytes().to_vec(); + key.extend_from_slice(&signing_key.verifying_key().to_bytes()); + hex::encode(key) +} + pub fn signer_did_for_pk(private_key_hex: &str) -> String { let seed_bytes = hex::decode(private_key_hex).expect("signer_did_pk must be valid hex"); @@ -36,6 +55,18 @@ mod tests { assert!(did.starts_with("did:key:z"), "got: {}", did); } + #[test] + fn ed25519_identity_hex_is_seed_then_public_key() { + let seed = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let identity = ed25519_identity_hex(seed); + assert_eq!(identity.len(), 128, "64 bytes as hex"); + assert!( + identity.starts_with(seed), + "the seed is the first half: {}", + identity + ); + } + #[test] fn signer_did_deterministic() { let key_hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; diff --git a/crates/orbis-harness/src/cli/events.rs b/crates/orbis-harness/src/cli/events.rs index 508850e..bb7dc37 100644 --- a/crates/orbis-harness/src/cli/events.rs +++ b/crates/orbis-harness/src/cli/events.rs @@ -86,34 +86,60 @@ impl BulletinEventSubscription { } } +/// Typed event emitted by Vera's bulletin keeper on `MsgCreatePost` +/// (`proto/vera/bulletin/events.proto`). CometBFT flattens typed events into +/// `.` keys whose values are JSON-encoded, so a string +/// attribute arrives wrapped in quotes. +const POST_CREATED_EVENT: &str = "vera.bulletin.EventPostCreated"; + fn extract_bulletin_post_event( msg: &serde_json::Value, _session_id: &str, ) -> Option { // CometBFT event structure: - // { "result": { "events": { "bulletin_post.post_id": ["..."], ... } } } + // { "result": { "events": { "vera.bulletin.EventPostCreated.post_id": ["\"...\""], ... } } } let events = msg.pointer("/result/events")?; - // Check for bulletin post events - let post_ids = events - .get("bulletin_post.post_id") - .or_else(|| events.get("sourcehub.bulletin.v1beta1.EventBulletinPost.post_id")) - .and_then(|v| v.as_array())?; - - let namespaces = events - .get("bulletin_post.namespace") - .or_else(|| events.get("sourcehub.bulletin.v1beta1.EventBulletinPost.namespace")) - .and_then(|v| v.as_array()); - - let post_id = post_ids.first()?.as_str()?; - let namespace = namespaces - .and_then(|ns| ns.first()) - .and_then(|v| v.as_str()) - .unwrap_or("orbis") - .to_string(); - - Some(BulletinPostEvent { - post_id: post_id.to_string(), - namespace, - }) + let post_id = first_event_attr(events, POST_CREATED_EVENT, "post_id")?; + let namespace = first_event_attr(events, POST_CREATED_EVENT, "namespace_id") + .unwrap_or_else(|| "orbis".to_string()); + + Some(BulletinPostEvent { post_id, namespace }) +} + +/// First value of `.`, with the typed-event JSON quoting removed. +fn first_event_attr(events: &serde_json::Value, event: &str, field: &str) -> Option { + let raw = events + .get(format!("{event}.{field}")) + .and_then(|v| v.as_array()) + .and_then(|values| values.first()) + .and_then(|v| v.as_str())?; + let unquoted = serde_json::from_str::(raw).unwrap_or_else(|_| raw.to_string()); + Some(unquoted) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_typed_post_created_event_with_json_quoted_values() { + let msg = serde_json::json!({ + "result": { + "events": { + "vera.bulletin.EventPostCreated.post_id": ["\"abc123\""], + "vera.bulletin.EventPostCreated.namespace_id": ["\"orbis\""] + } + } + }); + let event = extract_bulletin_post_event(&msg, "session").expect("event"); + assert_eq!(event.post_id, "abc123"); + assert_eq!(event.namespace, "orbis"); + } + + #[test] + fn ignores_messages_without_post_created_event() { + let msg = serde_json::json!({"result": {"events": {"tx.height": ["1"]}}}); + assert!(extract_bulletin_post_event(&msg, "session").is_none()); + } } diff --git a/crates/orbis-harness/src/cli/mod.rs b/crates/orbis-harness/src/cli/mod.rs index 1fd9e8a..f620a8c 100644 --- a/crates/orbis-harness/src/cli/mod.rs +++ b/crates/orbis-harness/src/cli/mod.rs @@ -4,7 +4,7 @@ mod orbis; mod sourcehub; pub mod types; -pub use did::signer_did_for_pk; +pub use did::{ed25519_identity_hex, signer_did_for_pk}; pub use events::BulletinEventSubscription; pub use orbis::OrbisCliClient; pub use sourcehub::SourceHubCliClient; diff --git a/crates/orbis-harness/src/cli/orbis.rs b/crates/orbis-harness/src/cli/orbis.rs index 22ce3ed..7e7f110 100644 --- a/crates/orbis-harness/src/cli/orbis.rs +++ b/crates/orbis-harness/src/cli/orbis.rs @@ -69,9 +69,14 @@ impl OrbisCliClient { } /// Run a command with `--output json` and deserialize the result. + /// + /// Some subcommands print a human-readable heading before the JSON body, + /// so parsing starts at the first `{` or `[`. fn parse(&self, args: &[&str]) -> Result { let stdout = self.exec_json(args)?; - serde_json::from_str(&stdout).map_err(|e| { + let json = json_body(&stdout) + .ok_or_else(|| eyre!("no JSON body in cli-tool output: stdout={}", stdout))?; + serde_json::from_str(json).map_err(|e| { eyre!( "failed to parse cli-tool JSON output: {}: stdout={}", e, @@ -205,8 +210,11 @@ impl OrbisCliClient { signer_did_pk: Option<&str>, acp: Option<&SignAcpFields>, ) -> Result { + // `utility-sign` is the UtilityService pathway that takes a ring id and + // an optional ACP tuple directly. The `sign` subcommand is the separate + // KeyDerivation pathway and takes a derivation id instead. let mut args = vec![ - "sign", + "utility-sign", "--endpoint", endpoint, "--ring-id", @@ -223,13 +231,13 @@ impl OrbisCliClient { args.push(pk); } if let Some(acp) = acp { - args.push("--acp-policy-id"); + args.push("--policy-id"); args.push(&acp.policy_id); - args.push("--acp-resource"); + args.push("--resource"); args.push(&acp.resource); - args.push("--acp-object-id"); + args.push("--object-id"); args.push(&acp.object_id); - args.push("--acp-permission"); + args.push("--permission"); args.push(&acp.permission); } self.parse(&args) @@ -310,7 +318,8 @@ impl OrbisCliClient { if with_proof { args.push("--with-proof"); } - self.parse(&args) + let stdout = self.exec(&args)?; + parse_store_secret_result(&stdout) } #[allow(clippy::too_many_arguments)] @@ -353,8 +362,104 @@ impl OrbisCliClient { .map_err(|e| eyre!("failed to decode PRE result hex: {}", e)) } + /// Generate a PRE reader keypair, returning `(secret_key_hex, public_key_hex)`. + /// + /// `generate-reader-key` prints a labelled text block and ignores + /// `--output json`, so the hex values are read from the lines following + /// their labels rather than parsed as JSON. pub fn generate_reader_key(&self) -> Result<(String, String)> { - let result: ReaderKeyResult = self.parse(&["generate-reader-key"])?; - Ok((result.secret_key, result.public_key)) + let stdout = self.exec(&["generate-reader-key"])?; + let secret_key = value_after_label(&stdout, "Reader Secret Key") + .ok_or_else(|| eyre!("no reader secret key in output: {}", stdout))?; + let public_key = value_after_label(&stdout, "Reader Public Key") + .ok_or_else(|| eyre!("no reader public key in output: {}", stdout))?; + Ok((secret_key, public_key)) + } +} + +/// Parse the labelled report `store-prepared-secret` prints. +/// +/// The command reports `Status`, `Message`, `Object ID`, `Ring ID` and +/// `signature` as ` Label: value` lines under a heading, and ignores +/// `--output json`. +fn parse_store_secret_result(output: &str) -> Result { + let field = |label: &str| -> Option { + output.lines().find_map(|line| { + let (key, value) = line.split_once(':')?; + (key.trim() == label).then(|| value.trim().to_string()) + }) + }; + let missing = |label: &str| eyre!("no `{}` in store-secret output: {}", label, output); + Ok(StoreSecretResult { + status: field("Status").ok_or_else(|| missing("Status"))?, + message: field("Message").ok_or_else(|| missing("Message"))?, + object_id: field("Object ID").ok_or_else(|| missing("Object ID"))?, + ring_id: field("Ring ID").ok_or_else(|| missing("Ring ID"))?, + signature: field("signature").ok_or_else(|| missing("signature"))?, + }) +} + +/// The JSON body of a cli-tool response, skipping any human-readable heading. +fn json_body(output: &str) -> Option<&str> { + let start = output.find(['{', '['])?; + Some(output[start..].trim()) +} + +/// First non-empty line after the line containing `label`. +fn value_after_label(output: &str, label: &str) -> Option { + let mut lines = output.lines().skip_while(|line| !line.contains(label)); + lines.next()?; + lines + .map(str::trim) + .find(|line| !line.is_empty()) + .map(str::to_string) +} + +#[cfg(test)] +mod tests { + use super::value_after_label; + + #[test] + fn reads_the_hex_under_a_label() { + let output = "Generated Reader Keypair:\n====\nReader Secret Key (--reader-sk):\nabc123\n\nReader Public Key (--reader-pk):\ndef456"; + assert_eq!( + value_after_label(output, "Reader Secret Key").as_deref(), + Some("abc123") + ); + assert_eq!( + value_after_label(output, "Reader Public Key").as_deref(), + Some("def456") + ); + } + + #[test] + fn parses_the_store_secret_report() { + let output = "StoreSecret Result:\n====\n Status: success\n Message: Secret stored successfully\n Object ID: abc\n Ring ID: ring1\n signature: sig1\n enc_cmt: cmt"; + let result = super::parse_store_secret_result(output).expect("parse"); + assert_eq!(result.status, "success"); + assert_eq!(result.object_id, "abc"); + assert_eq!(result.ring_id, "ring1"); + assert_eq!(result.signature, "sig1"); + } + + #[test] + fn store_secret_report_missing_a_field_is_an_error() { + assert!(super::parse_store_secret_result("Status: success").is_err()); + } + + #[test] + fn json_body_skips_a_heading() { + let output = "Prepared Secret (save this):\n====\n{\n \"a\": 1\n}"; + assert_eq!(super::json_body(output), Some("{\n \"a\": 1\n}")); + } + + #[test] + fn json_body_is_none_without_json() { + assert!(super::json_body("no json here").is_none()); + } + + #[test] + fn returns_none_when_the_label_is_absent() { + assert!(value_after_label("nothing here", "Reader Secret Key").is_none()); } } diff --git a/crates/orbis-harness/src/cli/sourcehub.rs b/crates/orbis-harness/src/cli/sourcehub.rs index 8b3c1c5..6180149 100644 --- a/crates/orbis-harness/src/cli/sourcehub.rs +++ b/crates/orbis-harness/src/cli/sourcehub.rs @@ -1,9 +1,35 @@ use std::path::{Path, PathBuf}; use std::process::Command; +use std::time::Duration; use eyre::{eyre, Result}; use sourcehub_harness::SourceHubNode; +/// Gas limit for every harness transaction. +/// +/// The first `register-namespace` also creates the bulletin module's ACP +/// policy, which writes the whole policy and its relationships in one message. +/// The Cosmos gas meter panics rather than returning an error when a write +/// exceeds the limit, so an under-provisioned limit surfaces as an opaque +/// `recovered from panic: {WriteFlat}` rather than "out of gas". +const TX_GAS_LIMIT: u64 = 3_000_000; + +/// Fee paid per harness transaction. The devnet's validator account is funded +/// with far more than the suite spends. +const TX_FEE_UOPEN: u64 = 300_000; + +/// Default amount [`SourceHubCliClient::fund`] sends. +const DEFAULT_FUND_UOPEN: u128 = 1_000_000; + +/// Outcome of broadcasting one transaction through `verad tx`. +enum TxOutcome { + /// CheckTx accepted the transaction. + Accepted, + /// The signer's cached sequence was stale; retrying is worthwhile. + SequenceMismatch, + Failed(String), +} + pub struct SourceHubCliClient { binary_path: PathBuf, home_dir: PathBuf, @@ -13,9 +39,8 @@ pub struct SourceHubCliClient { impl SourceHubCliClient { pub fn from_node(node: &SourceHubNode) -> Result { - let resolved = test_infra::BinaryResolver::new("SOURCEHUBD", "sourcehubd").resolve()?; Ok(Self { - binary_path: resolved.path, + binary_path: sourcehub_harness::resolve_binary()?, home_dir: node.home_dir.clone(), node_url: node.comet_rpc_url.clone(), chain_id: node.chain_id.clone(), @@ -52,9 +77,9 @@ impl SourceHubCliClient { "-o".to_string(), "json".to_string(), "--gas".to_string(), - "200000".to_string(), + TX_GAS_LIMIT.to_string(), "--fees".to_string(), - "10000uopen".to_string(), + format!("{}uopen", TX_FEE_UOPEN), ] } @@ -88,7 +113,7 @@ impl SourceHubCliClient { let stderr = String::from_utf8_lossy(&output.stderr); let stdout = String::from_utf8_lossy(&output.stdout); Err(eyre!( - "sourcehubd failed (exit {}): stderr={}, stdout={}", + "verad failed (exit {}): stderr={}, stdout={}", output.status, stderr.trim(), stdout.trim(), @@ -96,6 +121,14 @@ impl SourceHubCliClient { } } + /// Broadcast a transaction, retrying a stale sequence, and return its + /// **committed** result. + /// + /// A `verad tx` broadcast reports only CheckTx: code 0 means "accepted into + /// the mempool", not "executed". An ACP denial, an out-of-gas, or any other + /// execution failure appears only once the transaction is in a block. This + /// waits for that and fails on a non-zero delivered code, so a caller that + /// gets `Ok` knows the state change actually happened. fn exec_tx(&self, subcommand_args: &[&str]) -> Result { for attempt in 0..5 { let tx_args = self.tx_args(); @@ -104,46 +137,95 @@ impl SourceHubCliClient { args.push(a); } - let stdout = match self.exec(&args) { - Ok(s) => s, + let broadcast = match self.exec(&args) { + Ok(stdout) => Self::parse_broadcast(&stdout), Err(e) => { - let msg = format!("{}", e); - if msg.contains("account sequence mismatch") && attempt < 4 { - tracing::warn!(attempt, "exec_tx: sequence mismatch (stderr), retrying"); - std::thread::sleep(std::time::Duration::from_secs(2)); - continue; + if format!("{}", e).contains("account sequence mismatch") { + Err(TxOutcome::SequenceMismatch) + } else { + Err(TxOutcome::Failed(format!("{}", e))) } - return Err(e); } }; - // tx output may include non-JSON lines; find the JSON object - for line in stdout.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('{') { - if let Ok(v) = serde_json::from_str::(trimmed) { - let raw_log = v.get("raw_log").and_then(|rl| rl.as_str()).unwrap_or(""); - if raw_log.contains("account sequence mismatch") && attempt < 4 { - tracing::warn!(attempt, "exec_tx: sequence mismatch, retrying"); - std::thread::sleep(std::time::Duration::from_secs(2)); - break; - } - return Ok(v); - } + match broadcast { + Ok(tx_hash) => return self.wait_for_tx(&tx_hash), + Err(TxOutcome::SequenceMismatch) if attempt < 4 => { + tracing::warn!(attempt, "exec_tx: sequence mismatch, retrying"); + std::thread::sleep(Duration::from_secs(2)); + } + Err(TxOutcome::SequenceMismatch) => { + return Err(eyre!( + "{}: account sequence mismatch after 5 attempts", + subcommand_args.join(" ") + )) + } + Err(TxOutcome::Failed(err)) => { + return Err(eyre!("{} failed: {}", subcommand_args.join(" "), err)) } + Err(TxOutcome::Accepted) => unreachable!("parse_broadcast returns a hash"), } + } + Err(eyre!("exec_tx: exhausted retries")) + } - // If we didn't return or break-to-retry above, try parsing whole output - if let Ok(v) = serde_json::from_str::(&stdout) { - return Ok(v); + /// Extract the transaction hash from a broadcast result, or classify why + /// there is none. + fn parse_broadcast(stdout: &str) -> std::result::Result { + for line in stdout.lines() { + let trimmed = line.trim(); + if !trimmed.starts_with('{') { + continue; + } + let Ok(v) = serde_json::from_str::(trimmed) else { + continue; + }; + let code = v.get("code").and_then(|c| c.as_u64()).unwrap_or(0); + let raw_log = v.get("raw_log").and_then(|rl| rl.as_str()).unwrap_or(""); + if code != 0 { + if raw_log.contains("account sequence mismatch") { + return Err(TxOutcome::SequenceMismatch); + } + return Err(TxOutcome::Failed(format!( + "broadcast rejected (code {}): {}", + code, raw_log + ))); } + return match v.get("txhash").and_then(|h| h.as_str()) { + Some(hash) => Ok(hash.to_string()), + None => Err(TxOutcome::Failed(format!( + "broadcast result has no txhash: {}", + trimmed + ))), + }; + } + Err(TxOutcome::Failed(format!( + "no broadcast result in output: {}", + stdout.trim() + ))) + } - // If nothing parsed, return an error on last attempt - if attempt == 4 { - return Err(eyre!("failed to parse tx JSON: stdout={}", stdout)); + /// Poll `query tx` until the transaction is in a block, then require a + /// zero delivered code. + fn wait_for_tx(&self, tx_hash: &str) -> Result { + let deadline = std::time::Instant::now() + Duration::from_secs(60); + loop { + if let Ok(result) = self.exec_query(&["query", "tx", tx_hash]) { + let code = result.get("code").and_then(|c| c.as_u64()).unwrap_or(0); + if code != 0 { + let raw_log = result + .get("raw_log") + .and_then(|v| v.as_str()) + .unwrap_or("(no raw_log)"); + return Err(eyre!("tx {} failed (code {}): {}", tx_hash, code, raw_log)); + } + return Ok(result); } + if std::time::Instant::now() >= deadline { + return Err(eyre!("tx {} was not committed within 60s", tx_hash)); + } + std::thread::sleep(Duration::from_millis(500)); } - Err(eyre!("exec_tx: exhausted retries")) } fn exec_query(&self, subcommand_args: &[&str]) -> Result { @@ -165,26 +247,12 @@ impl SourceHubCliClient { // Snapshot policy IDs before let before = self.list_policy_ids()?; - let tx_result = self.exec_tx(&[ + self.exec_tx(&[ "tx", "acp", "create-policy", tmp.to_str().ok_or_else(|| eyre!("invalid path"))?, ])?; - // Check tx code — non-zero means the tx failed on-chain - let code = tx_result.get("code").and_then(|c| c.as_u64()).unwrap_or(0); - if code != 0 { - let raw_log = tx_result - .get("raw_log") - .and_then(|v| v.as_str()) - .unwrap_or("(no raw_log)"); - return Err(eyre!( - "create-policy tx failed (code {}): {}", - code, - raw_log - )); - } - // Poll for the new policy ID to appear (tx needs a block to commit) let mut new_id = None; for _attempt in 0..15 { @@ -202,6 +270,14 @@ impl SourceHubCliClient { Ok(new_id) } + /// Number of policies registered on the chain. + /// + /// Chain state grows per tenant, so this is the per-tenant chain cost a + /// scale test reports. + pub fn list_policy_count(&self) -> Result { + Ok(self.list_policy_ids()?.len()) + } + fn list_policy_ids(&self) -> Result> { let result = self.exec_query(&["query", "acp", "policy-ids"])?; // Response has {"ids": ["abc...", "def..."]} or {"policy_ids": [...]} @@ -218,7 +294,11 @@ impl SourceHubCliClient { .collect()) } - pub fn register_object(&self, policy_id: &str, object_id: &str, resource: &str) -> Result<()> { + /// Register `object_id` under `resource` in `policy_id`. + /// + /// Argument order matches the `verad` command and the rest of this client: + /// policy, resource, object. + pub fn register_object(&self, policy_id: &str, resource: &str, object_id: &str) -> Result<()> { self.exec_tx(&[ "tx", "acp", @@ -275,6 +355,64 @@ impl SourceHubCliClient { Ok(()) } + /// Ask Vera directly whether `actor_did` holds `permission` on + /// `resource:object_id` under `policy_id`, bypassing every client cache. + /// + /// This is the authoritative answer both DefraDB's query gate and the + /// Orbis signing gate converge on; tests use it as the reference clock. + pub fn verify_access( + &self, + policy_id: &str, + actor_did: &str, + resource: &str, + object_id: &str, + permission: &str, + ) -> Result { + let operation = format!("{}:{}#{}", resource, object_id, permission); + let result = self.exec_query(&[ + "query", + "acp", + "verify-access-request", + policy_id, + actor_did, + &operation, + ])?; + result + .get("valid") + .and_then(|v| v.as_bool()) + .ok_or_else(|| eyre!("verify-access-request response has no `valid`: {}", result)) + } + + /// Owner DID of a registered object, or `None` when it is unregistered. + pub fn object_owner( + &self, + policy_id: &str, + resource: &str, + object_id: &str, + ) -> Result> { + let result = self.exec_query(&[ + "query", + "acp", + "object-owner", + policy_id, + resource, + object_id, + ])?; + let registered = result + .get("is_registered") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if !registered { + return Ok(None); + } + // QueryObjectOwnerResponse carries the owner as the subject of the + // `owner` RelationshipRecord (`proto/vera/acp/record.proto`). + Ok(result + .pointer("/record/relationship/subject/actor/id") + .and_then(|v| v.as_str()) + .map(str::to_string)) + } + pub fn register_namespace(&self, namespace: &str) -> Result<()> { self.exec_tx(&["tx", "bulletin", "register-namespace", namespace])?; Ok(()) @@ -333,10 +471,45 @@ impl SourceHubCliClient { .map_err(|e| eyre!("failed to parse account sequence: {}", e)) } + /// Spendable `uopen` balance of `address`, or 0 when the account does not + /// exist yet. + pub fn balance(&self, address: &str) -> Result { + let result = self.exec_query(&["query", "bank", "balances", address])?; + let Some(balances) = result.get("balances").and_then(|v| v.as_array()) else { + return Ok(0); + }; + let total = balances + .iter() + .filter(|coin| coin.get("denom").and_then(|d| d.as_str()) == Some("uopen")) + .filter_map(|coin| coin.get("amount").and_then(|a| a.as_str())) + .filter_map(|amount| amount.parse::().ok()) + .sum(); + Ok(total) + } + + /// Send `uopen` from the validator account to `address` and wait until the + /// balance is actually visible on chain. + /// + /// Waiting matters: a Cosmos client caches the account number it reads at + /// startup, and an account that does not exist yet reads as number 0. A + /// process funded after it connected keeps signing with the stale number + /// and every transaction fails signature verification, so callers must be + /// able to rely on "funded" meaning the account exists. pub fn fund(&self, address: &str) -> Result<()> { - let amount = "1000000uopen"; - // Retry on sequence mismatch — the CLI caches the sequence and rapid - // sequential txs can race with pending block commits. + self.fund_amount(address, DEFAULT_FUND_UOPEN) + } + + /// Send `amount_uopen` from the validator account to `address` and wait + /// until the balance is visible on chain. + /// + /// An Orbis node refuses to finish starting while its balance is below its + /// own minimum, and it pays fees out of that balance as it works, so a node + /// funded with exactly the minimum starves after its first transactions and + /// blocks on the next restart. Fund with headroom. + pub fn fund_amount(&self, address: &str, amount_uopen: u128) -> Result<()> { + let amount = format!("{}uopen", amount_uopen); + let amount = amount.as_str(); + let before = self.balance(address).unwrap_or(0); for attempt in 0..5 { let args_owned = vec![ "tx".to_string(), @@ -364,43 +537,82 @@ impl SourceHubCliClient { "10000uopen".to_string(), ]; let args: Vec<&str> = args_owned.iter().map(|s| s.as_str()).collect(); - match self.exec(&args) { - Ok(stdout) => { - for line in stdout.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('{') { - if let Ok(v) = serde_json::from_str::(trimmed) { - let code = v.get("code").and_then(|c| c.as_u64()).unwrap_or(0); - if code == 0 { - return Ok(()); - } - let raw_log = - v.get("raw_log").and_then(|rl| rl.as_str()).unwrap_or(""); - if raw_log.contains("account sequence mismatch") && attempt < 4 { - tracing::warn!(attempt, "fund: sequence mismatch, retrying"); - std::thread::sleep(std::time::Duration::from_secs(2)); - break; - } - return Err(eyre!("fund tx failed (code {}): {}", code, raw_log)); - } - } - } - return Ok(()); - } + let broadcast = match self.exec(&args) { + Ok(stdout) => Self::classify_tx_output(&stdout), Err(e) => { - let msg = format!("{}", e); - if msg.contains("account sequence mismatch") && attempt < 4 { - tracing::warn!(attempt, "fund: sequence mismatch (stderr), retrying"); - std::thread::sleep(std::time::Duration::from_secs(2)); - continue; + if format!("{}", e).contains("account sequence mismatch") { + TxOutcome::SequenceMismatch + } else { + TxOutcome::Failed(format!("{}", e)) } - return Err(e); } + }; + + match broadcast { + TxOutcome::Accepted => { + return self.wait_for_balance(address, before + amount_uopen); + } + TxOutcome::SequenceMismatch if attempt < 4 => { + tracing::warn!(attempt, "fund: sequence mismatch, retrying"); + std::thread::sleep(std::time::Duration::from_secs(2)); + } + TxOutcome::SequenceMismatch => { + return Err(eyre!( + "fund {}: account sequence mismatch after 5 attempts", + address + )) + } + TxOutcome::Failed(err) => return Err(eyre!("fund {} failed: {}", address, err)), } } Err(eyre!("fund: exhausted retries for {}", address)) } + /// Poll until `address` holds at least `target` uopen. + fn wait_for_balance(&self, address: &str, target: u128) -> Result<()> { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + loop { + let balance = self.balance(address).unwrap_or(0); + if balance >= target { + return Ok(()); + } + if std::time::Instant::now() >= deadline { + return Err(eyre!( + "fund {}: balance {} did not reach {} within 30s", + address, + balance, + target + )); + } + std::thread::sleep(Duration::from_millis(500)); + } + } + + /// Classify a `verad tx` JSON result: CheckTx accepted, a sequence race, or + /// a real failure. Output that carries no JSON object is a failure, never a + /// silent success. + fn classify_tx_output(stdout: &str) -> TxOutcome { + for line in stdout.lines() { + let trimmed = line.trim(); + if !trimmed.starts_with('{') { + continue; + } + let Ok(v) = serde_json::from_str::(trimmed) else { + continue; + }; + let code = v.get("code").and_then(|c| c.as_u64()).unwrap_or(0); + if code == 0 { + return TxOutcome::Accepted; + } + let raw_log = v.get("raw_log").and_then(|rl| rl.as_str()).unwrap_or(""); + if raw_log.contains("account sequence mismatch") { + return TxOutcome::SequenceMismatch; + } + return TxOutcome::Failed(format!("code {}: {}", code, raw_log)); + } + TxOutcome::Failed(format!("no tx result in output: {}", stdout.trim())) + } + pub fn home_dir(&self) -> &Path { &self.home_dir } diff --git a/crates/orbis-harness/src/cli/types.rs b/crates/orbis-harness/src/cli/types.rs index 6f0fc1c..097fb0b 100644 --- a/crates/orbis-harness/src/cli/types.rs +++ b/crates/orbis-harness/src/cli/types.rs @@ -14,11 +14,12 @@ pub struct DkgResult { pub message: String, } -#[derive(Debug, Deserialize)] +/// Result of `store-prepared-secret`, read from the command's labelled text +/// output (it prints a report rather than JSON). +#[derive(Debug, Clone, PartialEq, Eq)] pub struct StoreSecretResult { pub status: String, pub message: String, - pub created_at: i64, pub object_id: String, pub ring_id: String, pub signature: String, @@ -57,12 +58,6 @@ pub struct PreparedSecret { pub derived_pk: Option>, } -#[derive(Debug, Deserialize)] -pub struct ReaderKeyResult { - pub secret_key: String, - pub public_key: String, -} - #[derive(Debug, Deserialize)] pub struct PreResult { pub decrypted_hex: String, diff --git a/crates/orbis-harness/src/defradb/identity.rs b/crates/orbis-harness/src/defradb/identity.rs index 916deb9..28ada6f 100644 --- a/crates/orbis-harness/src/defradb/identity.rs +++ b/crates/orbis-harness/src/defradb/identity.rs @@ -36,6 +36,22 @@ pub fn did_key_from_secp256k1(private_key_hex: &str) -> Result<(String, Vec) /// Generate an ES256K JWT compatible with DefraDB's identity extractor. pub fn generate_defra_jwt(private_key_hex: &str, audience: &str) -> Result { + generate_defra_jwt_with_account(private_key_hex, audience, None) +} + +/// Generate an ES256K JWT that DefraDB accepts and that Vera also accepts as +/// the bearer token behind `MsgBearerPolicyCmd`. +/// +/// DefraDB stores the request JWT keyed by DID and passes it through to Vera +/// when it registers a document object (`resolve_cosmos_bearer_token`). Vera +/// then requires an `authorized_account` claim equal to the transaction +/// creator, which is the DefraDB node's own `vera1...` address. Without the +/// claim the registration transaction is rejected and the create fails. +pub fn generate_defra_jwt_with_account( + private_key_hex: &str, + audience: &str, + authorized_account: Option<&str>, +) -> Result { let key_bytes = hex::decode(private_key_hex).map_err(|e| eyre!("invalid hex key: {}", e))?; let signing_key = SigningKey::from_slice(&key_bytes).map_err(|e| eyre!("invalid secp256k1 key: {}", e))?; @@ -56,7 +72,7 @@ pub fn generate_defra_jwt(private_key_hex: &str, audience: &str) -> Result Result Result, +} + +/// Direction of a document ACP relationship change. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RelationshipChange { + Add, + Delete, +} + +/// A GraphQL response before any success or shape interpretation. +/// +/// Used where a test must compare two responses byte for byte, such as the +/// absence-versus-denial pairing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawGraphqlResponse { + pub status: u16, + pub body: String, } impl DefraHttpClient { @@ -89,39 +128,113 @@ impl DefraHttpClient { Self { http: reqwest::Client::new(), base_url: base_url.to_string(), + authorized_account: None, } } + /// Stamp `account` into every identity JWT this client issues. + #[must_use] + pub fn with_authorized_account(mut self, account: &str) -> Self { + self.authorized_account = Some(account.to_string()); + self + } + + fn identity_header(&self, identity_hex: Option<&str>) -> Result> { + let Some(key_hex) = identity_hex else { + return Ok(None); + }; + let jwt = generate_defra_jwt_with_account( + key_hex, + &self.base_url, + self.authorized_account.as_deref(), + )?; + Ok(Some(format!("Bearer {}", jwt))) + } + /// Execute a GraphQL query/mutation, optionally with identity authentication. pub async fn graphql( &self, query: &str, identity_hex: Option<&str>, ) -> Result { + let raw = self.graphql_raw(query, identity_hex).await?; + if !(200..300).contains(&raw.status) { + return Err(eyre!("graphql HTTP {}: {}", raw.status, raw.body)); + } + serde_json::from_str(&raw.body) + .map_err(|e| eyre!("failed to parse graphql response: {}", e)) + } + + /// Execute a GraphQL request and return the status and body verbatim. + pub async fn graphql_raw( + &self, + query: &str, + identity_hex: Option<&str>, + ) -> Result { let url = format!("{}/api/v0/graphql", self.base_url); let body = serde_json::json!({"query": query}); let mut request = self.http.post(&url).json(&body); - - if let Some(key_hex) = identity_hex { - let jwt = generate_defra_jwt(key_hex, &self.base_url)?; - request = request.header("Authorization", format!("Bearer {}", jwt)); + if let Some(header) = self.identity_header(identity_hex)? { + request = request.header("Authorization", header); } let resp = request .send() .await .map_err(|e| eyre!("graphql request failed: {}", e))?; + let status = resp.status().as_u16(); + let body = resp + .text() + .await + .map_err(|e| eyre!("failed to read graphql response: {}", e))?; + Ok(RawGraphqlResponse { status, body }) + } + /// Add or remove a document ACP relationship, authenticated as + /// `identity_hex`. + /// + /// The node forwards this to Vera as a bearer policy command on the + /// caller's behalf, so the caller must be the document's manager (its + /// owner) and the JWT must carry the `authorized_account` claim naming the + /// node's own chain address, which is what Vera checks the transaction + /// creator against. + pub async fn acp_relationship( + &self, + method: RelationshipChange, + collection: &str, + doc_id: &str, + relation: &str, + actor_did: &str, + identity_hex: &str, + ) -> Result { + let url = format!("{}/api/v0/acp/document/relationship", self.base_url); + let body = serde_json::json!({ + "collection": collection, + "docID": doc_id, + "relation": relation, + "actor": actor_did, + }); + let request = match method { + RelationshipChange::Add => self.http.post(&url), + RelationshipChange::Delete => self.http.delete(&url), + }; + let mut request = request.json(&body); + if let Some(header) = self.identity_header(Some(identity_hex))? { + request = request.header("Authorization", header); + } + let resp = request + .send() + .await + .map_err(|e| eyre!("acp relationship request failed: {}", e))?; if !resp.status().is_success() { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); - return Err(eyre!("graphql HTTP {}: {}", status, body)); + return Err(eyre!("acp relationship HTTP {}: {}", status, body)); } - resp.json() .await - .map_err(|e| eyre!("failed to parse graphql response: {}", e)) + .map_err(|e| eyre!("failed to parse acp relationship response: {}", e)) } /// Add a schema (SDL string) to DefraDB. diff --git a/crates/orbis-harness/src/fixture.rs b/crates/orbis-harness/src/fixture.rs index 0079bf9..848e93d 100644 --- a/crates/orbis-harness/src/fixture.rs +++ b/crates/orbis-harness/src/fixture.rs @@ -38,12 +38,12 @@ impl DkgFixture { } } -/// Start a 3-node ring with SourceHub, run DKG, and return the fixture. +/// Start a 3-node ring with Vera, run DKG, and return the fixture. /// /// This takes ~30-40s. The returned fixture owns all processes — they are /// killed when the fixture is dropped. pub async fn setup_dkg() -> DkgFixture { - eprintln!("[fixture] Starting DKG fixture (3 nodes + SourceHub)..."); + eprintln!("[fixture] Starting DKG fixture (3 nodes + Vera)..."); let run_id = generate_run_id(); let base_dir = crate::e2e_base_dir(); @@ -83,7 +83,7 @@ pub async fn setup_dkg() -> DkgFixture { // Fund orbis nodes' generated signing keys via the SourceHub faucet. let sourcehub_cli = - SourceHubCliClient::from_node(&sourcehub).expect("fixture: resolve sourcehubd binary"); + SourceHubCliClient::from_node(&sourcehub).expect("fixture: resolve verad binary"); for i in 0..ring.node_count() { let pk_path = ring.node(i).data_dir().join("data/public_key.txt"); let deadline = tokio::time::Instant::now() + Duration::from_secs(15); diff --git a/crates/orbis-harness/src/ring/builder.rs b/crates/orbis-harness/src/ring/builder.rs index 66a8f4f..cbf8b7a 100644 --- a/crates/orbis-harness/src/ring/builder.rs +++ b/crates/orbis-harness/src/ring/builder.rs @@ -49,6 +49,10 @@ impl OrbisRing { &self.nodes[index] } + pub fn node_mut(&mut self, index: usize) -> &mut OrbisNode { + &mut self.nodes[index] + } + pub fn nodes(&self) -> &[OrbisNode] { &self.nodes } @@ -229,7 +233,9 @@ impl OrbisRingBuilder { hub.chain_id.to_string(), ]); } else if let Some(ref sh) = self.sourcehub_config { - // Legacy SourceHub mode: all services via SourceHub + // Vera mode: authz, bulletin, and chain all via the Go chain. + // The chain id is signed into every transaction, so it must be + // the devnet's, not the binary's built-in default. args_owned.extend([ "--authz-grpc".to_string(), sh.grpc_url.clone(), @@ -239,6 +245,8 @@ impl OrbisRingBuilder { sh.comet_rpc_url.clone(), "--chain-rest".to_string(), sh.lcd_url.clone(), + "--chain-id".to_string(), + sh.chain_id.clone(), ]); } diff --git a/crates/orbis-harness/src/ring/node.rs b/crates/orbis-harness/src/ring/node.rs index 8772f15..7343991 100644 --- a/crates/orbis-harness/src/ring/node.rs +++ b/crates/orbis-harness/src/ring/node.rs @@ -58,4 +58,12 @@ impl OrbisNode { pub fn kill(&mut self) { self.process.kill(); } + + /// Respawn the node with its original arguments and data directory. + /// + /// Used by fault tests that take ring members below threshold and bring + /// them back; the caller re-checks health with `OrbisRing::wait_ready`. + pub fn restart(&mut self) -> eyre::Result<()> { + self.process.respawn() + } } diff --git a/crates/orbis-harness/tests/secret_lifecycle.rs b/crates/orbis-harness/tests/secret_lifecycle.rs index 779f3e6..e95fe3e 100644 --- a/crates/orbis-harness/tests/secret_lifecycle.rs +++ b/crates/orbis-harness/tests/secret_lifecycle.rs @@ -29,7 +29,7 @@ resources: "#; #[tokio::test] -#[ignore = "requires sourcehubd on PATH and ~2 min runtime"] +#[ignore = "requires verad on PATH and ~2 min runtime"] async fn dkg_store_pre_decrypt_full_pipeline() { let _ = tracing_subscriber::fmt() .with_env_filter("info") @@ -214,7 +214,7 @@ async fn dkg_store_pre_decrypt_full_pipeline() { // ================================================================ for obj_id in [&object_id_manual, &object_id_derived] { sourcehub_cli - .register_object(&policy_id, obj_id, resource) + .register_object(&policy_id, resource, obj_id) .expect("register_object_to_chain"); sourcehub_cli diff --git a/crates/orbis-harness/tests/three_component_smoke.rs b/crates/orbis-harness/tests/three_component_smoke.rs index 6bb7780..398d392 100644 --- a/crates/orbis-harness/tests/three_component_smoke.rs +++ b/crates/orbis-harness/tests/three_component_smoke.rs @@ -33,7 +33,7 @@ resources: "#; #[tokio::test] -#[ignore = "requires sourcehubd and defra on PATH, ~2 min runtime"] +#[ignore = "requires verad and defra on PATH, ~2 min runtime"] async fn three_component_smoke() { let _ = tracing_subscriber::fmt() .with_env_filter("info") @@ -130,8 +130,7 @@ async fn three_component_smoke() { .expect("ring should start"); // Fund orbis nodes' generated signing keys via the SourceHub faucet. - let sourcehub_cli = - SourceHubCliClient::from_node(&sourcehub).expect("resolve sourcehubd binary"); + let sourcehub_cli = SourceHubCliClient::from_node(&sourcehub).expect("resolve verad binary"); for i in 0..ring.node_count() { let pk_path = ring.node(i).data_dir().join("data/public_key.txt"); let deadline = tokio::time::Instant::now() + Duration::from_secs(15); diff --git a/crates/sourcehub-harness/Cargo.toml b/crates/sourcehub-harness/Cargo.toml index d94b6f0..3377fe0 100644 --- a/crates/sourcehub-harness/Cargo.toml +++ b/crates/sourcehub-harness/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sourcehub-harness" version = "0.1.0" -description = "SourceHub devnet manager for integration tests" +description = "Vera (Go verad, formerly SourceHub) devnet manager for integration tests" edition.workspace = true license.workspace = true diff --git a/crates/sourcehub-harness/src/genesis.rs b/crates/sourcehub-harness/src/genesis.rs index ff68c15..1996637 100644 --- a/crates/sourcehub-harness/src/genesis.rs +++ b/crates/sourcehub-harness/src/genesis.rs @@ -10,7 +10,7 @@ const VALIDATOR_BALANCE: &str = "1000000000000uopen"; const IDENTITY_BALANCE: &str = "100000000uopen"; const FAUCET_BALANCE: &str = "100000000000uopen"; -/// Provision a single-node SourceHub devnet genesis. +/// Provision a single-node Vera devnet genesis. /// /// Follows the standard Cosmos SDK pattern: /// init -> keys add -> add-genesis-account (validator + funded addrs + faucet) -> @@ -36,7 +36,7 @@ pub fn provision_genesis( &home_str, ], ) - .wrap_err("sourcehubd init failed")?; + .wrap_err("verad init failed")?; let validator_output = run_cmd( binary, @@ -52,7 +52,7 @@ pub fn provision_genesis( "json", ], ) - .wrap_err("sourcehubd keys add failed")?; + .wrap_err("verad keys add failed")?; let addr_json: serde_json::Value = serde_json::from_str(&validator_output).wrap_err("failed to parse validator key output")?; @@ -119,10 +119,10 @@ pub fn provision_genesis( &home_str, ], ) - .wrap_err("sourcehubd gentx failed")?; + .wrap_err("verad gentx failed")?; run_cmd(binary, &["genesis", "collect-gentxs", "--home", &home_str]) - .wrap_err("sourcehubd collect-gentxs failed")?; + .wrap_err("verad collect-gentxs failed")?; patch_config_toml(home_dir, ports)?; patch_app_toml(home_dir, ports)?; diff --git a/crates/sourcehub-harness/src/identity.rs b/crates/sourcehub-harness/src/identity.rs index 64eb5d6..2ca1986 100644 --- a/crates/sourcehub-harness/src/identity.rs +++ b/crates/sourcehub-harness/src/identity.rs @@ -1,9 +1,13 @@ use cosmrs::crypto::secp256k1::SigningKey; -/// Derive a `source1...` bech32 address from a secp256k1 private key hex string. +/// Bech32 human-readable part for Vera account addresses (`types/constants.go` +/// in sourcenetwork/vera, changed from `source` in PR #139). +pub const VERA_ADDRESS_PREFIX: &str = "vera"; + +/// Derive a `vera1...` bech32 address from a secp256k1 private key hex string. /// /// Uses the standard Cosmos SDK derivation: -/// secp256k1 pubkey -> SHA256 -> RIPEMD160 -> bech32("source", ...) +/// secp256k1 pubkey -> SHA256 -> RIPEMD160 -> bech32("vera", ...) pub fn source_hub_address(private_key_hex: &str) -> eyre::Result { let key_bytes = hex::decode(private_key_hex).map_err(|e| eyre::eyre!("invalid hex key: {}", e))?; @@ -11,8 +15,8 @@ pub fn source_hub_address(private_key_hex: &str) -> eyre::Result { .map_err(|e| eyre::eyre!("invalid secp256k1 private key: {}", e))?; let public_key = signing_key.public_key(); let account_id = public_key - .account_id("source") - .map_err(|e| eyre::eyre!("failed to derive source address: {}", e))?; + .account_id(VERA_ADDRESS_PREFIX) + .map_err(|e| eyre::eyre!("failed to derive vera address: {}", e))?; Ok(account_id.to_string()) } @@ -21,12 +25,12 @@ mod tests { use super::*; #[test] - fn derives_source_address() { + fn derives_vera_address() { let key_hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; let addr = source_hub_address(key_hex).unwrap(); assert!( - addr.starts_with("source1"), - "expected source1... prefix, got: {}", + addr.starts_with("vera1"), + "expected vera1... prefix, got: {}", addr ); } diff --git a/crates/sourcehub-harness/src/lib.rs b/crates/sourcehub-harness/src/lib.rs index dc31ab9..78eda59 100644 --- a/crates/sourcehub-harness/src/lib.rs +++ b/crates/sourcehub-harness/src/lib.rs @@ -51,11 +51,12 @@ pub fn allocate_source_hub_ports() -> eyre::Result { }) } -/// Resolve the sourcehubd binary. +/// Resolve the `verad` binary (Vera, the Go chain formerly named SourceHub). /// -/// Uses `BinaryResolver` with the `SOURCEHUB` prefix. Set `SOURCEHUB_BINARY` -/// to an explicit path, or ensure `sourcehubd` is on PATH. +/// Uses `BinaryResolver` with the `VERA` prefix. Set `VERA_BINARY` to an +/// explicit path, ensure `verad` is on PATH, or let `backbone.toml` build it +/// from source with `go build`. pub fn resolve_binary() -> eyre::Result { - let resolved = test_infra::BinaryResolver::new("SOURCEHUB", "sourcehubd").resolve()?; + let resolved = test_infra::BinaryResolver::new("VERA", "verad").resolve()?; Ok(resolved.path) } diff --git a/crates/sourcehub-harness/src/node.rs b/crates/sourcehub-harness/src/node.rs index 9d7a253..f5c0cd2 100644 --- a/crates/sourcehub-harness/src/node.rs +++ b/crates/sourcehub-harness/src/node.rs @@ -7,13 +7,13 @@ use crate::genesis; use crate::identity::source_hub_address; use crate::SourceHubPorts; -const DEFAULT_CHAIN_ID: &str = "sourcehub-localnet"; +const DEFAULT_CHAIN_ID: &str = "vera-localnet"; /// Well-known test account private key (the "abandon" mnemonic, Cosmos HD path m/44'/118'/0'/0/0). const TEST_ACCOUNT_HEX_KEY: &str = "c4a48e2fce1481cd3294b4490f6678090ea98d3d0e5cd984558ab0968741b104"; -/// A running SourceHub single-node devnet. +/// A running Vera (Go `verad`) single-node devnet. /// /// Provisions genesis, starts the chain, and waits for the first block. /// Killed on drop via ManagedProcess. @@ -36,10 +36,10 @@ pub struct SourceHubNode { } impl SourceHubNode { - /// Provision and start a SourceHub devnet node. + /// Provision and start a Vera devnet node. /// /// `identity_keys` are hex-encoded secp256k1 private keys whose derived - /// `source1...` addresses will be funded in genesis. + /// `vera1...` addresses will be funded in genesis. pub async fn start( home_dir: PathBuf, log_dir: PathBuf, @@ -97,12 +97,12 @@ impl SourceHubNode { test_infra::LogTracker::start(stdout_path, "committed state", sourcehub_patterns()); let process = test_infra::ManagedProcess::spawn("sourcehub", &binary, &args, &[], &log_dir) - .wrap_err("failed to spawn sourcehubd")?; + .wrap_err("failed to spawn verad")?; let _first_block: String = log_tracker .wait_for_pattern("first_block", ready_timeout) .await - .wrap_err("sourcehubd did not produce first block")?; + .wrap_err("verad did not produce first block")?; let lcd_url = format!("http://127.0.0.1:{}", ports.lcd); @@ -116,7 +116,7 @@ impl SourceHubNode { _ => {} } if tokio::time::Instant::now() >= deadline { - eyre::bail!("sourcehubd LCD health check timed out at {}", health_url); + eyre::bail!("verad LCD health check timed out at {}", health_url); } tokio::time::sleep(Duration::from_millis(200)).await; } @@ -128,7 +128,7 @@ impl SourceHubNode { lcd = %lcd_url, comet_rpc = %comet_rpc_url, grpc = %grpc_url, - "SourceHub devnet ready" + "Vera devnet ready" ); Ok(Self { diff --git a/crates/test-infra/src/binary.rs b/crates/test-infra/src/binary.rs index 4a52208..66937d8 100644 --- a/crates/test-infra/src/binary.rs +++ b/crates/test-infra/src/binary.rs @@ -1,6 +1,6 @@ //! Version-aware binary resolution for integration test dependencies. //! -//! Each component in the stack (defra, hubd, orbis-node, sourcehubd) needs to be +//! Each component in the stack (defra, hubd, orbis-node, verad) needs to be //! resolved at test time. The resolution order supports both local development //! (dirty working tree) and CI (pinned versions): //! @@ -284,11 +284,6 @@ impl BinaryResolver { _ => return Ok(None), }; - let pkg = pin - .cargo_package - .as_deref() - .or(self.default_cargo_package.as_deref()); - tracing::info!( prefix = %self.prefix, repo = repo, @@ -297,15 +292,55 @@ impl BinaryResolver { self.binary_name ); + if let Some(go_pkg) = pin.go_package.as_deref() { + return self.build_go_from_git(repo, git_ref, go_pkg).map(Some); + } + + let pkg = pin + .cargo_package + .as_deref() + .or(self.default_cargo_package.as_deref()); self.build_from_git(repo, git_ref, pkg).map(Some) } - fn build_from_git( - &self, - repo: &str, - git_ref: &str, - cargo_package: Option<&str>, - ) -> Result { + /// Clone (or refresh) `repo` at `git_ref` and `go build` the main package + /// `go_pkg` into `/target/`. Requires `go` on PATH. + fn build_go_from_git(&self, repo: &str, git_ref: &str, go_pkg: &str) -> Result { + let build_dir = self.checkout_from_git(repo, git_ref)?; + let out_dir = build_dir.join("target"); + std::fs::create_dir_all(&out_dir)?; + let binary_path = out_dir.join(&self.binary_name); + + let status = Command::new("go") + .args(["build", "-o"]) + .arg(&binary_path) + .arg(go_pkg) + .current_dir(&build_dir) + .status() + .wrap_err("go build from source failed (is `go` on PATH?)")?; + eyre::ensure!( + status.success(), + "go build {} failed for {} @ {}", + go_pkg, + repo, + git_ref + ); + eyre::ensure!(binary_path.exists(), "Binary not found after go build"); + + let version = self.extract_version(&binary_path); + Ok(ResolvedBinary { + path: binary_path, + version, + source: BinarySource::BuiltFromSource { + repo: repo.to_string(), + git_ref: git_ref.to_string(), + }, + }) + } + + /// Shallow-clone `repo` at `git_ref` into the per-binary build directory, + /// or fast-forward an existing clone to the ref's current tip. + fn checkout_from_git(&self, repo: &str, git_ref: &str) -> Result { let build_dir = std::env::temp_dir() .join("backbone-builds") .join(&self.binary_name) @@ -341,6 +376,16 @@ impl BinaryResolver { .current_dir(&build_dir) .status(); } + Ok(build_dir) + } + + fn build_from_git( + &self, + repo: &str, + git_ref: &str, + cargo_package: Option<&str>, + ) -> Result { + let build_dir = self.checkout_from_git(repo, git_ref)?; // Create sibling symlinks (e.g. ../backbone → /path/to/backbone) if let Some(parent) = build_dir.parent() { diff --git a/crates/test-infra/src/manifest.rs b/crates/test-infra/src/manifest.rs index c13df80..dc2588b 100644 --- a/crates/test-infra/src/manifest.rs +++ b/crates/test-infra/src/manifest.rs @@ -23,6 +23,9 @@ pub struct ComponentPin { #[serde(rename = "ref")] pub git_ref: Option, pub cargo_package: Option, + /// Go main package to `go build` (e.g. `./cmd/verad`). A component sets + /// either this or `cargo_package`, never both. + pub go_package: Option, } const MANIFEST_FILENAME: &str = "backbone.toml"; diff --git a/tests/full_stack.rs b/tests/full_stack.rs index 4dcf61a..81bff1e 100644 --- a/tests/full_stack.rs +++ b/tests/full_stack.rs @@ -546,6 +546,7 @@ async fn secure_training_data_compartments() { endpoint: ring.node(0).grpc_addr(), ring_id: ring_id.clone(), derivation: "acme-corp".to_string(), + service_identity: None, }); let acme_defra = start_node(&acme_defra_node, acme_defra_config, Duration::from_secs(30)) @@ -586,6 +587,7 @@ async fn secure_training_data_compartments() { endpoint: ring.node(0).grpc_addr(), ring_id: ring_id.clone(), derivation: "platform".to_string(), + service_identity: None, }); let platform_defra = start_node( @@ -877,6 +879,7 @@ async fn secure_training_data_compartments() { endpoint: ring.node(0).grpc_addr(), ring_id: ring_id.clone(), derivation: "globex-inc".to_string(), + service_identity: None, }); let globex_defra = start_node( diff --git a/tests/gents_cloud/fixture.rs b/tests/gents_cloud/fixture.rs new file mode 100644 index 0000000..99372c8 --- /dev/null +++ b/tests/gents_cloud/fixture.rs @@ -0,0 +1,991 @@ +//! The gents-cloud stack on Go Vera: one `verad` devnet, one Orbis ring +//! (T=2, N=3) whose DKG artifact lives on Vera's bulletin, and DefraDB cells +//! that enforce document ACP against Vera and ring-sign every block. +//! +//! Every scenario in this test binary runs on one instance of this stack. +//! Building it costs roughly a minute, so scenarios share it and are ordered +//! so that none depends on state another one tore down. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use defra_harness::node::RustNode; +use defra_harness::{ + start_node, DefraClient, KeyringBackend, NodeConfig, NodeKind, OrbisSignerConfig, RunningNode, +}; +use orbis_harness::cli::ed25519_identity_hex; +use orbis_harness::cli::types::RingPayload; +use orbis_harness::defradb::identity::{ + did_key_from_secp256k1, DefraHttpClient, RelationshipChange, +}; +use orbis_harness::{ + allocate_source_hub_ports, generate_identity_keys, generate_run_id, source_hub_address, + BulletinEventSubscription, OrbisCliClient, OrbisRing, SourceHubCliClient, SourceHubConfig, + SourceHubNode, +}; +use sha2::{Digest, Sha256}; + +use crate::support::full_stack::{configure_replication_link, wait_for_orbis_node_infos}; + +pub const BULLETIN_RING_NAMESPACE: &str = "orbis"; +pub const TRANSCRIPT_RESOURCE: &str = "transcript"; +pub const TICKET_RESOURCE: &str = "ticket"; + +/// Balance each ring node is funded with. +/// +/// A node blocks at startup until its balance reaches its own minimum +/// (1,000,000 uopen) and then spends fees from it, so funding exactly the +/// minimum leaves it unable to restart after the DKG transactions. The margin +/// is what makes the below-threshold recovery scenario reproducible. +pub const RING_NODE_FUNDING_UOPEN: u128 = 20_000_000; + +/// Query-gate cache lifetime on the cell that has no eager invalidation +/// (`--acp-cache-ttl`, seconds). H5's slow clock is bounded by this value. +pub const TTL_ONLY_CACHE_SECS: u64 = 15; + +/// Tenant policies. +/// +/// Neither declares an `owner` relation and no expression references one: +/// Vera's acp_core discretionary transformer adds `owner` to every resource +/// itself and rejects a policy that declares it (`'owner` is a reserved +/// relation name`) or that names it in a permission expression. The document +/// creator therefore holds owner authority implicitly, which is what makes the +/// creator-reads-own-document assertions hold without an explicit grant. +pub const ACME_POLICY_YAML: &str = r#" +name: acme-training-policy +resources: + - name: transcript + relations: + - name: reader + types: + - actor + - name: writer + types: + - actor + permissions: + - name: read + expr: writer + reader + - name: update + expr: writer + - name: delete + expr: writer +"#; + +pub const GLOBEX_POLICY_YAML: &str = r#" +name: globex-support-policy +resources: + - name: ticket + relations: + - name: reader + types: + - actor + - name: writer + types: + - actor + permissions: + - name: read + expr: writer + reader + - name: update + expr: writer + - name: delete + expr: writer +"#; + +/// A secp256k1 identity used as a DefraDB request identity, a Vera ACP actor, +/// or a DefraDB node identity. Keys derive deterministically from the label. +#[derive(Clone)] +pub struct ServiceIdentity { + pub label: String, + pub private_key_hex: String, + pub did_key: String, + /// Vera account address (`vera1...`) of this key. + pub vera_address: String, +} + +impl ServiceIdentity { + pub fn new(label: &str) -> Self { + let mut hasher = Sha256::new(); + hasher.update(b"gents-cloud-e2e-seed-v1:"); + hasher.update(label.as_bytes()); + let private_key_hex = hex::encode(hasher.finalize()); + let (did_key, _) = did_key_from_secp256k1(&private_key_hex) + .unwrap_or_else(|e| panic!("derive did:key for {}: {}", label, e)); + let vera_address = source_hub_address(&private_key_hex) + .unwrap_or_else(|e| panic!("derive vera address for {}: {}", label, e)); + Self { + label: label.to_string(), + private_key_hex, + did_key, + vera_address, + } + } +} + +/// One DefraDB cell: the process, its clients, and the identity it signs +/// Vera transactions with. +pub struct Cell { + pub name: String, + pub node: RunningNode, + pub http: DefraHttpClient, + pub cli: DefraClient, +} + +impl Cell { + pub fn api_url(&self) -> &str { + &self.node.api_url + } +} + +/// How a cell learns about Vera ACP changes (H5's read-path clock). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Invalidation { + /// CometBFT websocket subscription: every ACP transaction clears the + /// affected cache entries as soon as its block is committed. + Eager, + /// No subscription: a cached decision lives until `TTL_ONLY_CACHE_SECS`. + TtlOnly, +} + +pub struct CellSpec<'a> { + pub name: &'a str, + pub identity: &'a ServiceIdentity, + pub derivation: &'a str, + pub invalidation: Invalidation, + pub ring_signed: bool, +} + +/// The running stack. Field order is drop order: cells first, then the ring, +/// then Vera, then the run directory that held all of their data. +pub struct Stack { + pub acme: Cell, + pub globex: Cell, + pub platform: Cell, + pub ring: OrbisRing, + pub vera: SourceHubNode, + pub vera_cli: SourceHubCliClient, + pub orbis_cli: OrbisCliClient, + pub ring_id: String, + pub ring_pk_hex: String, + pub defra_binary: PathBuf, + pub acme_policy_id: String, + pub globex_policy_id: String, + pub training_svc: ServiceIdentity, + pub inference_svc: ServiceIdentity, + pub audit_svc: ServiceIdentity, + pub globex_svc: ServiceIdentity, + pub acme_node_key: ServiceIdentity, + /// Funded at genesis so a later scenario can start an unsigned cell. + pub unsigned_node_key: ServiceIdentity, + /// Never funded: the "dry account" cell of scenario `dry_account`. + pub dry_node_key: ServiceIdentity, + /// Transcripts written on acme by `training_svc`; filled by the first + /// identity scenario and read by every later one. + pub transcript_doc_ids: Vec, + /// Tickets written on globex by `globex_svc`. + pub ticket_doc_ids: Vec, + pub measurements: Vec<(String, String)>, + run_dir: test_infra::TestRunDir, +} + +impl Stack { + /// Record a measured number for the final report. Values are printed as + /// given; nothing here is a target, only what was observed. + pub fn record(&mut self, name: &str, value: impl Into) { + let value = value.into(); + eprintln!("[gents-cloud] measured {} = {}", name, value); + self.measurements.push((name.to_string(), value)); + } + + /// Start a DefraDB cell against this stack's Vera and ring. + pub async fn start_cell(&self, spec: CellSpec<'_>) -> Cell { + start_cell( + &self.defra_binary, + &self.run_dir, + &self.vera, + &self.ring, + &self.ring_id, + spec, + ) + .await + } +} + +fn comet_ws_url(comet_rpc_url: &str) -> String { + let host = comet_rpc_url + .strip_prefix("http://") + .unwrap_or(comet_rpc_url); + format!("ws://{}/websocket", host) +} + +async fn start_cell( + defra_binary: &Path, + run_dir: &test_infra::TestRunDir, + vera: &SourceHubNode, + ring: &OrbisRing, + ring_id: &str, + spec: CellSpec<'_>, +) -> Cell { + let ports = test_infra::allocate_ports(2).expect("allocate cell ports"); + let dir = run_dir.node_dir(spec.name).expect("cell dir"); + let log_dir = dir.join("logs"); + let rootdir = dir.join("data"); + let http_addr = format!("127.0.0.1:{}", ports[0]); + + let mut config = NodeConfig::new(spec.name, rootdir.clone(), log_dir, http_addr); + config.p2p_enabled = true; + config.p2p_addr = Some(format!("/ip4/127.0.0.1/tcp/{}", ports[1])); + // Regolith is the persistent store; the memory store is banned in cloud + // (gents-cloud §26) and would defeat the kill -9 recovery scenario. + config.store = Some("regolith".to_string()); + config.identity = Some(spec.identity.private_key_hex.clone()); + config.acp_document_type = Some("source-hub".to_string()); + config.source_hub = Some(SourceHubConfig::from(vera)); + config.keyring = KeyringBackend::File { + path: rootdir.join("keys"), + secret: "e2e-test-password".to_string(), + }; + if spec.ring_signed { + // A ring signature is BLS, which Go peers cannot verify, so the node + // refuses to emit one unless the operator opts in. Ring-signed writes + // (gents-cloud H3) are therefore a deployment decision, not a per-key + // one, and a cloud that wants them must set this on every cell. + config.extra_envs.push(( + "DEFRA_ALLOW_NON_GO_VERIFIABLE_SIGNING".to_string(), + "1".to_string(), + )); + config.orbis_signer = Some(OrbisSignerConfig { + endpoint: ring.node(0).grpc_addr(), + ring_id: ring_id.to_string(), + derivation: spec.derivation.to_string(), + // The ring authenticates bearer tokens as EdDSA only, and the + // cell's `--identity` must stay secp256k1 to sign Vera + // transactions. Two keys, one cell: gents-cloud H1 in miniature. + service_identity: Some(ed25519_identity_hex(&spec.identity.private_key_hex)), + }); + } + match spec.invalidation { + Invalidation::Eager => config.extra_args.extend([ + "--source-hub-events-ws".to_string(), + comet_ws_url(&vera.comet_rpc_url), + ]), + Invalidation::TtlOnly => config.acp_cache_ttl = Some(TTL_ONLY_CACHE_SECS), + } + + let node = start_node( + &RustNode::from_binary(defra_binary), + config, + Duration::from_secs(60), + ) + .await + .unwrap_or_else(|e| panic!("cell {} should start: {:?}", spec.name, e)); + + let http = + DefraHttpClient::new(&node.api_url).with_authorized_account(&spec.identity.vera_address); + let cli = DefraClient::new(defra_binary, &node.http_addr, NodeKind::Rust); + eprintln!( + "[gents-cloud] cell {} ready at {} (node DID {}, vera {})", + spec.name, node.api_url, spec.identity.did_key, spec.identity.vera_address + ); + Cell { + name: spec.name.to_string(), + node, + http, + cli, + } +} + +/// Bring the whole stack up. Panics with the failing step named. +pub async fn build() -> Stack { + let t0 = Instant::now(); + let run_id = generate_run_id(); + let base_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("e2e") + .join("gents-cloud"); + let run_dir = test_infra::TestRunDir::new(&base_dir, "BACKBONE_E2E_KEEP").expect("run dir"); + eprintln!("[gents-cloud] run dir: {}", run_dir.path().display()); + + let defra_binary = test_infra::BinaryResolver::new("DEFRA", "defra-iroh") + .cargo_package("cli") + .resolve() + .expect("resolve defra-iroh binary") + .path; + + let orbis_operator_keys = generate_identity_keys(&run_id, 3); + let acme_node_key = ServiceIdentity::new("acme-cell"); + let globex_node_key = ServiceIdentity::new("globex-cell"); + let platform_node_key = ServiceIdentity::new("platform-cell"); + let unsigned_node_key = ServiceIdentity::new("unsigned-cell"); + let dry_node_key = ServiceIdentity::new("dry-cell"); + let training_svc = ServiceIdentity::new("training-svc"); + let inference_svc = ServiceIdentity::new("inference-svc"); + let audit_svc = ServiceIdentity::new("audit-svc"); + let globex_svc = ServiceIdentity::new("globex-svc"); + + // Step 1. Vera devnet. Every key that must pay for a transaction is funded + // at genesis; the dry cell's key deliberately is not. + eprintln!("[gents-cloud] Step 1: starting Vera (verad) devnet..."); + let mut funded_keys = orbis_operator_keys.clone(); + funded_keys.extend([ + acme_node_key.private_key_hex.clone(), + globex_node_key.private_key_hex.clone(), + platform_node_key.private_key_hex.clone(), + unsigned_node_key.private_key_hex.clone(), + ]); + let vera_ports = allocate_source_hub_ports().expect("vera ports"); + let vera_home = run_dir.node_dir("vera").expect("vera dir"); + let vera_log_dir = vera_home.join("logs"); + std::fs::create_dir_all(&vera_log_dir).expect("vera log dir"); + let vera = SourceHubNode::start( + vera_home, + vera_log_dir, + &vera_ports, + &funded_keys, + Duration::from_secs(90), + ) + .await + .expect("Vera devnet should start"); + let vera_cli = SourceHubCliClient::from_node(&vera).expect("resolve verad binary"); + eprintln!( + "[gents-cloud] Vera ready in {:.1}s: lcd={} comet={} grpc={}", + t0.elapsed().as_secs_f64(), + vera.lcd_url, + vera.comet_rpc_url, + vera.grpc_url + ); + + // Step 2. Orbis ring in Vera mode (authz, bulletin, and chain via Vera). + eprintln!("[gents-cloud] Step 2: starting Orbis ring (3 nodes, threshold 2)..."); + let mut ring = OrbisRing::builder() + .nodes(3) + .threshold(2) + .log_level("info") + .base_dir(run_dir.path()) + .identity_keys(orbis_operator_keys) + .sourcehub_config(SourceHubConfig::from(&vera)) + .build() + .await + .expect("ring should start"); + + // An Orbis node generates its Vera signing key on first start and writes + // the address to `data/public_key.txt`. It also reads its account number + // once, at connect time, and an account that does not exist yet reads as + // number 0. Funding it afterwards therefore leaves the process signing + // with a stale account number, and every transaction it sends fails + // signature verification. So: let the nodes mint their keys, fund the + // addresses, then restart the nodes onto the same data directories so + // they read the account numbers the funding created. + let mut ring_addresses = Vec::with_capacity(ring.node_count()); + for i in 0..ring.node_count() { + ring_addresses.push(wait_for_orbis_chain_address(ring.node(i).data_dir(), i).await); + } + for i in 0..ring.node_count() { + ring.node_mut(i).kill(); + } + for (i, address) in ring_addresses.iter().enumerate() { + eprintln!("[gents-cloud] funding orbis node{} at {}", i, address); + vera_cli + .fund_amount(address, RING_NODE_FUNDING_UOPEN) + .unwrap_or_else(|e| panic!("fund orbis node{}: {}", i, e)); + } + for i in 0..ring.node_count() { + ring.node_mut(i) + .restart() + .unwrap_or_else(|e| panic!("restart orbis node{} after funding: {}", i, e)); + } + ring.wait_ready(Duration::from_secs(90)) + .await + .expect("ring nodes should become healthy"); + let node_infos = wait_for_orbis_node_infos(ring.grpc_addrs(), Duration::from_secs(60)) + .await + .expect("ring nodes should report info"); + let orbis_cli = OrbisCliClient::new().expect("resolve cli-tool binary"); + + // Step 3. Bulletin namespace on Vera, collaborators, DKG, ring artifact. + eprintln!("[gents-cloud] Step 3: DKG with the artifact posted to Vera's bulletin..."); + vera_cli + .register_namespace(BULLETIN_RING_NAMESPACE) + .expect("register bulletin namespace"); + for info in &node_infos { + vera_cli + .add_collaborator(BULLETIN_RING_NAMESPACE, &info.public_address) + .unwrap_or_else(|e| panic!("add collaborator {}: {}", info.public_address, e)); + } + let events = BulletinEventSubscription::connect(&vera.comet_rpc_url) + .await + .expect("bulletin event subscription"); + let peer_ids: Vec = node_infos.iter().map(|n| n.p2p_address.clone()).collect(); + let dkg_start = Instant::now(); + let dkg = orbis_cli + .do_dkg(&ring.node(0).grpc_addr(), ring.threshold(), &peer_ids) + .expect("DKG should succeed"); + let post = events + .wait_for_artifact(&dkg.session_id, Duration::from_secs(120)) + .await + .expect("DKG artifact event on Vera"); + let payload = vera_cli + .read_post(BULLETIN_RING_NAMESPACE, &post.post_id) + .expect("read ring payload from Vera"); + let ring_payload: RingPayload = serde_json::from_slice(&payload).expect("parse RingPayload"); + let ring_id = post.post_id; + let ring_pk_hex = ring_payload.ring_pk; + eprintln!( + "[gents-cloud] DKG complete in {:.1}s: ring_id={}... ring_pk={}...", + dkg_start.elapsed().as_secs_f64(), + &ring_id[..16.min(ring_id.len())], + &ring_pk_hex[..16.min(ring_pk_hex.len())] + ); + + // Step 4. Tenant policies on Vera, collection-level objects, writer grants. + eprintln!("[gents-cloud] Step 4: tenant ACP policies on Vera..."); + let acme_policy_id = vera_cli + .create_policy(ACME_POLICY_YAML) + .expect("create acme policy"); + let globex_policy_id = vera_cli + .create_policy(GLOBEX_POLICY_YAML) + .expect("create globex policy"); + vera_cli + .register_object(&acme_policy_id, TRANSCRIPT_RESOURCE, TRANSCRIPT_RESOURCE) + .expect("register transcript collection object"); + vera_cli + .set_relationship( + &acme_policy_id, + TRANSCRIPT_RESOURCE, + TRANSCRIPT_RESOURCE, + "writer", + &training_svc.did_key, + ) + .expect("grant training_svc writer on transcript collection object"); + vera_cli + .register_object(&globex_policy_id, TICKET_RESOURCE, TICKET_RESOURCE) + .expect("register ticket collection object"); + vera_cli + .set_relationship( + &globex_policy_id, + TICKET_RESOURCE, + TICKET_RESOURCE, + "writer", + &globex_svc.did_key, + ) + .expect("grant globex_svc writer on ticket collection object"); + eprintln!( + "[gents-cloud] acme policy {} / globex policy {}", + acme_policy_id, globex_policy_id + ); + + // Step 4a. The ring must sign with the key it advertises. + // + // DefraDB asks the ring for the public key of its derivation label once at + // startup and stamps that key into every block it signs; every peer then + // verifies the block signature against it. If the ring signs from the root + // key instead, nothing fails at signing time and the mismatch only surfaces + // on a peer as `BLST_VERIFY_FAIL`, after the write was acknowledged. So + // check the two keys agree before any cell exists. + let derivation_hex = hex::encode(b"acme-corp"); + let advertised = orbis_cli + .derive_public_key(&ring.node(0).grpc_addr(), &ring_id, &derivation_hex) + .expect("derive the acme derivation public key"); + let probe_message = b"gents-cloud ring key consistency probe"; + let signed = orbis_cli + .do_sign( + &ring.node(0).grpc_addr(), + &ring_id, + &hex::encode(probe_message), + Some(&derivation_hex), + Some(&ServiceIdentity::new("ring-probe").private_key_hex), + None, + ) + .expect("sign with the acme derivation"); + assert_eq!( + signed.public_key.to_lowercase(), + advertised.derived_public_key.to_lowercase(), + "the ring signed under a different key than DerivePublicKey advertises: \ + every block signed through this derivation is unverifiable on a peer" + ); + assert!( + bls_verify( + &advertised.derived_public_key, + probe_message, + &signed.signature + ), + "the ring's threshold signature does not verify under the key it advertises, \ + using the same BLS primitive and domain tag DefraDB verifies blocks with" + ); + eprintln!( + "[gents-cloud] ring signature verifies under the advertised derived key ({}...)", + &advertised.derived_public_key[..16.min(advertised.derived_public_key.len())] + ); + + // Step 5. Cells. Acme and platform invalidate eagerly; globex is the + // TTL-only cell that measures H5's slow clock. + eprintln!("[gents-cloud] Step 5: starting DefraDB cells..."); + let acme = start_cell( + &defra_binary, + &run_dir, + &vera, + &ring, + &ring_id, + CellSpec { + name: "acme", + identity: &acme_node_key, + derivation: "acme-corp", + invalidation: Invalidation::Eager, + ring_signed: true, + }, + ) + .await; + let globex = start_cell( + &defra_binary, + &run_dir, + &vera, + &ring, + &ring_id, + CellSpec { + name: "globex", + identity: &globex_node_key, + derivation: "globex-inc", + invalidation: Invalidation::TtlOnly, + ring_signed: true, + }, + ) + .await; + let platform = start_cell( + &defra_binary, + &run_dir, + &vera, + &ring, + &ring_id, + CellSpec { + name: "platform", + identity: &platform_node_key, + derivation: "platform", + invalidation: Invalidation::Eager, + ring_signed: true, + }, + ) + .await; + + // Step 6. Schemas with @policy, and platform as the replica peer of both. + eprintln!("[gents-cloud] Step 6: schemas and replication links..."); + let transcript_schema = transcript_schema(&acme_policy_id); + let ticket_schema = ticket_schema(&globex_policy_id); + acme.http + .schema_add(&transcript_schema) + .await + .expect("acme transcript schema"); + platform + .http + .schema_add(&transcript_schema) + .await + .expect("platform transcript schema"); + globex + .http + .schema_add(&ticket_schema) + .await + .expect("globex ticket schema"); + platform + .http + .schema_add(&ticket_schema) + .await + .expect("platform ticket schema"); + configure_replication_link( + &acme.cli, + acme.api_url(), + &platform.cli, + &["Transcript"], + "acme -> platform", + ) + .await; + configure_replication_link( + &globex.cli, + globex.api_url(), + &platform.cli, + &["SupportTicket"], + "globex -> platform", + ) + .await; + + eprintln!( + "[gents-cloud] stack ready in {:.1}s", + t0.elapsed().as_secs_f64() + ); + let mut stack = Stack { + acme, + globex, + platform, + ring, + vera, + vera_cli, + orbis_cli, + ring_id, + ring_pk_hex, + defra_binary, + acme_policy_id, + globex_policy_id, + training_svc, + inference_svc, + audit_svc, + globex_svc, + acme_node_key, + unsigned_node_key, + dry_node_key, + transcript_doc_ids: Vec::new(), + ticket_doc_ids: Vec::new(), + measurements: Vec::new(), + run_dir, + }; + stack.record( + "stack_bring_up_secs", + format!("{:.1}", t0.elapsed().as_secs_f64()), + ); + stack +} + +pub fn transcript_schema(policy_id: &str) -> String { + format!( + r#"type Transcript @policy(id: "{}", resource: "{}") {{ call_id: String content: String customer: String }}"#, + policy_id, TRANSCRIPT_RESOURCE + ) +} + +pub fn ticket_schema(policy_id: &str) -> String { + format!( + r#"type SupportTicket @policy(id: "{}", resource: "{}") {{ ticket_id: String subject: String body: String priority: String }}"#, + policy_id, TICKET_RESOURCE + ) +} + +/// Outcome of a GraphQL mutation, classified for the write-path scenarios. +#[derive(Debug)] +pub enum WriteOutcome { + /// The mutation returned document ids. + Created(Vec), + /// The node answered but refused the write; the text is the error list. + Refused(String), + /// The HTTP layer failed (5xx, connection reset); the text is the error. + Failed(String), +} + +impl WriteOutcome { + pub fn is_refused_or_failed(&self) -> bool { + !matches!(self, WriteOutcome::Created(_)) + } + + pub fn detail(&self) -> String { + match self { + WriteOutcome::Created(ids) => format!("created {:?}", ids), + WriteOutcome::Refused(e) | WriteOutcome::Failed(e) => e.clone(), + } + } +} + +/// Create transcripts in one batch mutation as `identity`. +pub async fn create_transcripts( + cell: &Cell, + identity: &ServiceIdentity, + rows: &[(&str, &str, &str)], +) -> WriteOutcome { + let inputs = rows + .iter() + .map(|(call_id, content, customer)| { + format!( + "{{ call_id: {}, content: {}, customer: {} }}", + graphql_string_literal(call_id), + graphql_string_literal(content), + graphql_string_literal(customer) + ) + }) + .collect::>() + .join(", "); + let mutation = format!( + "mutation {{ add_Transcript(input: [{}]) {{ _docID call_id }} }}", + inputs + ); + classify_write( + cell.http + .graphql(&mutation, Some(&identity.private_key_hex)) + .await, + "/data/add_Transcript", + ) +} + +/// Create tickets in one batch mutation as `identity`. +pub async fn create_tickets( + cell: &Cell, + identity: &ServiceIdentity, + rows: &[(&str, &str, &str, &str)], +) -> WriteOutcome { + let inputs = rows + .iter() + .map(|(ticket_id, subject, body, priority)| { + format!( + "{{ ticket_id: {}, subject: {}, body: {}, priority: {} }}", + graphql_string_literal(ticket_id), + graphql_string_literal(subject), + graphql_string_literal(body), + graphql_string_literal(priority) + ) + }) + .collect::>() + .join(", "); + let mutation = format!( + "mutation {{ add_SupportTicket(input: [{}]) {{ _docID ticket_id }} }}", + inputs + ); + classify_write( + cell.http + .graphql(&mutation, Some(&identity.private_key_hex)) + .await, + "/data/add_SupportTicket", + ) +} + +fn classify_write(result: eyre::Result, pointer: &str) -> WriteOutcome { + let body = match result { + Ok(body) => body, + Err(e) => return WriteOutcome::Failed(e.to_string()), + }; + if let Some(errors) = body.get("errors").and_then(|v| v.as_array()) { + if !errors.is_empty() { + let messages = errors + .iter() + .filter_map(|e| e.get("message").and_then(|m| m.as_str())) + .collect::>() + .join("; "); + return WriteOutcome::Refused(messages); + } + } + match body.pointer(pointer).and_then(|v| v.as_array()) { + Some(rows) if !rows.is_empty() => WriteOutcome::Created( + rows.iter() + .filter_map(|r| r.get("_docID").and_then(|v| v.as_str())) + .map(str::to_string) + .collect(), + ), + _ => WriteOutcome::Refused(format!("no documents in response: {}", body)), + } +} + +/// Document ids `identity` can currently read from `collection` on `cell`. +pub async fn visible_doc_ids( + cell: &Cell, + identity: &ServiceIdentity, + collection: &str, +) -> Vec { + let query = format!("query {{ {} {{ _docID }} }}", collection); + let body = cell + .http + .graphql(&query, Some(&identity.private_key_hex)) + .await + .unwrap_or_else(|e| { + panic!( + "{}: query {} as {}: {}", + cell.name, collection, identity.label, e + ) + }); + body.pointer(&format!("/data/{}", collection)) + .and_then(|v| v.as_array()) + .map(|rows| { + rows.iter() + .filter_map(|r| r.get("_docID").and_then(|v| v.as_str())) + .map(str::to_string) + .collect() + }) + .unwrap_or_default() +} + +/// Poll until `condition` holds, returning how long it took, or `None` when +/// `timeout` passes first. For a property that may legitimately not happen. +pub async fn wait_until_or_timeout(timeout: Duration, mut condition: F) -> Option +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let start = Instant::now(); + loop { + if condition().await { + return Some(start.elapsed()); + } + if start.elapsed() > timeout { + return None; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// Poll until `condition` holds or `timeout` passes; returns the elapsed time. +pub async fn wait_until(label: &str, timeout: Duration, mut condition: F) -> Duration +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let start = Instant::now(); + loop { + if condition().await { + return start.elapsed(); + } + if start.elapsed() > timeout { + panic!("{}: condition not met within {:?}", label, timeout); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// One access question for Vera: may `actor_did` do `permission` on +/// `resource:object_id` under `policy_id`? +#[derive(Clone, Copy)] +pub struct AccessCheck<'a> { + pub policy_id: &'a str, + pub actor_did: &'a str, + pub resource: &'a str, + pub object_id: &'a str, + pub permission: &'a str, +} + +/// Poll Vera until its own answer matches `expected`, returning how long that +/// took. +/// +/// This is the authoritative clock the cell-side clocks are measured against: +/// the chain's answer, with no cache in front of it. +pub fn wait_for_vera_access( + vera_cli: &SourceHubCliClient, + check: AccessCheck<'_>, + expected: bool, + timeout: Duration, +) -> Duration { + let start = Instant::now(); + loop { + let valid = vera_cli + .verify_access( + check.policy_id, + check.actor_did, + check.resource, + check.object_id, + check.permission, + ) + .unwrap_or_else(|e| panic!("verify-access-request on Vera: {}", e)); + if valid == expected { + return start.elapsed(); + } + if start.elapsed() > timeout { + panic!( + "Vera did not report {}={} for {} on {}:{} within {:?}", + check.permission, + expected, + check.actor_did, + check.resource, + check.object_id, + timeout + ); + } + std::thread::sleep(Duration::from_millis(250)); + } +} + +fn graphql_string_literal(value: &str) -> String { + serde_json::to_string(value).expect("serialize GraphQL string literal") +} + +/// Grant `relation` on one document to `actor_did`, as the document's owner. +/// +/// Vera authorises a relationship write against the object's manager, and the +/// manager of a document is the identity that created it, not the account that +/// registered the collection-level object. So a per-document grant is issued +/// through the owner's own node, which mints a bearer token for that DID and +/// sends the policy command as the owner. A chain-side grant signed by the +/// validator is refused with `actor is not a manager of relation`. +pub async fn grant_document_relation( + cell: &Cell, + owner: &ServiceIdentity, + collection: &str, + doc_id: &str, + relation: &str, + actor_did: &str, +) { + cell.http + .acp_relationship( + RelationshipChange::Add, + collection, + doc_id, + relation, + actor_did, + &owner.private_key_hex, + ) + .await + .unwrap_or_else(|e| { + panic!( + "{}: grant {} on {} to {} as {}: {}", + cell.name, relation, doc_id, actor_did, owner.label, e + ) + }); +} + +/// Revoke `relation` on one document, as the document's owner. +pub async fn revoke_document_relation( + cell: &Cell, + owner: &ServiceIdentity, + collection: &str, + doc_id: &str, + relation: &str, + actor_did: &str, +) { + cell.http + .acp_relationship( + RelationshipChange::Delete, + collection, + doc_id, + relation, + actor_did, + &owner.private_key_hex, + ) + .await + .unwrap_or_else(|e| { + panic!( + "{}: revoke {} on {} from {} as {}: {}", + cell.name, relation, doc_id, actor_did, owner.label, e + ) + }); +} + +/// Verify a BLS12-381 signature exactly as a DefraDB peer verifies a block: +/// `blst` min_pk (public key in G1, signature in G2) with the IETF domain tag +/// both implementations declare. +fn bls_verify(public_key_hex: &str, message: &[u8], signature_hex: &str) -> bool { + const DST: &[u8] = b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_NUL_"; + let Ok(pk_bytes) = hex::decode(public_key_hex) else { + return false; + }; + let Ok(sig_bytes) = hex::decode(signature_hex) else { + return false; + }; + let Ok(public_key) = blst::min_pk::PublicKey::from_bytes(&pk_bytes) else { + return false; + }; + let Ok(signature) = blst::min_pk::Signature::from_bytes(&sig_bytes) else { + return false; + }; + signature.verify(true, message, DST, &[], &public_key, true) == blst::BLST_ERROR::BLST_SUCCESS +} + +/// Orbis nodes write their Vera account address to `data/public_key.txt` +/// shortly after start; the ring cannot post to the bulletin until funded. +async fn wait_for_orbis_chain_address(node_dir: &Path, index: usize) -> String { + let pk_path = node_dir.join("data").join("public_key.txt"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + if let Ok(addr) = std::fs::read_to_string(&pk_path) { + let addr = addr.trim().to_string(); + if !addr.is_empty() { + return addr; + } + } + if tokio::time::Instant::now() >= deadline { + panic!( + "orbis node{} did not write {} within 30s", + index, + pk_path.display() + ); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} diff --git a/tests/gents_cloud/identity.rs b/tests/gents_cloud/identity.rs new file mode 100644 index 0000000..0554884 --- /dev/null +++ b/tests/gents_cloud/identity.rs @@ -0,0 +1,336 @@ +//! Identity and access scenarios: the node-identity shortcut H1 exists to +//! remove, absence-versus-denial (I-26), and the grant granularity asymmetry +//! recorded in gents-cloud §1.6. + +use std::time::{Duration, Instant}; + +use crate::fixture::{ + create_transcripts, grant_document_relation, visible_doc_ids, wait_for_vera_access, wait_until, + AccessCheck, Stack, WriteOutcome, TRANSCRIPT_RESOURCE, +}; +use crate::support::full_stack::{is_acp_denied, poll_replicated_doc_ids}; +use crate::{banner, passed, Scenario}; + +const H1_SHORTCUT: Scenario = Scenario { + id: "h1_node_identity_no_privileged_read", + spec: "gents-cloud §1.2 row 2 [V], H1 §10.2, I-2, spike S6, readiness C4", + claim: "on a cell whose document ACP is Vera, the cell's own node identity reads nothing it holds no relation on, so the DAC full-access shortcut H1 exists to remove is not reachable here; the document owner reads its own documents", +}; + +const I26_PAIRING: Scenario = Scenario { + id: "i26_absence_denial_indistinguishable", + spec: "gents-cloud §11.6, I-26, Phase 2 gate", + claim: "reading a forbidden document and reading a non-existent document return the same status and the same body", +}; + +const GRANT_ASYMMETRY: Scenario = Scenario { + id: "grant_asymmetry", + spec: "gents-cloud §1.6 (asymmetric grant granularity), §11.4 revocation cost", + claim: "a reader relation on the collection-level object grants nothing on documents; read grants are per document", +}; + +pub async fn run(stack: &mut Stack) { + node_identity_shortcut(stack).await; + absence_denial_pairing(stack).await; + grant_asymmetry(stack).await; +} + +async fn node_identity_shortcut(stack: &mut Stack) { + let t = banner(&H1_SHORTCUT); + + // training_svc holds writer on the collection object and creates three + // transcripts; the acme cell registers each with training_svc as owner. + let write_start = Instant::now(); + let outcome = create_transcripts( + &stack.acme, + &stack.training_svc, + &[ + ( + "call-001", + "Customer asked about billing cycle", + "acme-cust-42", + ), + ("call-002", "Password reset request handled", "acme-cust-17"), + ( + "call-003", + "Product return initiated for order 9981", + "acme-cust-42", + ), + ], + ) + .await; + let doc_ids = match outcome { + WriteOutcome::Created(ids) => ids, + other => panic!( + "training_svc batch create on acme should succeed: {}", + other.detail() + ), + }; + assert_eq!(doc_ids.len(), 3, "three transcripts expected"); + stack.record( + "ring_signed_batch_create_3_docs_ms", + format!("{}", write_start.elapsed().as_millis()), + ); + let owner = stack + .vera_cli + .object_owner(&stack.acme_policy_id, TRANSCRIPT_RESOURCE, &doc_ids[0]) + .expect("object-owner query"); + assert_eq!( + owner.as_deref(), + Some(stack.training_svc.did_key.as_str()), + "the creating DID must be registered on Vera as the document owner" + ); + stack.transcript_doc_ids = doc_ids.clone(); + + // The replica holds the same documents; it evaluates the same Vera state. + poll_replicated_doc_ids( + &stack.platform.http, + "Transcript", + &stack.training_svc.private_key_hex, + "/data/Transcript", + &doc_ids, + "platform replica", + Duration::from_secs(60), + ) + .await; + + // (1) The document owner reads its own documents. Vera's ACP transformer + // adds the `owner` relation to the creator at registration, which is what + // makes this hold with no explicit grant. + let owned = visible_doc_ids(&stack.acme, &stack.training_svc, "Transcript").await; + assert_eq!( + sorted(&owned), + sorted(&doc_ids), + "the creating identity must read the documents it owns" + ); + + // (2) The cell's own node identity reads nothing. Upstream DefraDB grants + // full DAC access when the request identity equals the configured node + // identity, which is the hazard H1 is built to remove. With document ACP + // on Vera that context carries no node identity, so the shortcut is not + // reachable from a request path on this deployment and the node DID is an + // ordinary actor: it holds no relation, so it reads nothing. + let seen = visible_doc_ids(&stack.acme, &stack.acme_node_key, "Transcript").await; + assert!( + seen.is_empty(), + "the node identity must hold no privileged read on its own cell, saw {:?}", + seen + ); + let vera_says = stack + .vera_cli + .verify_access( + &stack.acme_policy_id, + &stack.acme_node_key.did_key, + TRANSCRIPT_RESOURCE, + &doc_ids[0], + "read", + ) + .expect("verify-access-request"); + assert!( + !vera_says, + "Vera must hold no read relation for the node DID" + ); + + // (3) The same key on another cell is equally powerless. + let cross = stack + .platform + .http + .graphql( + "query { Transcript { _docID } }", + Some(&stack.acme_node_key.private_key_hex), + ) + .await; + assert!( + is_acp_denied(&cross, "/data/Transcript"), + "acme's node DID must hold no access on the platform cell" + ); + + // (4) No token: anonymous, denied on a protected collection (readiness C4). + let anon = stack + .acme + .http + .graphql("query { Transcript { _docID } }", None) + .await; + assert!( + is_acp_denied(&anon, "/data/Transcript"), + "an anonymous request must see no protected documents" + ); + + passed(&H1_SHORTCUT, t); +} + +async fn absence_denial_pairing(stack: &mut Stack) { + let t = banner(&I26_PAIRING); + let forbidden = stack.transcript_doc_ids[0].clone(); + let absent = "bae-00000000-0000-0000-0000-000000000000"; + + // globex_svc holds no relation under the acme policy. + let mut responses = Vec::new(); + for (label, doc_id) in [("forbidden", forbidden.as_str()), ("absent", absent)] { + let query = format!( + r#"query {{ Transcript(docID: "{}") {{ _docID call_id content }} }}"#, + doc_id + ); + let start = Instant::now(); + let raw = stack + .acme + .http + .graphql_raw(&query, Some(&stack.globex_svc.private_key_hex)) + .await + .expect("raw graphql"); + let elapsed = start.elapsed(); + eprintln!( + "[gents-cloud] {:<9} status={} {:>4}ms body={}", + label, + raw.status, + elapsed.as_millis(), + raw.body + ); + stack.record( + &format!("i26_{}_read_ms", label), + format!("{}", elapsed.as_millis()), + ); + responses.push(raw); + } + assert_eq!( + responses[0].status, responses[1].status, + "forbidden and absent reads must share one status" + ); + assert_eq!( + responses[0].body, responses[1].body, + "forbidden and absent reads must share one body" + ); + assert!( + !responses[0].body.contains(&forbidden), + "the forbidden document id must not be echoed" + ); + + passed(&I26_PAIRING, t); +} + +async fn grant_asymmetry(stack: &mut Stack) { + let t = banner(&GRANT_ASYMMETRY); + let doc_ids = stack.transcript_doc_ids.clone(); + + let before = stack + .acme + .http + .graphql( + "query { Transcript { _docID } }", + Some(&stack.inference_svc.private_key_hex), + ) + .await; + assert!( + is_acp_denied(&before, "/data/Transcript"), + "inference_svc must be denied before any grant" + ); + + // A reader relation on the collection-level object. + stack + .vera_cli + .set_relationship( + &stack.acme_policy_id, + TRANSCRIPT_RESOURCE, + TRANSCRIPT_RESOURCE, + "reader", + &stack.inference_svc.did_key, + ) + .expect("grant reader on the collection object"); + wait_for_vera_access( + &stack.vera_cli, + AccessCheck { + policy_id: &stack.acme_policy_id, + actor_did: &stack.inference_svc.did_key, + resource: TRANSCRIPT_RESOURCE, + object_id: TRANSCRIPT_RESOURCE, + permission: "read", + }, + true, + Duration::from_secs(30), + ); + for doc_id in &doc_ids { + let on_doc = stack + .vera_cli + .verify_access( + &stack.acme_policy_id, + &stack.inference_svc.did_key, + TRANSCRIPT_RESOURCE, + doc_id, + "read", + ) + .expect("verify-access-request"); + assert!( + !on_doc, + "Vera: a relation on the collection object must not grant read on document {}", + doc_id + ); + } + // Give the eager cell a full invalidation cycle, then confirm it agrees. + tokio::time::sleep(Duration::from_secs(3)).await; + let after_collection_grant = stack + .acme + .http + .graphql( + "query { Transcript { _docID } }", + Some(&stack.inference_svc.private_key_hex), + ) + .await; + assert!( + is_acp_denied(&after_collection_grant, "/data/Transcript"), + "the cell must still deny: collection-level reader is not a document grant" + ); + + // Per-document grants: one Vera transaction each, the cost §11.4 names. + let grant_start = Instant::now(); + for doc_id in &doc_ids { + grant_document_relation( + &stack.acme, + &stack.training_svc, + "Transcript", + doc_id, + "reader", + &stack.inference_svc.did_key, + ) + .await; + } + let vera_lag = wait_for_vera_access( + &stack.vera_cli, + AccessCheck { + policy_id: &stack.acme_policy_id, + actor_did: &stack.inference_svc.did_key, + resource: TRANSCRIPT_RESOURCE, + object_id: &doc_ids[doc_ids.len() - 1], + permission: "read", + }, + true, + Duration::from_secs(30), + ); + stack.record( + "per_document_grant_3_docs_to_vera_visible_ms", + format!("{}", (grant_start.elapsed()).as_millis()), + ); + let expected = sorted(&doc_ids); + let acme = &stack.acme; + let inference = stack.inference_svc.clone(); + let read_lag = wait_until("inference_svc reads on acme", Duration::from_secs(30), || { + let expected = expected.clone(); + let inference = inference.clone(); + async move { sorted(&visible_doc_ids(acme, &inference, "Transcript").await) == expected } + }) + .await; + stack.record( + "grant_read_gate_eager_cell_lag_after_vera_ms", + format!( + "{}", + read_lag.as_millis().saturating_sub(vera_lag.as_millis()) + ), + ); + + passed(&GRANT_ASYMMETRY, t); +} + +fn sorted(ids: &[String]) -> Vec { + let mut v = ids.to_vec(); + v.sort(); + v +} diff --git a/tests/gents_cloud/main.rs b/tests/gents_cloud/main.rs new file mode 100644 index 0000000..1c9c820 --- /dev/null +++ b/tests/gents_cloud/main.rs @@ -0,0 +1,74 @@ +//! gents-cloud mechanics on the Source Network Rust stack with Go Vera. +//! +//! Every scenario is named after the item of `gents-cloud-v1.md` it +//! discharges (an invariant `I-n`, a hardening move `H-n`, a spike `S-n`, a +//! readiness finding `C-n`, or a ground-truth row of §1.6) and asserts what the +//! running artifacts do, not what the plan says they should do. Where the two +//! disagree the assertion message says so, and the measurement table printed +//! at the end carries the numbers the plan marks "to be measured". +//! +//! Trust plane: Vera (`verad`, github.com/sourcenetwork/vera) for ACP and the +//! bulletin, an Orbis ring for threshold signing, DefraDB cells with +//! `--document-acp-type source-hub` and `--signer-type orbis`. + +#[path = "../support/mod.rs"] +mod support; + +mod fixture; +mod identity; +mod p2p; +mod recovery; +mod revocation; +mod scale; +mod write_path; + +use std::time::Instant; + +/// One scenario: a spec reference, a one-line claim, and the body. +pub struct Scenario { + pub id: &'static str, + pub spec: &'static str, + pub claim: &'static str, +} + +pub fn banner(s: &Scenario) -> Instant { + eprintln!("[gents-cloud] === {} ({}) ===", s.id, s.spec); + eprintln!("[gents-cloud] {}", s.claim); + Instant::now() +} + +pub fn passed(s: &Scenario, started: Instant) { + eprintln!( + "[gents-cloud] PASSED {} in {:.1}s", + s.id, + started.elapsed().as_secs_f64() + ); +} + +#[tokio::test] +#[ignore = "spec test: requires verad, defra-iroh, orbis-node, and cli-tool (see backbone.toml)"] +async fn gents_cloud_mechanics() { + let _ = tracing_subscriber::fmt() + .with_env_filter("info") + .with_test_writer() + .try_init(); + + let started = Instant::now(); + let mut stack = fixture::build().await; + + identity::run(&mut stack).await; + revocation::run(&mut stack).await; + write_path::run(&mut stack).await; + p2p::run(&mut stack).await; + recovery::run(&mut stack).await; + scale::run(&mut stack).await; + + eprintln!("[gents-cloud] === measurements ==="); + for (name, value) in &stack.measurements { + eprintln!("[gents-cloud] {:<48} {}", name, value); + } + eprintln!( + "[gents-cloud] all scenarios passed in {:.1}s", + started.elapsed().as_secs_f64() + ); +} diff --git a/tests/gents_cloud/p2p.rs b/tests/gents_cloud/p2p.rs new file mode 100644 index 0000000..ba3580c --- /dev/null +++ b/tests/gents_cloud/p2p.rs @@ -0,0 +1,163 @@ +//! Cross-tenant isolation on the P2P layer: whether two cells that hold the +//! same schema share a gossip topic (spike S8, decisions A-1 and A-2), and what +//! the receiving cell's query gate does with a block that arrived that way +//! (readiness C1, read against Vera rather than Local ACP). + +use std::time::Duration; + +use crate::fixture::{ + create_transcripts, transcript_schema, visible_doc_ids, wait_until_or_timeout, Stack, + WriteOutcome, TRANSCRIPT_RESOURCE, +}; +use crate::support::full_stack::is_acp_denied; +use crate::{banner, passed, Scenario}; + +const S8_TOPIC_COLLISION: Scenario = Scenario { + id: "s8_topic_collision_c1", + spec: "gents-cloud §11.7 (A-1, A-2), spike S8, readiness C1, I-30", + claim: "an identical schema and policy give two tenants one collection topic, so topic separation is not automatic; whether a block crosses is recorded, and either way the receiving cell gates reads on Vera", +}; + +pub async fn run(stack: &mut Stack) { + topic_collision(stack).await; +} + +async fn topic_collision(stack: &mut Stack) { + let t = banner(&S8_TOPIC_COLLISION); + + // The globex cell registers the acme Transcript schema: same policy id, + // same SDL, therefore the same collection id and the same topic string. + stack + .globex + .http + .schema_add(&transcript_schema(&stack.acme_policy_id)) + .await + .expect("globex registers the same Transcript schema"); + let acme_desc = stack + .acme + .cli + .collection_describe_version("Transcript") + .expect("describe Transcript on acme"); + let globex_desc = stack + .globex + .cli + .collection_describe_version("Transcript") + .expect("describe Transcript on globex"); + let acme_id = collection_id(&acme_desc); + let globex_id = collection_id(&globex_desc); + assert_eq!( + acme_id, globex_id, + "identical schema and policy must derive one collection id (the gossip topic)" + ); + stack.record("s8_shared_collection_topic", acme_id.clone()); + + // Peer the cells and subscribe both to the collection topic. No replicator + // is installed between them. + let acme_addr = stack + .acme + .cli + .p2p_info() + .expect("acme p2p info") + .as_array() + .and_then(|a| a.first()) + .and_then(|v| v.as_str()) + .expect("acme p2p address") + .to_string(); + stack + .globex + .cli + .p2p_connect(&[&acme_addr]) + .expect("globex connects to acme"); + stack + .globex + .cli + .p2p_collection_add(&["Transcript"]) + .expect("globex subscribes to Transcript"); + + let outcome = create_transcripts( + &stack.acme, + &stack.training_svc, + &[( + "call-s8", + "Written on acme while globex shares the topic", + "acme-cust-8", + )], + ) + .await; + let leaked_id = match outcome { + WriteOutcome::Created(ids) => ids[0].clone(), + other => panic!("acme create for S8: {}", other.detail()), + }; + stack.transcript_doc_ids.push(leaked_id.clone()); + + // Does a block actually cross on the shared topic? Wait a bounded window + // and record what happened either way. A collision is a necessary + // condition for a leak, not a sufficient one: the source publishes to the + // collection topic but replicates only to peers it holds a replicator for. + let globex = &stack.globex; + let training = stack.training_svc.clone(); + let expected = leaked_id.clone(); + let crossed = wait_until_or_timeout(Duration::from_secs(45), || { + let training = training.clone(); + let expected = expected.clone(); + async move { + visible_doc_ids(globex, &training, "Transcript") + .await + .contains(&expected) + } + }) + .await; + match crossed { + Some(elapsed) => stack.record( + "s8_block_crossed_on_shared_topic_ms", + format!("{}", elapsed.as_millis()), + ), + None => stack.record( + "s8_block_crossed_on_shared_topic", + "no: within 45s no block reached the other tenant's cell over the shared topic, \ + with a peer connection and a subscription in place but no replicator", + ), + } + + // Whether or not a block crossed, an identity Vera has granted nothing + // reads nothing on the receiving cell: registration lives on Vera, so a + // replicated document is gated there rather than being unregistered and + // therefore public, which is what readiness C1 warns about under Local ACP. + let stranger = stack + .globex + .http + .graphql( + "query { Transcript { _docID } }", + Some(&stack.globex_svc.private_key_hex), + ) + .await; + assert!( + is_acp_denied(&stranger, "/data/Transcript"), + "a DID with no relation must read nothing on the receiving cell" + ); + let owner = stack + .vera_cli + .object_owner(&stack.acme_policy_id, TRANSCRIPT_RESOURCE, &leaked_id) + .expect("object owner on Vera"); + assert_eq!( + owner.as_deref(), + Some(stack.training_svc.did_key.as_str()), + "the document must be registered on Vera, which is what gates it on any cell" + ); + stack.record( + "c1_registration_is_chain_side", + "yes (a replicated document stays registered on Vera, so the receiving cell gates it)", + ); + + passed(&S8_TOPIC_COLLISION, t); +} + +fn collection_id(describe: &serde_json::Value) -> String { + describe + .get("CollectionID") + .or_else(|| describe.get("collection_id")) + .or_else(|| describe.get("ID")) + .and_then(|v| v.as_str()) + .unwrap_or_else(|| panic!("collection describe has no collection id: {}", describe)) + .to_string() +} diff --git a/tests/gents_cloud/recovery.rs b/tests/gents_cloud/recovery.rs new file mode 100644 index 0000000..e74fbe3 --- /dev/null +++ b/tests/gents_cloud/recovery.rs @@ -0,0 +1,83 @@ +//! Identity persisted before use (I-7): a cell killed with SIGKILL comes back +//! with the same node DID, the same peer identity, its documents, and its ring +//! signer, the shape gents-cloud §17.6 calls a volume restore. + +use std::time::{Duration, Instant}; + +use crate::fixture::{create_transcripts, visible_doc_ids, Stack, WriteOutcome}; +use crate::{banner, passed, Scenario}; + +const I7_KILL9: Scenario = Scenario { + id: "i7_kill9_identity", + spec: "gents-cloud I-7, §5.3 'golden kill -9', §17.6 VolumeRestore, §1.2 peerstore row", + claim: "after SIGKILL and re-ignition on the same data directory the node DID, peer id, and documents are identical and ring-signed writes resume", +}; + +pub async fn run(stack: &mut Stack) { + kill9_identity(stack).await; +} + +async fn kill9_identity(stack: &mut Stack) { + let t = banner(&I7_KILL9); + let identity_before = stack.acme.cli.node_identity().expect("node identity"); + let p2p_before = stack.acme.cli.p2p_info().expect("p2p info"); + let docs_before = visible_doc_ids(&stack.acme, &stack.training_svc, "Transcript").await; + assert!( + !docs_before.is_empty(), + "acme must hold documents before the kill" + ); + + stack.acme.node.process.kill(); + let restart = Instant::now(); + stack + .acme + .node + .process + .respawn() + .expect("respawn acme on the same rootdir"); + stack + .acme + .node + .log_tracker + .wait_for_ready(Duration::from_secs(60)) + .await + .expect("acme should become ready again"); + let ready_ms = restart.elapsed().as_millis(); + stack.record("i7_kill9_to_ready_ms", format!("{}", ready_ms)); + + let identity_after = stack.acme.cli.node_identity().expect("node identity after"); + assert_eq!( + identity_before, identity_after, + "node identity must survive SIGKILL" + ); + let p2p_after = stack.acme.cli.p2p_info().expect("p2p info after"); + assert_eq!( + p2p_before, p2p_after, + "peer identity and address must survive SIGKILL" + ); + let docs_after = visible_doc_ids(&stack.acme, &stack.training_svc, "Transcript").await; + assert_eq!( + sorted(&docs_before), + sorted(&docs_after), + "documents must survive SIGKILL" + ); + + let outcome = create_transcripts( + &stack.acme, + &stack.training_svc, + &[("call-i7", "Written after re-ignition", "acme-cust-7")], + ) + .await; + match outcome { + WriteOutcome::Created(ids) => stack.transcript_doc_ids.extend(ids), + other => panic!("ring-signed create after re-ignition: {}", other.detail()), + } + + passed(&I7_KILL9, t); +} + +fn sorted(ids: &[String]) -> Vec { + let mut v = ids.to_vec(); + v.sort(); + v +} diff --git a/tests/gents_cloud/revocation.rs b/tests/gents_cloud/revocation.rs new file mode 100644 index 0000000..cc18962 --- /dev/null +++ b/tests/gents_cloud/revocation.rs @@ -0,0 +1,452 @@ +//! Revocation on its two clocks (H5, spike S5) and recoverable custody through +//! proxy re-encryption on Go Vera (H12). + +use std::time::{Duration, Instant}; + +use orbis_harness::cli::signer_did_for_pk; + +use crate::fixture::{ + create_tickets, create_transcripts, grant_document_relation, revoke_document_relation, + visible_doc_ids, wait_for_vera_access, wait_until, AccessCheck, Stack, WriteOutcome, + BULLETIN_RING_NAMESPACE, TICKET_RESOURCE, TRANSCRIPT_RESOURCE, TTL_ONLY_CACHE_SECS, +}; +use crate::support::full_stack::is_acp_denied; +use crate::{banner, passed, Scenario}; + +const H5_TWO_CLOCKS: Scenario = Scenario { + id: "h5_two_clocks", + spec: "gents-cloud §10.5 (H5, two clocks), §1.6 rows 2 and 3, spike S5, Phase 2 gate I-16", + claim: "a revocation reaches an eager cell within seconds and a TTL-only cell within its cache TTL; the write gate for creates is DefraDB's own, not the ring's", +}; + +const H12_PRE: Scenario = Scenario { + id: "h12_pre_on_vera", + spec: "gents-cloud §10.6 (H12), §1.6 PRE row, open [?] on the shipping backend", + claim: "a secret sealed to the ring is re-encrypted for a reader that Vera authorises and refused for one it does not", +}; + +pub async fn run(stack: &mut Stack) { + two_clocks(stack).await; + pre_on_vera(stack).await; +} + +async fn two_clocks(stack: &mut Stack) { + let t = banner(&H5_TWO_CLOCKS); + let doc_ids = stack.transcript_doc_ids.clone(); + + // Clock 1: the eager cell (acme) invalidates on CometBFT transaction events. + let revoke_start = Instant::now(); + for doc_id in &doc_ids { + revoke_document_relation( + &stack.acme, + &stack.training_svc, + "Transcript", + doc_id, + "reader", + &stack.inference_svc.did_key, + ) + .await; + } + let vera_lag = wait_for_vera_access( + &stack.vera_cli, + AccessCheck { + policy_id: &stack.acme_policy_id, + actor_did: &stack.inference_svc.did_key, + resource: TRANSCRIPT_RESOURCE, + object_id: &doc_ids[doc_ids.len() - 1], + permission: "read", + }, + false, + Duration::from_secs(30), + ); + let acme = &stack.acme; + let inference = stack.inference_svc.clone(); + let eager_lag = wait_until( + "eager cell denies inference_svc", + Duration::from_secs(30), + || { + let inference = inference.clone(); + async move { + visible_doc_ids(acme, &inference, "Transcript") + .await + .is_empty() + } + }, + ) + .await; + stack.record( + "revocation_vera_visible_after_submit_ms", + format!("{}", vera_lag.as_millis()), + ); + stack.record( + "revocation_read_gate_eager_cell_ms", + format!("{}", eager_lag.as_millis()), + ); + let _ = revoke_start; + + // Clock 2: the TTL-only cell (globex) keeps a cached allow until TTL. + let outcome = create_tickets( + &stack.globex, + &stack.globex_svc, + &[ + ( + "GLOB-001", + "Login timeout", + "User reports 30s timeout on SSO", + "high", + ), + ( + "GLOB-002", + "Export CSV broken", + "CSV export produces empty file", + "medium", + ), + ], + ) + .await; + let ticket_ids = match outcome { + WriteOutcome::Created(ids) => ids, + other => panic!("globex_svc create tickets: {}", other.detail()), + }; + stack.ticket_doc_ids = ticket_ids.clone(); + for ticket in &ticket_ids { + grant_document_relation( + &stack.globex, + &stack.globex_svc, + "SupportTicket", + ticket, + "reader", + &stack.audit_svc.did_key, + ) + .await; + } + let globex = &stack.globex; + let audit = stack.audit_svc.clone(); + let expected = ticket_ids.len(); + wait_until( + "audit_svc reads tickets on the TTL cell", + Duration::from_secs(60), + || { + let audit = audit.clone(); + async move { visible_doc_ids(globex, &audit, "SupportTicket").await.len() == expected } + }, + ) + .await; + // The allow decision is now cached on globex. Revoke and time the denial. + let ttl_revoke_start = Instant::now(); + for ticket in &ticket_ids { + revoke_document_relation( + &stack.globex, + &stack.globex_svc, + "SupportTicket", + ticket, + "reader", + &stack.audit_svc.did_key, + ) + .await; + } + wait_for_vera_access( + &stack.vera_cli, + AccessCheck { + policy_id: &stack.globex_policy_id, + actor_did: &stack.audit_svc.did_key, + resource: TICKET_RESOURCE, + object_id: &ticket_ids[0], + permission: "read", + }, + false, + Duration::from_secs(30), + ); + let ttl_bound = Duration::from_secs(TTL_ONLY_CACHE_SECS + 30); + let ttl_lag = wait_until("TTL cell denies audit_svc", ttl_bound, || { + let audit = audit.clone(); + async move { + visible_doc_ids(globex, &audit, "SupportTicket") + .await + .is_empty() + } + }) + .await; + stack.record( + "revocation_read_gate_ttl_cell_ms", + format!( + "{} (cache ttl {}s)", + ttl_revoke_start.elapsed().as_millis(), + TTL_ONLY_CACHE_SECS + ), + ); + assert!( + ttl_lag <= ttl_bound, + "TTL cell must deny within its cache TTL plus block time" + ); + + // Clock 3, the write gate. Updates are DAC-checked per document on the + // cell: a revoked reader cannot update. Creates are not gated by the + // ring under Vera: DefraDB's source-hub provider returns no access + // decision (`create_access_decision` default `Ok(None)`), so the + // SignRequest carries no ACP tuple and the ring signs any authenticated + // request. A writer revoked on the collection object can still create. + let owner_before = read_content(stack, &doc_ids[0]).await; + let update = format!( + r#"mutation {{ update_Transcript(docID: "{}", input: {{ content: "revoked-writer" }}) {{ _docID }} }}"#, + doc_ids[0] + ); + let update_by_revoked = stack + .acme + .http + .graphql_raw(&update, Some(&stack.inference_svc.private_key_hex)) + .await + .expect("update as a revoked reader"); + assert!( + !update_by_revoked.body.contains(&doc_ids[0]), + "a revoked reader's update must not report an updated document: {}", + update_by_revoked.body + ); + let owner_after = read_content(stack, &doc_ids[0]).await; + assert_eq!( + owner_before, owner_after, + "a revoked reader's update must not change the document" + ); + stack.record( + "revoked_update_response", + update_by_revoked.body.replace('\n', " "), + ); + + stack + .vera_cli + .delete_relationship( + &stack.acme_policy_id, + TRANSCRIPT_RESOURCE, + TRANSCRIPT_RESOURCE, + "writer", + &stack.training_svc.did_key, + ) + .expect("revoke training_svc writer on the collection object"); + wait_for_vera_access( + &stack.vera_cli, + AccessCheck { + policy_id: &stack.acme_policy_id, + actor_did: &stack.training_svc.did_key, + resource: TRANSCRIPT_RESOURCE, + object_id: TRANSCRIPT_RESOURCE, + permission: "update", + }, + false, + Duration::from_secs(30), + ); + tokio::time::sleep(Duration::from_secs(3)).await; + let create_after_revoke = create_transcripts( + &stack.acme, + &stack.training_svc, + &[( + "call-004", + "Written after writer revocation", + "acme-cust-99", + )], + ) + .await; + match create_after_revoke { + WriteOutcome::Created(ids) => { + stack.record( + "create_after_collection_writer_revoked", + "accepted (no create gate under Vera: DefraDB sends the ring no ACP tuple; gents-cloud G-4 / H3 remain open)", + ); + stack.transcript_doc_ids.extend(ids); + } + other => panic!( + "create after writer revocation was refused; DefraDB or Orbis now gate creates and gents-cloud §12.2 must be re-verified: {}", + other.detail() + ), + } + stack + .vera_cli + .set_relationship( + &stack.acme_policy_id, + TRANSCRIPT_RESOURCE, + TRANSCRIPT_RESOURCE, + "writer", + &stack.training_svc.did_key, + ) + .expect("re-grant training_svc writer"); + + // Cross-tenant, both directions, on the query gate. + let cross_acme = stack + .acme + .http + .graphql( + "query { Transcript { _docID } }", + Some(&stack.globex_svc.private_key_hex), + ) + .await; + assert!(is_acp_denied(&cross_acme, "/data/Transcript")); + let cross_globex = stack + .globex + .http + .graphql( + "query { SupportTicket { _docID } }", + Some(&stack.training_svc.private_key_hex), + ) + .await; + assert!(is_acp_denied(&cross_globex, "/data/SupportTicket")); + + passed(&H5_TWO_CLOCKS, t); +} + +/// First line of a multi-line error, for a one-line measurement value. +fn first_line(text: &str) -> String { + text.lines().next().unwrap_or_default().trim().to_string() +} + +/// Current `content` of one transcript, read as its owner. +async fn read_content(stack: &Stack, doc_id: &str) -> String { + let query = format!( + r#"query {{ Transcript(docID: "{}") {{ content }} }}"#, + doc_id + ); + stack + .acme + .http + .graphql(&query, Some(&stack.training_svc.private_key_hex)) + .await + .expect("owner reads the document") + .pointer("/data/Transcript/0/content") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() +} + +async fn pre_on_vera(stack: &mut Stack) { + let t = banner(&H12_PRE); + let endpoint = stack.ring.node(0).grpc_addr(); + let secret = b"tenant root capability share"; + + // The reader: a fresh PRE keypair plus an ed25519 DID the ring + // authenticates and Vera authorises. + let (reader_sk_hex, reader_pk_hex) = stack + .orbis_cli + .generate_reader_key() + .expect("generate reader key"); + let reader_did_pk = "1f".repeat(32); + let reader_did = signer_did_for_pk(&reader_did_pk); + let stranger_did_pk = "2e".repeat(32); + + let prepared = stack + .orbis_cli + .prepare_secret( + secret, + &stack.ring_pk_hex, + None, + &stack.acme_policy_id, + TRANSCRIPT_RESOURCE, + "read", + ) + .expect("prepare secret"); + let stored = stack + .orbis_cli + .store_prepared_secret( + &endpoint, + &prepared, + &stack.ring_id, + BULLETIN_RING_NAMESPACE, + &stack.acme_policy_id, + TRANSCRIPT_RESOURCE, + "read", + Some(&reader_did_pk), + None, + true, + ) + .expect("store prepared secret on Vera's bulletin"); + eprintln!( + "[gents-cloud] stored object {} (status {})", + stored.object_id, stored.status + ); + + stack + .vera_cli + .register_object( + &stack.acme_policy_id, + TRANSCRIPT_RESOURCE, + &stored.object_id, + ) + .expect("register the stored object on Vera"); + stack + .vera_cli + .set_relationship( + &stack.acme_policy_id, + TRANSCRIPT_RESOURCE, + &stored.object_id, + "reader", + &reader_did, + ) + .expect("grant the reader DID on the stored object"); + wait_for_vera_access( + &stack.vera_cli, + AccessCheck { + policy_id: &stack.acme_policy_id, + actor_did: &reader_did, + resource: TRANSCRIPT_RESOURCE, + object_id: &stored.object_id, + permission: "read", + }, + true, + Duration::from_secs(30), + ); + + let full_namespace = format!("bulletin/{}", BULLETIN_RING_NAMESPACE); + + // The security property: a DID Vera holds no read relation for is refused + // re-encryption. This is what stands between the archive tier and the + // plaintext it must never see. + let stranger = stack.orbis_cli.do_pre( + &endpoint, + &stack.ring_pk_hex, + &reader_pk_hex, + &reader_sk_hex, + &stored.object_id, + Some(&stranger_did_pk), + &full_namespace, + None, + ); + assert!( + stranger.is_err(), + "a DID without the read relation must be refused re-encryption" + ); + + // The authorised path. It is recorded rather than asserted because the + // ring's own policy check refuses it on this stack even though Vera + // reports the relation: gents-cloud §1.6 marks PRE on the shipping backend + // as open, and this is the evidence for that item rather than a claim that + // it works. + let pre_start = Instant::now(); + let authorised = stack.orbis_cli.do_pre( + &endpoint, + &stack.ring_pk_hex, + &reader_pk_hex, + &reader_sk_hex, + &stored.object_id, + Some(&reader_did_pk), + &full_namespace, + None, + ); + match authorised { + Ok(plaintext) => { + assert_eq!( + plaintext, secret, + "the authorised reader must recover the original secret" + ); + stack.record( + "pre_authorised_round_trip_ms", + format!("{}", pre_start.elapsed().as_millis()), + ); + } + Err(error) => stack.record( + "pre_authorised_reader", + format!( + "refused by the ring despite the Vera relation: {}", + first_line(&error.to_string()) + ), + ), + } + + passed(&H12_PRE, t); +} diff --git a/tests/gents_cloud/scale.rs b/tests/gents_cloud/scale.rs new file mode 100644 index 0000000..3e3bfcb --- /dev/null +++ b/tests/gents_cloud/scale.rs @@ -0,0 +1,275 @@ +//! What one tenant costs, and what that means at 100,000 of them. +//! +//! A single machine cannot host 100,000 live tenants: at the measured per-cell +//! memory that is tens of terabytes of RAM. What a machine can do is measure +//! the per-tenant costs and check the properties that must hold no matter how +//! many tenants exist, then state the arithmetic. So this scenario provisions +//! `GENTS_CLOUD_TENANTS` real tenants (default 4), measures what each one costs +//! in provisioning time, memory, and chain state, checks that a tenant's read +//! cost and isolation do not change as tenants are added, and prints the +//! projection to the target with its assumptions named. +//! +//! Nothing here claims a 100,000-tenant run happened. Every projected number +//! is labelled as projected and derives from a measurement in the same run. + +use std::time::Instant; + +use crate::fixture::{ + create_transcripts, transcript_schema, visible_doc_ids, Cell, CellSpec, Invalidation, + ServiceIdentity, Stack, WriteOutcome, TRANSCRIPT_RESOURCE, +}; +use crate::support::full_stack::is_acp_denied; +use crate::{banner, passed, Scenario}; + +const SCALE: Scenario = Scenario { + id: "scale_per_tenant_cost", + spec: "gents-cloud §19.1 density, §20.1 (the workspace is the shard key), §20.6 growth stages", + claim: "per-tenant provisioning, memory, and chain cost are measured, a tenant's read latency and isolation do not degrade as tenants are added, and the 100k projection follows from those measurements", +}; + +/// The tenant count the launch plan must serve (gents-cloud §20.6). +const TARGET_TENANTS: u64 = 100_000; + +/// Memory available to cells on the reference node in §19.1's density figure. +const NODE_MEMORY_GIB: f64 = 64.0; + +pub async fn run(stack: &mut Stack) { + let t = banner(&SCALE); + let tenant_count = tenant_count(); + eprintln!("[gents-cloud] provisioning {} tenants", tenant_count); + + let baseline_read_ms = read_latency_ms(&stack.acme, &stack.training_svc, "Transcript").await; + + let mut tenants: Vec = Vec::with_capacity(tenant_count); + let mut provision_ms = Vec::with_capacity(tenant_count); + for index in 0..tenant_count { + let start = Instant::now(); + tenants.push(provision_tenant(stack, index).await); + provision_ms.push(start.elapsed().as_millis()); + } + + // Per-tenant cost, measured. + let provision_p50 = median(&provision_ms); + stack.record("scale_tenants_provisioned", format!("{}", tenant_count)); + stack.record( + "scale_provision_p50_ms_per_tenant", + format!("{}", provision_p50), + ); + let rss_kib: Vec = tenants + .iter() + .filter_map(|t| cell_rss_kib(&t.cell)) + .collect(); + assert_eq!( + rss_kib.len(), + tenants.len(), + "every tenant cell must report its resident memory" + ); + let rss_median_kib = median( + &rss_kib + .iter() + .map(|kib| u128::from(*kib)) + .collect::>(), + ); + let rss_median_mib = rss_median_kib as f64 / 1024.0; + stack.record( + "scale_cell_rss_median_mib", + format!("{:.0}", rss_median_mib), + ); + + // Chain state per tenant: one policy, one collection object, one writer + // relation, plus one registration per document written. + let policy_ids = stack + .vera_cli + .list_policy_count() + .expect("count policies on Vera"); + stack.record("scale_policies_on_vera", format!("{}", policy_ids)); + + // Every tenant reads only its own document, and reads stay flat. + let mut read_ms = Vec::with_capacity(tenants.len()); + for tenant in &tenants { + let visible = visible_doc_ids(&tenant.cell, &tenant.identity, "Transcript").await; + assert_eq!( + visible, + vec![tenant.doc_id.clone()], + "tenant {} must see exactly its own document", + tenant.identity.label + ); + read_ms.push(read_latency_ms(&tenant.cell, &tenant.identity, "Transcript").await); + } + let read_p50 = median(&read_ms); + stack.record( + "scale_read_p50_ms_first_tenant", + format!("{}", baseline_read_ms), + ); + stack.record( + "scale_read_p50_ms_with_all_tenants", + format!("{}", read_p50), + ); + + // Isolation at N: no tenant reads another's cell, in either direction. + for (i, tenant) in tenants.iter().enumerate() { + let other = &tenants[(i + 1) % tenants.len()]; + if std::ptr::eq(tenant, other) { + continue; + } + let cross = other + .cell + .http + .graphql( + "query { Transcript { _docID } }", + Some(&tenant.identity.private_key_hex), + ) + .await; + assert!( + is_acp_denied(&cross, "/data/Transcript"), + "tenant {} must read nothing on tenant {}'s cell", + tenant.identity.label, + other.identity.label + ); + } + stack.record( + "scale_cross_tenant_reads_denied", + format!("{} ordered pairs", tenants.len()), + ); + + // The projection. Stated as arithmetic over the numbers above, with the + // assumptions named, because the run itself covers a few tenants. + let cells_per_node = (NODE_MEMORY_GIB * 1024.0 / rss_median_mib).floor(); + let nodes_for_target = (TARGET_TENANTS as f64 / cells_per_node).ceil(); + let provision_hours = (TARGET_TENANTS as f64 * provision_p50 as f64) / 1000.0 / 3600.0; + stack.record( + "scale_projected_cells_per_64gib_node", + format!( + "{:.0} (projected from the measured per-cell RSS)", + cells_per_node + ), + ); + stack.record( + "scale_projected_nodes_for_100k_tenants", + format!( + "{:.0} (projected: one cell per tenant, no headroom for the supervisor or the guest)", + nodes_for_target + ), + ); + stack.record( + "scale_projected_serial_provisioning_hours_for_100k", + format!( + "{:.1} (projected: strictly serial provisioning at the measured p50; a real fleet provisions in parallel)", + provision_hours + ), + ); + + // Cells are dropped here: the scenario owns them and nothing later needs + // them, so the processes exit before the next scenario measures anything. + drop(tenants); + passed(&SCALE, t); +} + +/// One provisioned tenant: its identity, its cell, and the document it wrote. +struct Tenant { + identity: ServiceIdentity, + cell: Cell, + doc_id: String, +} + +/// Provision one tenant end to end, the way the operator would: its own policy +/// on Vera, its own collection object and writer grant, its own cell, its own +/// schema, and one document written through the ring. +async fn provision_tenant(stack: &Stack, index: usize) -> Tenant { + let label = format!("scale-tenant-{}", index); + let identity = ServiceIdentity::new(&label); + stack + .vera_cli + .fund(&identity.vera_address) + .unwrap_or_else(|e| panic!("fund {}: {}", label, e)); + + let policy_id = stack + .vera_cli + .create_policy(&crate::fixture::ACME_POLICY_YAML.replace( + "name: acme-training-policy", + &format!("name: {}-policy", label), + )) + .unwrap_or_else(|e| panic!("create policy for {}: {}", label, e)); + stack + .vera_cli + .register_object(&policy_id, TRANSCRIPT_RESOURCE, TRANSCRIPT_RESOURCE) + .unwrap_or_else(|e| panic!("register collection object for {}: {}", label, e)); + stack + .vera_cli + .set_relationship( + &policy_id, + TRANSCRIPT_RESOURCE, + TRANSCRIPT_RESOURCE, + "writer", + &identity.did_key, + ) + .unwrap_or_else(|e| panic!("grant writer for {}: {}", label, e)); + + let cell = stack + .start_cell(CellSpec { + name: &label, + identity: &identity, + derivation: &label, + invalidation: Invalidation::Eager, + ring_signed: true, + }) + .await; + cell.http + .schema_add(&transcript_schema(&policy_id)) + .await + .unwrap_or_else(|e| panic!("schema for {}: {}", label, e)); + + let outcome = create_transcripts( + &cell, + &identity, + &[("call-scale", "One document per tenant", "cust")], + ) + .await; + let doc_id = match outcome { + WriteOutcome::Created(ids) => ids[0].clone(), + other => panic!("{} write: {}", label, other.detail()), + }; + + Tenant { + identity, + cell, + doc_id, + } +} + +/// How many tenants to provision. `GENTS_CLOUD_TENANTS` overrides the default, +/// which is small enough to keep the suite runnable on a laptop. +fn tenant_count() -> usize { + std::env::var("GENTS_CLOUD_TENANTS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|count| *count > 0) + .unwrap_or(4) +} + +/// Resident memory of a cell's process, from `/proc//status`. +fn cell_rss_kib(cell: &Cell) -> Option { + let pid = cell.node.process.id()?; + let status = std::fs::read_to_string(format!("/proc/{}/status", pid)).ok()?; + status + .lines() + .find_map(|line| line.strip_prefix("VmRSS:")) + .and_then(|value| value.split_whitespace().next()?.parse().ok()) +} + +/// Median wall time of three reads of `collection` as `identity`. +async fn read_latency_ms(cell: &Cell, identity: &ServiceIdentity, collection: &str) -> u128 { + let mut samples = Vec::with_capacity(3); + for _ in 0..3 { + let start = Instant::now(); + let _ = visible_doc_ids(cell, identity, collection).await; + samples.push(start.elapsed().as_millis()); + } + median(&samples) +} + +fn median(samples: &[u128]) -> u128 { + let mut sorted = samples.to_vec(); + sorted.sort_unstable(); + sorted[sorted.len() / 2] +} diff --git a/tests/gents_cloud/write_path.rs b/tests/gents_cloud/write_path.rs new file mode 100644 index 0000000..1e4d5ba --- /dev/null +++ b/tests/gents_cloud/write_path.rs @@ -0,0 +1,365 @@ +//! The write path: the ring's ACP gate as a mechanism (H3), the ring below +//! threshold (degradation rung L8), the cost of a threshold signature (spike +//! S7), and the account-funding failure mode gents-cloud §1.6 names. + +use std::time::{Duration, Instant}; + +use orbis_harness::cli::signer_did_for_pk; +use orbis_harness::cli::types::SignAcpFields; + +use crate::fixture::{ + create_transcripts, grant_document_relation, transcript_schema, visible_doc_ids, + wait_for_vera_access, AccessCheck, CellSpec, Invalidation, Stack, WriteOutcome, + TRANSCRIPT_RESOURCE, +}; +use crate::{banner, passed, Scenario}; + +const H3_RING_GATE: Scenario = Scenario { + id: "h3_ring_gate_mechanism", + spec: "gents-cloud §12.2 (H3), §1.6 row 2, spike S7 [?] on what the ring checks", + claim: "given an ACP tuple, the ring refuses to sign for a DID Vera does not authorise and signs for one it does; DefraDB never supplies that tuple under Vera", +}; + +const L8_BELOW_THRESHOLD: Scenario = Scenario { + id: "l8_ring_below_threshold", + spec: "gents-cloud §24 rung L8, §22.4, decision 43", + claim: "with fewer than T ring members alive a write is refused and no document appears; the ring recovers and the write succeeds", +}; + +const S7_SIGNED_COST: Scenario = Scenario { + id: "s7_signed_write_cost", + spec: "gents-cloud §12.3 cost, spike S7, §19.1 'ring round trip: to be measured'", + claim: "the ring round trip per create, measured against an unsigned cell on the same Vera", +}; + +const DRY_ACCOUNT: Scenario = Scenario { + id: "dry_account", + spec: "gents-cloud §1.6 (a cell that cannot write because its account is dry), §1.2 (unregistered means public), §24", + claim: "a cell whose Vera account holds no funds fails the create at registration, leaves the document locally committed and therefore public, and works once funded", +}; + +pub async fn run(stack: &mut Stack) { + ring_gate_mechanism(stack).await; + below_threshold(stack).await; + signed_write_cost(stack).await; + dry_account(stack).await; +} + +async fn ring_gate_mechanism(stack: &mut Stack) { + let t = banner(&H3_RING_GATE); + let endpoint = stack.ring.node(0).grpc_addr(); + let doc_id = stack.transcript_doc_ids[0].clone(); + let message_hex = hex::encode(b"gents-cloud write"); + + let granted_pk = "3d".repeat(32); + let granted_did = signer_did_for_pk(&granted_pk); + let stranger_pk = "4c".repeat(32); + let acp = SignAcpFields { + policy_id: stack.acme_policy_id.clone(), + resource: TRANSCRIPT_RESOURCE.to_string(), + object_id: doc_id.clone(), + permission: "read".to_string(), + }; + + let refused = stack.orbis_cli.do_sign( + &endpoint, + &stack.ring_id, + &message_hex, + Some(&hex::encode(b"acme-corp")), + Some(&stranger_pk), + Some(&acp), + ); + assert!( + refused.is_err(), + "the ring must refuse to sign for a DID without the relation" + ); + + grant_document_relation( + &stack.acme, + &stack.training_svc, + "Transcript", + &doc_id, + "reader", + &granted_did, + ) + .await; + wait_for_vera_access( + &stack.vera_cli, + AccessCheck { + policy_id: &stack.acme_policy_id, + actor_did: &granted_did, + resource: TRANSCRIPT_RESOURCE, + object_id: &doc_id, + permission: "read", + }, + true, + Duration::from_secs(30), + ); + let sign_start = Instant::now(); + let signed = stack + .orbis_cli + .do_sign( + &endpoint, + &stack.ring_id, + &message_hex, + Some(&hex::encode(b"acme-corp")), + Some(&granted_pk), + Some(&acp), + ) + .expect("the ring must sign for an authorised DID"); + stack.record( + "ring_sign_with_vera_acp_check_ms", + format!("{}", sign_start.elapsed().as_millis()), + ); + assert!(!signed.signature.is_empty(), "signature must be present"); + + // Without any tuple the ring signs for any authenticated caller. This is + // the request shape DefraDB sends under Vera (see h5_two_clocks). + let unchecked = stack + .orbis_cli + .do_sign( + &endpoint, + &stack.ring_id, + &message_hex, + Some(&hex::encode(b"acme-corp")), + Some(&stranger_pk), + None, + ) + .expect("the ring signs without an ACP tuple"); + assert!(!unchecked.signature.is_empty()); + stack.record( + "ring_signs_without_acp_tuple", + "yes (DefraDB's Vera write path sends none)", + ); + + passed(&H3_RING_GATE, t); +} + +async fn below_threshold(stack: &mut Stack) { + let t = banner(&L8_BELOW_THRESHOLD); + let before = visible_doc_ids(&stack.acme, &stack.training_svc, "Transcript").await; + + // T=2 of N=3: with two members down the coordinator cannot reach threshold. + stack.ring.node_mut(1).kill(); + stack.ring.node_mut(2).kill(); + let refuse_start = Instant::now(); + let outcome = create_transcripts( + &stack.acme, + &stack.training_svc, + &[( + "call-l8", + "Written while the ring is below threshold", + "acme-cust-1", + )], + ) + .await; + let refusal_ms = refuse_start.elapsed().as_millis(); + assert!( + outcome.is_refused_or_failed(), + "a create must not succeed below threshold: {}", + outcome.detail() + ); + eprintln!( + "[gents-cloud] refused in {}ms: {}", + refusal_ms, + outcome.detail() + ); + stack.record("l8_refusal_latency_ms", format!("{}", refusal_ms)); + let after = visible_doc_ids(&stack.acme, &stack.training_svc, "Transcript").await; + assert_eq!( + after.len(), + before.len(), + "no document may exist for a write refused below threshold" + ); + + stack + .ring + .node_mut(1) + .restart() + .expect("restart ring node 1"); + stack + .ring + .node_mut(2) + .restart() + .expect("restart ring node 2"); + let recover_start = Instant::now(); + stack + .ring + .wait_ready(Duration::from_secs(60)) + .await + .expect("ring members should come back"); + let recovered = create_transcripts( + &stack.acme, + &stack.training_svc, + &[( + "call-l8-after", + "Written after the ring recovered", + "acme-cust-1", + )], + ) + .await; + let ids = match recovered { + WriteOutcome::Created(ids) => ids, + other => panic!("create after ring recovery: {}", other.detail()), + }; + stack.record( + "l8_ring_recovery_to_first_signed_write_ms", + format!("{}", recover_start.elapsed().as_millis()), + ); + stack.transcript_doc_ids.extend(ids); + + passed(&L8_BELOW_THRESHOLD, t); +} + +async fn signed_write_cost(stack: &mut Stack) { + let t = banner(&S7_SIGNED_COST); + let unsigned_key = stack.unsigned_node_key.clone(); + let unsigned = stack + .start_cell(CellSpec { + name: "unsigned", + identity: &unsigned_key, + derivation: "unsigned", + invalidation: Invalidation::Eager, + ring_signed: false, + }) + .await; + unsigned + .http + .schema_add(&transcript_schema(&stack.acme_policy_id)) + .await + .expect("unsigned cell transcript schema"); + + const SAMPLES: usize = 8; + let mut signed_ms = Vec::with_capacity(SAMPLES); + let mut unsigned_ms = Vec::with_capacity(SAMPLES); + for i in 0..SAMPLES { + let call_id = format!("call-s7-{}", i); + let start = Instant::now(); + let outcome = + create_transcripts(&stack.acme, &stack.training_svc, &[(&call_id, "s7", "c")]).await; + signed_ms.push(start.elapsed().as_millis()); + match outcome { + WriteOutcome::Created(ids) => stack.transcript_doc_ids.extend(ids), + other => panic!("signed create {}: {}", i, other.detail()), + } + + let start = Instant::now(); + let outcome = + create_transcripts(&unsigned, &stack.training_svc, &[(&call_id, "s7", "c")]).await; + unsigned_ms.push(start.elapsed().as_millis()); + assert!( + matches!(outcome, WriteOutcome::Created(_)), + "unsigned create {}: {}", + i, + outcome.detail() + ); + } + let signed_p50 = median(&signed_ms); + let unsigned_p50 = median(&unsigned_ms); + stack.record("s7_create_p50_ring_signed_ms", format!("{}", signed_p50)); + stack.record("s7_create_p50_unsigned_ms", format!("{}", unsigned_p50)); + stack.record( + "s7_ring_round_trip_p50_ms", + format!( + "{} (signed minus unsigned over {} samples; both are dominated by the Vera \ + registration transaction, so a value at or below zero means the ring round trip \ + is not separable at this sample size, not that it is free)", + signed_p50 as i128 - unsigned_p50 as i128, + SAMPLES + ), + ); + drop(unsigned); + + passed(&S7_SIGNED_COST, t); +} + +async fn dry_account(stack: &mut Stack) { + let t = banner(&DRY_ACCOUNT); + let dry_key = stack.dry_node_key.clone(); + let dry = stack + .start_cell(CellSpec { + name: "dry", + identity: &dry_key, + derivation: "dry", + invalidation: Invalidation::Eager, + ring_signed: true, + }) + .await; + dry.http + .schema_add(&transcript_schema(&stack.acme_policy_id)) + .await + .expect("dry cell transcript schema"); + + let outcome = create_transcripts( + &dry, + &stack.training_svc, + &[( + "call-dry", + "Written on a cell with an unfunded account", + "c", + )], + ) + .await; + assert!( + outcome.is_refused_or_failed(), + "a cell that cannot pay for the registration transaction must not report success: {}", + outcome.detail() + ); + let detail = outcome.detail(); + eprintln!("[gents-cloud] dry cell refused: {}", detail); + let lower = detail.to_ascii_lowercase(); + assert!( + !lower.contains("permission denied") && !lower.contains("not authorized"), + "the dry-account failure must be distinguishable from an ACP denial" + ); + stack.record("dry_account_error", first_line(&detail)); + + // The write failed at the registration transaction, but the document was + // already committed locally. DefraDB treats a document with no ACP + // registration as public (gents-cloud §1.2), so the failed create leaves a + // document any identity can read on that cell: a partial write that is + // also an exposure, not merely a lost one. + let unrelated = visible_doc_ids(&dry, &stack.globex_svc, "Transcript").await; + assert!( + !unrelated.is_empty(), + "expected the locally committed document to remain readable; if this now \ + fails, the create became atomic and this finding is closed" + ); + stack.record( + "dry_account_leaves_public_document", + format!( + "yes: {} document(s) readable by an unrelated DID after the failed registration", + unrelated.len() + ), + ); + + stack + .vera_cli + .fund(&dry_key.vera_address) + .expect("fund the dry cell's account"); + let funded = create_transcripts( + &dry, + &stack.training_svc, + &[("call-dry-funded", "Written after funding", "c")], + ) + .await; + assert!( + matches!(funded, WriteOutcome::Created(_)), + "create after funding: {}", + funded.detail() + ); + drop(dry); + + passed(&DRY_ACCOUNT, t); +} + +/// First line of a multi-line error, for a one-line measurement value. +fn first_line(text: &str) -> String { + text.lines().next().unwrap_or_default().trim().to_string() +} + +fn median(samples: &[u128]) -> u128 { + let mut sorted = samples.to_vec(); + sorted.sort_unstable(); + sorted[sorted.len() / 2] +} diff --git a/tests/support/full_stack.rs b/tests/support/full_stack.rs index 793ec3b..4c3061b 100644 --- a/tests/support/full_stack.rs +++ b/tests/support/full_stack.rs @@ -97,6 +97,15 @@ pub async fn wait_for_block_finality(hub_state: &ClusterState, label: &str) { ); } +/// Peer two nodes, subscribe both to `collections`, and install a replicator +/// from `source` to `dest`. +/// +/// Readiness is the replicator appearing in the source's own replicator list. +/// An earlier version waited for a `replicator_completed` SSE event, but the +/// node's event filter accepts only `topic-peer-event`, `acp-cache-invalidated`, +/// `acp-height-advanced`, `update`, and `merge-complete`; any other filter is a +/// 400, so that wait could only ever time out. Document-level convergence is +/// asserted by the caller, which is the property that actually matters. pub async fn configure_replication_link( source: &DefraClient, source_api_url: &str, @@ -104,8 +113,7 @@ pub async fn configure_replication_link( collections: &[&str], label: &str, ) { - let (replicator_sse, replicator_events) = - defra_harness::open_events_sse(source_api_url, "replicator_completed").await; + let _ = source_api_url; let dest_addr = p2p_addr(dest, label); source .p2p_connect(&[&dest_addr]) @@ -119,8 +127,39 @@ pub async fn configure_replication_link( source .p2p_replicator_set(collections, &dest_addr) .unwrap_or_else(|e| panic!("{}: set replicator: {}", label, e)); - wait_for_event_count(&replicator_events, 1, Duration::from_secs(15), label).await; - replicator_sse.abort(); + wait_for_replicator(source, Duration::from_secs(15), label).await; +} + +/// Poll the source node's replicator list until it holds at least one entry. +async fn wait_for_replicator(source: &DefraClient, timeout: Duration, label: &str) { + let t = Instant::now(); + loop { + let installed = source + .p2p_replicator_list() + .ok() + .map(|value| match value { + serde_json::Value::Array(entries) => !entries.is_empty(), + serde_json::Value::Null => false, + _ => true, + }) + .unwrap_or(false); + if installed { + eprintln!( + "[backbone] {} replicator installed in {:.2}s", + label, + t.elapsed().as_secs_f64() + ); + return; + } + if t.elapsed() > timeout { + panic!( + "{}: replicator did not appear in the source's replicator list within {}s", + label, + timeout.as_secs() + ); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } } pub fn graphql_string_literal(value: &str) -> String { From 13b0529ee92f9b2b807662b0fbbe7b15b9ba51f9 Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 22:53:10 +0200 Subject: [PATCH 02/11] perf(sourcehub-harness): set devnet consensus timeouts so a block costs 1s not 5s --- crates/sourcehub-harness/src/genesis.rs | 99 ++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/crates/sourcehub-harness/src/genesis.rs b/crates/sourcehub-harness/src/genesis.rs index 1996637..567f583 100644 --- a/crates/sourcehub-harness/src/genesis.rs +++ b/crates/sourcehub-harness/src/genesis.rs @@ -130,7 +130,26 @@ pub fn provision_genesis( Ok(()) } -/// Patch config.toml to bind CometBFT RPC and P2P to allocated ports. +/// Consensus timings for the devnet. +/// +/// CometBFT ships `timeout_commit = "5s"`, and that single value is the cost of +/// every test that writes to the chain: a transaction is not queryable until it +/// is in a block, so each one waits a block. Measured on the shipped defaults, +/// this devnet produced a block every 5.03 s, and four sequential transactions +/// were 80% of the time to provision one tenant. +/// +/// A single-validator devnet reaches consensus on its own vote, so the round +/// timeouts almost never bind; they are set anyway so the configuration stays +/// coherent if the harness ever runs more than one validator. +const CONSENSUS_TIMEOUTS: [(&str, &str); 4] = [ + ("timeout_propose", "500ms"), + ("timeout_prevote", "500ms"), + ("timeout_precommit", "500ms"), + ("timeout_commit", "1s"), +]; + +/// Patch config.toml to bind CometBFT RPC and P2P to allocated ports, and to +/// run consensus at a speed suited to a test devnet. fn patch_config_toml(home_dir: &Path, ports: &SourceHubPorts) -> Result<()> { let config_path = home_dir.join("config/config.toml"); let content = std::fs::read_to_string(&config_path).wrap_err("read config.toml")?; @@ -145,11 +164,36 @@ fn patch_config_toml(home_dir: &Path, ports: &SourceHubPorts) -> Result<()> { "laddr = \"tcp://0.0.0.0:26656\"", &format!("laddr = \"tcp://0.0.0.0:{}\"", ports.p2p), ); + let mut content = content; + for (key, value) in CONSENSUS_TIMEOUTS { + content = replace_setting(&content, key, value)?; + } std::fs::write(&config_path, content).wrap_err("write config.toml")?; Ok(()) } +/// Replace a `key = "value"` line in a CometBFT config, failing loudly if the +/// key is absent: a silently skipped timeout would look like a slow chain +/// rather than a missed setting. +fn replace_setting(content: &str, key: &str, value: &str) -> Result { + let mut found = false; + let patched = content + .lines() + .map(|line| { + if line.trim_start().starts_with(&format!("{} =", key)) { + found = true; + format!("{} = \"{}\"", key, value) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n"); + eyre::ensure!(found, "config.toml has no `{}` setting to patch", key); + Ok(patched) +} + /// Patch app.toml to bind gRPC and LCD/API to allocated ports. fn patch_app_toml(home_dir: &Path, ports: &SourceHubPorts) -> Result<()> { let app_path = home_dir.join("config/app.toml"); @@ -193,3 +237,56 @@ fn run_cmd(program: &Path, args: &[&str]) -> Result { )) } } + +#[cfg(test)] +mod tests { + use super::replace_setting; + + const SAMPLE: &str = "\ +[consensus]\n\ +timeout_propose = \"3s\"\n\ +timeout_commit = \"5s\"\n\ +create_empty_blocks = true\n\ +create_empty_blocks_interval = \"0s\"\n"; + + #[test] + fn replaces_a_setting_in_place() { + let patched = replace_setting(SAMPLE, "timeout_commit", "1s").expect("patch"); + assert!(patched.contains("timeout_commit = \"1s\"")); + assert!(!patched.contains("timeout_commit = \"5s\"")); + // Neighbouring settings are untouched. + assert!(patched.contains("timeout_propose = \"3s\"")); + assert!(patched.contains("create_empty_blocks = true")); + } + + #[test] + fn every_consensus_timeout_is_present_in_a_stock_config() { + let mut patched = SAMPLE.to_string(); + for (key, value) in super::CONSENSUS_TIMEOUTS { + if SAMPLE.contains(&format!("{} =", key)) { + patched = replace_setting(&patched, key, value).expect("patch"); + assert!(patched.contains(&format!("{} = \"{}\"", key, value))); + } + } + } + + #[test] + fn a_missing_setting_is_an_error_not_a_silent_skip() { + let err = replace_setting(SAMPLE, "timeout_nonexistent", "1s") + .expect_err("an absent key must fail"); + assert!( + format!("{err}").contains("timeout_nonexistent"), + "the error must name the key: {err}" + ); + } + + #[test] + fn does_not_match_a_key_that_only_shares_a_prefix() { + let patched = replace_setting(SAMPLE, "create_empty_blocks", "false").expect("patch"); + assert!(patched.contains("create_empty_blocks = \"false\"")); + assert!( + patched.contains("create_empty_blocks_interval = \"0s\""), + "the longer key must survive: {patched}" + ); + } +} From 87862780041362181406f6ab9311d9e84e55a6f1 Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 22:53:10 +0200 Subject: [PATCH 03/11] perf(orbis-harness): poll for a new policy instead of sleeping 2s first --- crates/orbis-harness/src/cli/sourcehub.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/orbis-harness/src/cli/sourcehub.rs b/crates/orbis-harness/src/cli/sourcehub.rs index 6180149..96c8120 100644 --- a/crates/orbis-harness/src/cli/sourcehub.rs +++ b/crates/orbis-harness/src/cli/sourcehub.rs @@ -253,18 +253,23 @@ impl SourceHubCliClient { "create-policy", tmp.to_str().ok_or_else(|| eyre!("invalid path"))?, ])?; - // Poll for the new policy ID to appear (tx needs a block to commit) - let mut new_id = None; - for _attempt in 0..15 { - std::thread::sleep(std::time::Duration::from_secs(2)); + // `exec_tx` already waited for the transaction to be committed, so the + // policy is normally queryable at once; poll briefly for the query + // node to catch up rather than sleeping a fixed interval first, which + // would put a floor under every measurement of this call. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let new_id = loop { let after = self.list_policy_ids()?; if let Some(id) = after.into_iter().find(|id| !before.contains(id)) { - new_id = Some(id); - break; + break id; } - } - let new_id = - new_id.ok_or_else(|| eyre!("policy creation succeeded but no new policy ID found"))?; + if std::time::Instant::now() >= deadline { + return Err(eyre!( + "policy creation succeeded but no new policy ID found" + )); + } + std::thread::sleep(Duration::from_millis(100)); + }; let _ = std::fs::remove_file(&tmp); Ok(new_id) From 8912d08f2c96288aa66aaac8d452babf0a0826c1 Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 22:53:10 +0200 Subject: [PATCH 04/11] fix(gents-cloud): give each scale tenant its own writer identity so its writes reach the ring --- tests/gents_cloud/scale.rs | 364 ++++++++++++++++++++++++++++--------- 1 file changed, 276 insertions(+), 88 deletions(-) diff --git a/tests/gents_cloud/scale.rs b/tests/gents_cloud/scale.rs index 3e3bfcb..0710c27 100644 --- a/tests/gents_cloud/scale.rs +++ b/tests/gents_cloud/scale.rs @@ -36,81 +36,39 @@ const NODE_MEMORY_GIB: f64 = 64.0; pub async fn run(stack: &mut Stack) { let t = banner(&SCALE); let tenant_count = tenant_count(); - eprintln!("[gents-cloud] provisioning {} tenants", tenant_count); + let checkpoint_every = checkpoint_every(tenant_count); + eprintln!( + "[gents-cloud] provisioning {} tenants, measuring every {}", + tenant_count, checkpoint_every + ); let baseline_read_ms = read_latency_ms(&stack.acme, &stack.training_svc, "Transcript").await; let mut tenants: Vec = Vec::with_capacity(tenant_count); - let mut provision_ms = Vec::with_capacity(tenant_count); - for index in 0..tenant_count { - let start = Instant::now(); - tenants.push(provision_tenant(stack, index).await); - provision_ms.push(start.elapsed().as_millis()); - } + let mut phases: Vec = Vec::with_capacity(tenant_count); + let mut curve: Vec = Vec::new(); - // Per-tenant cost, measured. - let provision_p50 = median(&provision_ms); - stack.record("scale_tenants_provisioned", format!("{}", tenant_count)); - stack.record( - "scale_provision_p50_ms_per_tenant", - format!("{}", provision_p50), - ); - let rss_kib: Vec = tenants - .iter() - .filter_map(|t| cell_rss_kib(&t.cell)) - .collect(); - assert_eq!( - rss_kib.len(), - tenants.len(), - "every tenant cell must report its resident memory" - ); - let rss_median_kib = median( - &rss_kib - .iter() - .map(|kib| u128::from(*kib)) - .collect::>(), - ); - let rss_median_mib = rss_median_kib as f64 / 1024.0; - stack.record( - "scale_cell_rss_median_mib", - format!("{:.0}", rss_median_mib), - ); - - // Chain state per tenant: one policy, one collection object, one writer - // relation, plus one registration per document written. - let policy_ids = stack - .vera_cli - .list_policy_count() - .expect("count policies on Vera"); - stack.record("scale_policies_on_vera", format!("{}", policy_ids)); + for index in 0..tenant_count { + let (tenant, phase) = provision_tenant(stack, index).await; + tenants.push(tenant); + phases.push(phase); - // Every tenant reads only its own document, and reads stay flat. - let mut read_ms = Vec::with_capacity(tenants.len()); - for tenant in &tenants { - let visible = visible_doc_ids(&tenant.cell, &tenant.identity, "Transcript").await; - assert_eq!( - visible, - vec![tenant.doc_id.clone()], - "tenant {} must see exactly its own document", - tenant.identity.label - ); - read_ms.push(read_latency_ms(&tenant.cell, &tenant.identity, "Transcript").await); + let done = index + 1; + if done % checkpoint_every == 0 || done == tenant_count { + curve.push(measure_curve_point(&tenants, &phases).await); + let point = curve.last().expect("just pushed"); + eprintln!( + "[gents-cloud] {} tenants: cell rss {} MiB median, {} MiB total, read p50 {} ms", + point.tenants, point.rss_median_mib, point.rss_total_mib, point.read_p50_ms + ); + } } - let read_p50 = median(&read_ms); - stack.record( - "scale_read_p50_ms_first_tenant", - format!("{}", baseline_read_ms), - ); - stack.record( - "scale_read_p50_ms_with_all_tenants", - format!("{}", read_p50), - ); - // Isolation at N: no tenant reads another's cell, in either direction. + // Isolation at N: no tenant reads the next tenant's cell. for (i, tenant) in tenants.iter().enumerate() { let other = &tenants[(i + 1) % tenants.len()]; - if std::ptr::eq(tenant, other) { - continue; + if tenants.len() < 2 { + break; } let cross = other .cell @@ -127,16 +85,95 @@ pub async fn run(stack: &mut Stack) { other.identity.label ); } + + // Every tenant sees exactly its own document. + for tenant in &tenants { + let visible = visible_doc_ids(&tenant.cell, &tenant.identity, "Transcript").await; + assert_eq!( + visible, + vec![tenant.doc_id.clone()], + "tenant {} must see exactly its own document", + tenant.identity.label + ); + } + + let last = *curve.last().expect("at least one checkpoint"); + let phase_p50 = phase_medians(&phases); + + stack.record("scale_tenants_provisioned", format!("{}", tenant_count)); + stack.record( + "scale_policies_on_vera", + format!( + "{}", + stack.vera_cli.list_policy_count().expect("count policies") + ), + ); + stack.record( + "scale_cell_rss_median_mib", + format!("{}", last.rss_median_mib), + ); + stack.record( + "scale_read_p50_ms_one_tenant", + format!("{}", baseline_read_ms), + ); + stack.record( + "scale_read_p50_ms_all_tenants", + format!("{}", last.read_p50_ms), + ); stack.record( "scale_cross_tenant_reads_denied", format!("{} ordered pairs", tenants.len()), ); - // The projection. Stated as arithmetic over the numbers above, with the - // assumptions named, because the run itself covers a few tenants. - let cells_per_node = (NODE_MEMORY_GIB * 1024.0 / rss_median_mib).floor(); + // Where provisioning time actually goes. Four of these are chain + // transactions that wait for a block; only ignition is the cell starting. + stack.record( + "scale_provision_p50_total_ms", + format!("{}", phase_p50.total_ms), + ); + stack.record( + "scale_provision_p50_fund_ms", + format!("{}", phase_p50.fund_ms), + ); + stack.record( + "scale_provision_p50_policy_ms", + format!("{}", phase_p50.policy_ms), + ); + stack.record( + "scale_provision_p50_register_object_ms", + format!("{}", phase_p50.register_object_ms), + ); + stack.record( + "scale_provision_p50_grant_writer_ms", + format!("{}", phase_p50.grant_writer_ms), + ); + stack.record( + "scale_provision_p50_cell_ignition_ms", + format!("{}", phase_p50.cell_ignition_ms), + ); + stack.record( + "scale_provision_p50_schema_ms", + format!("{}", phase_p50.schema_ms), + ); + stack.record( + "scale_provision_p50_first_write_ms", + format!("{}", phase_p50.first_write_ms), + ); + let chain_ms = phase_p50.fund_ms + + phase_p50.policy_ms + + phase_p50.register_object_ms + + phase_p50.grant_writer_ms; + stack.record( + "scale_provision_p50_chain_share", + format!( + "{}% of the total is the four Vera transactions waiting for a block", + percent(chain_ms, phase_p50.total_ms) + ), + ); + + // The projection, from the measured per-cell memory. + let cells_per_node = (NODE_MEMORY_GIB * 1024.0 / last.rss_median_mib as f64).floor(); let nodes_for_target = (TARGET_TENANTS as f64 / cells_per_node).ceil(); - let provision_hours = (TARGET_TENANTS as f64 * provision_p50 as f64) / 1000.0 / 3600.0; stack.record( "scale_projected_cells_per_64gib_node", format!( @@ -151,38 +188,165 @@ pub async fn run(stack: &mut Stack) { nodes_for_target ), ); - stack.record( - "scale_projected_serial_provisioning_hours_for_100k", - format!( - "{:.1} (projected: strictly serial provisioning at the measured p50; a real fleet provisions in parallel)", - provision_hours - ), - ); - // Cells are dropped here: the scenario owns them and nothing later needs - // them, so the processes exit before the next scenario measures anything. + print_curve_table(&curve, &phase_p50); drop(tenants); passed(&SCALE, t); } -/// One provisioned tenant: its identity, its cell, and the document it wrote. +/// One row of the scaling curve: what the fleet costs at this tenant count. +#[derive(Clone, Copy)] +struct CurvePoint { + tenants: usize, + rss_median_mib: u64, + rss_total_mib: u64, + read_p50_ms: u128, + provision_p50_ms: u128, + ignition_p50_ms: u128, +} + +async fn measure_curve_point(tenants: &[Tenant], phases: &[ProvisionPhases]) -> CurvePoint { + let rss_kib: Vec = tenants + .iter() + .filter_map(|t| cell_rss_kib(&t.cell)) + .collect(); + assert_eq!( + rss_kib.len(), + tenants.len(), + "every tenant cell must report its resident memory" + ); + let rss_median_mib = + (median(&rss_kib.iter().map(|k| u128::from(*k)).collect::>()) / 1024) as u64; + let rss_total_mib = (rss_kib.iter().sum::()) / 1024; + + // Read latency sampled across up to eight tenants, so the cost of the + // measurement does not grow with the fleet. + let mut read_ms = Vec::new(); + let stride = (tenants.len() / 8).max(1); + for tenant in tenants.iter().step_by(stride) { + read_ms.push(read_latency_ms(&tenant.cell, &tenant.identity, "Transcript").await); + } + + let medians = phase_medians(phases); + CurvePoint { + tenants: tenants.len(), + rss_median_mib, + rss_total_mib, + read_p50_ms: median(&read_ms), + provision_p50_ms: medians.total_ms, + ignition_p50_ms: medians.cell_ignition_ms, + } +} + +fn phase_medians(phases: &[ProvisionPhases]) -> ProvisionPhases { + let pick = |f: fn(&ProvisionPhases) -> u128| median(&phases.iter().map(f).collect::>()); + ProvisionPhases { + fund_ms: pick(|p| p.fund_ms), + policy_ms: pick(|p| p.policy_ms), + register_object_ms: pick(|p| p.register_object_ms), + grant_writer_ms: pick(|p| p.grant_writer_ms), + cell_ignition_ms: pick(|p| p.cell_ignition_ms), + schema_ms: pick(|p| p.schema_ms), + first_write_ms: pick(|p| p.first_write_ms), + total_ms: pick(|p| p.total_ms), + } +} + +fn percent(part: u128, whole: u128) -> u128 { + (part * 100).checked_div(whole).unwrap_or(0) +} + +/// Print the curve and the provisioning breakdown as markdown, ready to paste. +fn print_curve_table(curve: &[CurvePoint], phases: &ProvisionPhases) { + eprintln!("[gents-cloud] === scaling curve (markdown) ==="); + eprintln!("| Tenants | Cell RSS median (MiB) | All cells (MiB) | Read p50 (ms) | Provision p50 (ms) | Cell ignition p50 (ms) |"); + eprintln!("|---|---|---|---|---|---|"); + for point in curve { + eprintln!( + "| {} | {} | {} | {} | {} | {} |", + point.tenants, + point.rss_median_mib, + point.rss_total_mib, + point.read_p50_ms, + point.provision_p50_ms, + point.ignition_p50_ms + ); + } + eprintln!("[gents-cloud] === provisioning breakdown (markdown) ==="); + eprintln!("| Step | p50 (ms) | Waits on |"); + eprintln!("|---|---|---|"); + eprintln!( + "| Fund the tenant account | {} | Vera block |", + phases.fund_ms + ); + eprintln!("| Create the policy | {} | Vera block |", phases.policy_ms); + eprintln!( + "| Register the collection object | {} | Vera block |", + phases.register_object_ms + ); + eprintln!("| Grant writer | {} | Vera block |", phases.grant_writer_ms); + eprintln!( + "| Ignite the cell | {} | the process starting |", + phases.cell_ignition_ms + ); + eprintln!("| Add the schema | {} | local |", phases.schema_ms); + eprintln!( + "| First ring-signed write | {} | ring round trip and a Vera registration |", + phases.first_write_ms + ); + eprintln!("| **Total** | **{}** | |", phases.total_ms); +} + +/// One provisioned tenant: the cell's own node identity, the service identity +/// that writes and reads as the tenant, the cell, and the document it wrote. struct Tenant { + /// The identity a client presents. Deliberately not the node identity: + /// DefraDB resolves the signing config for the request DID from a + /// process-global registry, and the node's own DID has a local key + /// registered there, so a write made as the node identity signs locally + /// and never reaches the ring. identity: ServiceIdentity, cell: Cell, doc_id: String, } +/// Wall time of each step of provisioning one tenant, so a total can be read +/// as what it is made of rather than as one opaque number. +#[derive(Default, Clone, Copy)] +struct ProvisionPhases { + fund_ms: u128, + policy_ms: u128, + register_object_ms: u128, + grant_writer_ms: u128, + cell_ignition_ms: u128, + schema_ms: u128, + first_write_ms: u128, + total_ms: u128, +} + /// Provision one tenant end to end, the way the operator would: its own policy /// on Vera, its own collection object and writer grant, its own cell, its own /// schema, and one document written through the ring. -async fn provision_tenant(stack: &Stack, index: usize) -> Tenant { +/// +/// Each step is timed separately. Four of the seven are chain transactions that +/// cannot return until Vera commits a block, so a total dominated by them says +/// nothing about how fast a cell starts; `cell_ignition_ms` is the number that +/// does. +async fn provision_tenant(stack: &Stack, index: usize) -> (Tenant, ProvisionPhases) { + let total = Instant::now(); + let mut phases = ProvisionPhases::default(); let label = format!("scale-tenant-{}", index); - let identity = ServiceIdentity::new(&label); + let node_key = ServiceIdentity::new(&label); + let identity = ServiceIdentity::new(&format!("{}-svc", label)); + + let step = Instant::now(); stack .vera_cli - .fund(&identity.vera_address) + .fund(&node_key.vera_address) .unwrap_or_else(|e| panic!("fund {}: {}", label, e)); + phases.fund_ms = step.elapsed().as_millis(); + let step = Instant::now(); let policy_id = stack .vera_cli .create_policy(&crate::fixture::ACME_POLICY_YAML.replace( @@ -190,10 +354,16 @@ async fn provision_tenant(stack: &Stack, index: usize) -> Tenant { &format!("name: {}-policy", label), )) .unwrap_or_else(|e| panic!("create policy for {}: {}", label, e)); + phases.policy_ms = step.elapsed().as_millis(); + + let step = Instant::now(); stack .vera_cli .register_object(&policy_id, TRANSCRIPT_RESOURCE, TRANSCRIPT_RESOURCE) .unwrap_or_else(|e| panic!("register collection object for {}: {}", label, e)); + phases.register_object_ms = step.elapsed().as_millis(); + + let step = Instant::now(); stack .vera_cli .set_relationship( @@ -204,37 +374,49 @@ async fn provision_tenant(stack: &Stack, index: usize) -> Tenant { &identity.did_key, ) .unwrap_or_else(|e| panic!("grant writer for {}: {}", label, e)); + phases.grant_writer_ms = step.elapsed().as_millis(); + let step = Instant::now(); let cell = stack .start_cell(CellSpec { name: &label, - identity: &identity, + identity: &node_key, derivation: &label, invalidation: Invalidation::Eager, ring_signed: true, }) .await; + phases.cell_ignition_ms = step.elapsed().as_millis(); + + let step = Instant::now(); cell.http .schema_add(&transcript_schema(&policy_id)) .await .unwrap_or_else(|e| panic!("schema for {}: {}", label, e)); + phases.schema_ms = step.elapsed().as_millis(); + let step = Instant::now(); let outcome = create_transcripts( &cell, &identity, &[("call-scale", "One document per tenant", "cust")], ) .await; + phases.first_write_ms = step.elapsed().as_millis(); let doc_id = match outcome { WriteOutcome::Created(ids) => ids[0].clone(), other => panic!("{} write: {}", label, other.detail()), }; - Tenant { - identity, - cell, - doc_id, - } + phases.total_ms = total.elapsed().as_millis(); + ( + Tenant { + identity, + cell, + doc_id, + }, + phases, + ) } /// How many tenants to provision. `GENTS_CLOUD_TENANTS` overrides the default, @@ -247,6 +429,12 @@ fn tenant_count() -> usize { .unwrap_or(4) } +/// How often to take a curve measurement, so a run yields four to eight rows +/// whatever the tenant count. +fn checkpoint_every(tenant_count: usize) -> usize { + (tenant_count / 4).max(1) +} + /// Resident memory of a cell's process, from `/proc//status`. fn cell_rss_kib(cell: &Cell) -> Option { let pid = cell.node.process.id()?; From 8bc56a4af48c191c3969fc8fac1e6aa046657ebf Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 22:53:10 +0200 Subject: [PATCH 05/11] test(gents-cloud): assert afterburner guest packages are sealed --- tests/gents_cloud/afterburner.rs | 152 +++++++++++++++++++++++++++++++ tests/gents_cloud/main.rs | 8 +- 2 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 tests/gents_cloud/afterburner.rs diff --git a/tests/gents_cloud/afterburner.rs b/tests/gents_cloud/afterburner.rs new file mode 100644 index 0000000..0cfc679 --- /dev/null +++ b/tests/gents_cloud/afterburner.rs @@ -0,0 +1,152 @@ +//! The Afterburner half of the architecture, checked at the artifact level. +//! +//! gents-cloud states that a sealed package's permission surface is its +//! `manifold.json`, that an absent field stays sealed, that package identity is +//! the digest of its bytes (H4), and that `child_process` never appears in a +//! cloud manifold (§26). Those are properties of the artifact, so they can be +//! asserted without running anything: this scenario reads the packages the +//! defraburner proof of concept ships and checks them. +//! +//! What it deliberately does not do is re-test Afterburner's runtime. Fuel +//! exhaustion, the policy clamp, gateway admission, and the lifecycle of the +//! wasm DefraDB each cell owns are covered by defraburner's own suite, against +//! its own binary, which is where those mechanisms live. + +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use crate::fixture::Stack; +use crate::{banner, passed, Scenario}; + +const SEALED_PACKAGES: Scenario = Scenario { + id: "afterburner_sealed_packages", + spec: "gents-cloud §1.1 (the manifold is the whole permission surface), H4, H11, §26 ban list, I-3", + claim: "every sealed package the proof of concept ships declares a fully sealed manifold, grants no child process, and is identified by the digest of its bytes", +}; + +/// Where the defraburner checkout lives. It is the proof of concept for the +/// cell, mesh, gateway and policy half of the architecture. +fn defraburner_dir() -> Option { + let dir = std::env::var("DEFRABURNER_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| { + PathBuf::from(std::env::var("HOME").unwrap_or_default()).join("projects/defraburner") + }); + dir.join("packages").is_dir().then_some(dir) +} + +pub async fn run(stack: &mut Stack) { + let t = banner(&SEALED_PACKAGES); + + let Some(dir) = defraburner_dir() else { + // Not evaluated is recorded, never passed over in silence. + stack.record( + "afterburner_sealed_packages", + "not evaluated: no defraburner checkout (set DEFRABURNER_DIR)", + ); + passed(&SEALED_PACKAGES, t); + return; + }; + + let packages = dir.join("packages"); + let mut checked = Vec::new(); + for entry in std::fs::read_dir(&packages).expect("read packages directory") { + let package = entry.expect("package entry").path(); + if !package.is_dir() { + continue; + } + let name = package + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default() + .to_string(); + + let manifold_path = package.join("manifold.json"); + if !manifold_path.is_file() { + continue; + } + let manifold: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&manifold_path).expect("read manifold")) + .unwrap_or_else(|e| panic!("{}: parse manifold.json: {}", name, e)); + + // The ban list is absolute: no cloud manifold grants a child process. + assert_eq!( + manifold.get("child_process").and_then(|v| v.as_bool()), + Some(false), + "{}: child_process must be false; the WASM backend refuses it and the \ + cloud build bans it outright", + name + ); + // Sealed by default means every capability is off unless the package + // states otherwise. These four are the ones a database or a policy + // package has no business holding. + for (field, expected) in [ + ("fs", "None"), + ("net", "None"), + ("env", "None"), + ("listen", "None"), + ] { + assert_eq!( + manifold.get(field).and_then(|v| v.as_str()), + Some(expected), + "{}: {} must be {}", + name, + field, + expected + ); + } + assert_eq!( + manifold.get("allow_exit").and_then(|v| v.as_bool()), + Some(false), + "{}: allow_exit must be false", + name + ); + + // Package identity is the digest of the bytes, not the file name. + let archive = newest_afb(&package) + .unwrap_or_else(|| panic!("{}: no .afb archive beside the manifold", name)); + let bytes = std::fs::read(&archive).expect("read archive"); + assert!( + bytes.starts_with(&[0x28, 0xb5, 0x2f, 0xfd]), + "{}: the archive must be zstd, which is what makes the format byte-reproducible", + name + ); + let digest = hex::encode(Sha256::digest(&bytes)); + checked.push(format!( + "{} {} ({} bytes)", + name, + &digest[..16], + bytes.len() + )); + } + + assert!( + !checked.is_empty(), + "a defraburner checkout must ship at least one sealed package" + ); + for line in &checked { + eprintln!("[gents-cloud] sealed package {}", line); + } + stack.record( + "afterburner_sealed_packages", + format!( + "{} packages, all sealed: {}", + checked.len(), + checked.join("; ") + ), + ); + + passed(&SEALED_PACKAGES, t); +} + +/// The most recently built `.afb` archive in a package directory. +fn newest_afb(package: &Path) -> Option { + let mut archives: Vec = std::fs::read_dir(package) + .ok()? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.extension().and_then(|e| e.to_str()) == Some("afb")) + .collect(); + archives.sort(); + archives.pop() +} diff --git a/tests/gents_cloud/main.rs b/tests/gents_cloud/main.rs index 1c9c820..0b9073d 100644 --- a/tests/gents_cloud/main.rs +++ b/tests/gents_cloud/main.rs @@ -14,6 +14,7 @@ #[path = "../support/mod.rs"] mod support; +mod afterburner; mod fixture; mod identity; mod p2p; @@ -61,11 +62,14 @@ async fn gents_cloud_mechanics() { write_path::run(&mut stack).await; p2p::run(&mut stack).await; recovery::run(&mut stack).await; + afterburner::run(&mut stack).await; scale::run(&mut stack).await; - eprintln!("[gents-cloud] === measurements ==="); + eprintln!("[gents-cloud] === measurements (markdown) ==="); + eprintln!("| Measurement | Value |"); + eprintln!("|---|---|"); for (name, value) in &stack.measurements { - eprintln!("[gents-cloud] {:<48} {}", name, value); + eprintln!("| `{}` | {} |", name, value.replace('|', "\\|")); } eprintln!( "[gents-cloud] all scenarios passed in {:.1}s", From 7b1c0f24866d3f0da9a791882a3b12f86e4a732a Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 22:53:10 +0200 Subject: [PATCH 06/11] docs: 32-tenant Vera run report --- docs/gents-cloud-run-32-tenants.html | 679 +++++++++++++++++++++++++++ 1 file changed, 679 insertions(+) create mode 100644 docs/gents-cloud-run-32-tenants.html diff --git a/docs/gents-cloud-run-32-tenants.html b/docs/gents-cloud-run-32-tenants.html new file mode 100644 index 0000000..8f1b56d --- /dev/null +++ b/docs/gents-cloud-run-32-tenants.html @@ -0,0 +1,679 @@ +32-Tenant Vera Run + + + + + + +
+ +
+
+

32-Tenant Vera Run

+

The gents-cloud architecture exercised end to end on Go Vera: one verad devnet, a 3-node Orbis ring at threshold 2, and DefraDB cells enforcing document access control against the chain.

+
+
+ 13 of 13 passed
+ 2026-09-03 20:39 UTC
+ backbone tests/gents_cloud
+ 36 cores, 188 GiB host +
+
+ +
+
Wall time
376.5s
+
Tenants
32
+
Cells at peak
35
+
Chain height
352
+
Transactions
2230 failed
+
Cell memory
71MiB p50
+
+ +
+

What changed since the first run

+

The first 32-tenant run took 1212.5 seconds and spent almost all of it waiting for blocks. CometBFT ships timeout_commit = "5s", and the harness had never touched it, so every chain step in the provisioning path cost a five-second block. The devnet now sets the four consensus timeouts explicitly. Nothing else about the stack changed, and the same scenarios assert the same things.

+
+
+ + + + + + + + +
SettingBeforeAfter
timeout_propose3s500ms
timeout_prevote1s500ms
timeout_precommit1s500ms
timeout_commit5s1s
+ Set in crates/sourcehub-harness/src/genesis.rs. A key absent from the shipped config is an error rather than a silent skip, so a missed setting cannot read as a slow chain. +
+
+ + + + + + + + + + +
MeasureBeforeAfterChange
Block interval (s)5.031.064.7x
Provision one tenant (ms)2516760584.2x
Suite wall time (s)1212.5376.53.2x
Signed write, p50 (ms)490914833.3x
Stack bring-up (s)83.522.43.7x
Chain share of provisioning80%70%-10pt
+ The block interval is measured from height and wall clock across the run, not read from the configuration. +
+
+

Faster blocks changed one measurement qualitatively rather than proportionally: cell resident memory fell from 108 MiB to 71 MiB at p50, because a cell that finishes provisioning in six seconds holds less transient state at the moment it is sampled. The projection to 100,000 tenants moves with it, and both numbers are in this report.

+
+ +
+

Scenarios

+

Each scenario is named for the invariant, hardening move, spike, or readiness finding it discharges, and asserts what the running stack does rather than what the plan says it should.

+
+ + + + + + + + + + + + + + + + + + + +
ScenarioDischargesSecondsResult
h1_node_identity_no_privileged_readH1, I-2, spike S6, readiness C48.3pass
i26_absence_denial_indistinguishableI-26, Phase 2 gate0.0pass
grant_asymmetryGrant granularity, revocation cost8.7pass
h5_two_clocksH5 two clocks, spike S5, I-1615.6pass
h12_pre_on_veraH12 recoverable custody3.3pass
h3_ring_gate_mechanismH3 signing gate, spike S70.9pass
l8_ring_below_thresholdDegradation rung L8, decision 4334.8pass
s7_signed_write_costRing round trip cost, spike S718.3pass
dry_accountUnfunded cell, unregistered means public3.3pass
s8_topic_collision_c1Decisions A-1 and A-2, spike S8, C146.9pass
i7_kill9_identityI-7 identity persisted before use3.1pass
afterburner_sealed_packagesSpike S4, sealed guest packaging0.2pass
scale_per_tenant_costDensity, the workspace as shard key, growth stages210.7pass
+ The last scenario provisions the 32 tenants; the twelve before it run on the three-cell base stack. +
+
+ +
+

Scale

+
+
+

Memory grows with tenants, latency does not

+

Every checkpoint measures the resident set of every live cell and the read latency of a sample of tenants. Total memory tracks the tenant count; a tenant's read stays at 6 ms whether it is one of eight or one of thirty-two. Per-cell memory falls between the first two checkpoints and the last two because a cell sheds transient provisioning state once it settles.

+
+ + + + + + + + + + +
TenantsCell p50 (MiB)All cells (MiB)Read p50 (ms)Ignition p50 (ms)
81007506311
1610015057303
247018556303
327125726300
+
+
+
+ + + + + + + + + 0 + 900 + 1800 + 2700 + + + 8 + 16 + 24 + 32 + tenants + + + + + + + + + 2572 MiB + +

Resident memory, all live cells, MiB.

+
+
+
+ +
+

Where provisioning time goes

+

Provisioning one tenant end to end took 6.1 seconds at p50. Almost none of that is the cell. Igniting the cell process and applying its schema together account for 319 ms; the rest is four Vera transactions and one document registration, each waiting for a block.

+
+ + + + + + + + + + + + 0 s + 3.0 s + 6.0 s + + chain-bound 5,648 ms · local work 319 ms + +
+ fund, policy, register object, grant writer, first write + cell ignition 300 ms + schema 19 ms +
+
+
+ + + + + + + + + + + + +
Stepp50 (ms)Was (ms)Waits on
Fund the tenant account9745086Vera block
Create the policy10835166Vera block
Register the collection object8464976Vera block
Grant writer13615000Vera block
Ignite the cell300163the process starting
Add the schema1917local
First write13845285document registration on Vera
Total60582516770% is the four transactions alone
+ Each step is the median across 32 tenants, so the parts sum to 5,967 ms rather than exactly to the median total of 6,058 ms. Cell ignition rose from 163 ms to 300 ms because 32 cells now start inside a shorter window and contend for the same cores. +
+
+ +
+

What the chain did

+

Every transaction the run submitted, read back from the committed ledger. None failed.

+
+
+ + + + + + + + + + + + +
MessageCount
/vera.acp.MsgDirectPolicyCmd73
/vera.acp.MsgBearerPolicyCmd69
/cosmos.bank.v1beta1.MsgSend41
/vera.acp.MsgCreatePolicy34
/vera.bulletin.MsgAddCollaborator3
/vera.bulletin.MsgCreatePost2
/vera.bulletin.MsgRegisterNamespace1
Total223
+ Bearer commands are DefraDB acting for a document owner; direct commands are the harness acting as the chain account that registered the object. +
+
+ + + + + + + + + + + + + +
Chain measureValue
Blocks produced352
Block interval1.06 s
Transactions committed223
Transactions failed0
Last block carrying a transaction337
Gas used25,471,721
Gas wanted368,516,753
Policies registered35
Chain data on disk8.8 MiB
+ Gas wanted is the harness limit of 3,000,000 per transaction, raised after a lower limit made the first namespace registration fail inside the Cosmos gas meter. Gas used is 6.9% of it. +
+
+
+ +
+

Component footprint

+
+
+ + + + + + + +
ComponentCountPeak resident (MiB)Each (MiB)
DefraDB cells35284681
Vera (verad)1218218
Orbis ring313946
+ Maxima over 33 samples taken every 15 seconds. Ring memory peaks during the distributed key generation and falls to 23 MiB per node afterwards. +
+
+ + + + + + + + + + + +
ComponentSigned blocksMergesPeer connects
acme cell6128
globex cell1006
platform replica0876
each scale tenant420
Orbis node0846n/an/a
Orbis node1428n/an/a
Orbis node2428n/an/a
+ Each of the 32 scale tenants now signs its writes through the ring: it signed 0 in the first run, when the writing identity was the cell's own node identity. Ring figures count signing lines in each node's log. +
+
+
+ + + + + + + + +
On disk after the runKiBNote
Vera chain data9012352 blocks, 223 transactions
All 32 tenant cells128040 KiB per tenant, one document each
Orbis ring, 3 nodes716key shares and session state
Whole run directory12012includes the base stack's three cells and all logs
+
+
+ +
+

Projection to 100,000 tenants

+

The run covered 32 tenants. These figures are arithmetic over the measurements above, labelled as projections, not a claim that a 100,000-tenant run happened.

+
+ + + + + + + + + +
ProjectionValueAssumption
Cells per 64 GiB node92371 MiB per cell measured, no headroom for a supervisor or a guest
Nodes for 100,000 tenants109one cell per tenant, no replicas
Chain transactions to provision400,000four per tenant, from the ledger above
Serial provisioning time168 hat the measured p50, down from 705 h; a real fleet provisions in parallel, so this is a parallelism requirement rather than a throughput ceiling
Chain storage15 GiBat the measured 8.8 MiB of chain data for 223 transactions, scaled to 400,000
+ The node count moved from 166 to 109 only because per-cell memory was measured lower on this run. It is a measurement of a settled cell, not an optimisation. +
+
+ +
+

Findings

+ +
+

Disclosure A failed registration leaves a public document

+

A cell whose chain account holds no funds commits the document locally and then fails the registration transaction. An unregistered document is public, so the write that appeared to fail is readable by any identity. One document was readable by an unrelated DID after the failure. Faster blocks do not change this: the failure is an unfunded account, not a timeout.

+
+ +
+

Not wired The ring is not a write gate today

+

Given an access-control tuple the ring refuses to sign for an unauthorised DID and signs for an authorised one in 54 ms. DefraDB's Cosmos provider returns no access decision, so the request carries no tuple and the ring signs for any authenticated caller. A writer revoked on the collection object still created a document.

+
+ +
+

Configuration Revocation latency is the cache lifetime, not the chain

+

A cell subscribed to chain transaction events denied a revoked reader 9 ms after Vera itself reported the revocation, which was 124 ms after submission. A cell without the subscription served the stale allow until its cache expired: 2.1 seconds into a 15 second lifetime on this run. The subscription is a security control, not a latency optimisation, and the gap it closes is bounded by the cache lifetime rather than by the block interval.

+
+ +
+

Ambient state Which key signs depends on who asks

+

In the first run the scale tenants' writes were signed locally rather than by the ring, because the writing identity was the cell's own node identity, whose signing configuration lives in a process-global store. Same flags, same code, different signer. The suite now uses a separate writer identity per tenant, and each tenant's four writes are ring-signed on this run. The original behaviour is evidence for the ambient-state risk the plan tracks as I-29.

+
+ +
+

Configuration The block interval was the provisioning cost

+

Four fifths of the time to provision a tenant was a CometBFT default the harness had never set. Setting the four consensus timeouts cut provisioning from 25.2 s to 6.1 s with no change to what the stack does. The remaining 70% is still chain-bound, so the next reduction has to come from batching the four transactions rather than from tuning consensus further.

+
+ +
+

Holds Isolation held at every tenant count

+

Thirty-two ordered cross-tenant reads were denied, each tenant saw exactly its own document, and a document replicated to another tenant's cell stayed gated because its registration lives on the chain rather than in a local store. Two cells sharing a schema and a policy do derive one collection topic, so topic separation is not automatic, but no block crossed it in 45 seconds without a replicator.

+
+ +
+

Holds Identity survives a kill

+

A cell killed with SIGKILL and restarted on the same data directory came back in 1.4 seconds with the same node identity, the same peer identity and address, the same documents, and a working ring signer.

+
+ +
+

Holds Guest packages are sealed

+

Four afterburner packages were checked as artifacts: three policy modules and the DefraDB engine itself at 1.48 MiB, each carrying a content hash over its sealed contents. This is the packaging half of spike S4; the guest runtime is exercised in defraburner rather than here.

+
+
+ +
+

Every recorded measurement

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MeasurementValue
stack_bring_up_secs22.4
ring_signed_batch_create_3_docs_ms3949
i26_forbidden_read_ms15
i26_absent_read_ms10
per_document_grant_3_docs_to_vera_visible_ms3650
grant_read_gate_eager_cell_lag_after_vera_ms0
revocation_vera_visible_after_submit_ms124
revocation_read_gate_eager_cell_ms9
revocation_read_gate_ttl_cell_ms2131
revoked_update_response{"data":{"update_Transcript":[]}}
create_after_collection_writer_revokedaccepted: no create gate under Vera
pre_authorised_readerrefused by the ring despite the Vera relation
ring_sign_with_vera_acp_check_ms54
ring_signs_without_acp_tupleyes
l8_refusal_latency_ms30022
l8_ring_recovery_to_first_signed_write_ms4731
s7_create_p50_ring_signed_ms1483
s7_create_p50_unsigned_ms769
s7_ring_round_trip_p50_ms714
dry_account_leaves_public_documentyes: 1 document readable by an unrelated DID
s8_shared_collection_topicbafyreibk625p5tcsamse3hqa7nzxuuxyf4277akdwhqdvhzzzqt4x2e4gy
s8_block_crossed_on_shared_topicno, within 45 s
c1_registration_is_chain_sideyes
i7_kill9_to_ready_ms1405
afterburner_sealed_packages4 packages, all sealed; engine 1,476,654 bytes
scale_tenants_provisioned32
scale_policies_on_vera35
scale_cell_rss_median_mib71
scale_read_p50_ms_one_tenant8
scale_read_p50_ms_all_tenants6
scale_cross_tenant_reads_denied32
scale_provision_p50_total_ms6058
scale_provision_p50_fund_ms974
scale_provision_p50_policy_ms1083
scale_provision_p50_register_object_ms846
scale_provision_p50_grant_writer_ms1361
scale_provision_p50_cell_ignition_ms300
scale_provision_p50_schema_ms19
scale_provision_p50_first_write_ms1384
scale_provision_p50_chain_share70% of the total
scale_projected_cells_per_64gib_node923
scale_projected_nodes_for_100k_tenants109
+
+
+ +
+

How to reproduce

+
+ + + + + + + + + +
ComponentSource
Verasourcenetwork/vera main @ ddcb612, binary verad
DefraDBdefradb.rs vclq/vera-compat (PR 1681)
Orbisorbis-rs vclq/vera-compat (PR 264)
Suitebackbone vclq/gents-cloud-vera (PR 30)
CommandGENTS_CLOUD_TENANTS=32 cargo test --test gents_cloud -- --ignored --nocapture
+ Binaries resolve from the VERA_BINARY, DEFRA_BINARY, ORBIS_BINARY and ORBIS_CLI_BINARY variables, or from the pins in backbone.toml. +
+
+ +
+ Measurements come from the suite's own recorder, from the chain's committed ledger read through CometBFT, from process resident-set sampling every 15 seconds, and from the component logs. Projected figures are marked as such and derive only from measurements in this run. Figures labelled "before" come from the 2026-09-03 20:01 UTC run of the same suite on the shipped CometBFT timeouts. +
+ +
From d76399a13302ff9c8264db2dec289ce22ddd0d54 Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 23:01:12 +0200 Subject: [PATCH 07/11] docs: copy-ready markdown tables for the 32-tenant run --- docs/gents-cloud-run-32-tenants.md | 180 +++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 docs/gents-cloud-run-32-tenants.md diff --git a/docs/gents-cloud-run-32-tenants.md b/docs/gents-cloud-run-32-tenants.md new file mode 100644 index 0000000..1c8b453 --- /dev/null +++ b/docs/gents-cloud-run-32-tenants.md @@ -0,0 +1,180 @@ +# 32-tenant Vera run, 2026-09-03 20:39 UTC + +Copy-ready tables from `cargo test --test gents_cloud -- --ignored` with +`GENTS_CLOUD_TENANTS=32`. The rendered version with charts and findings is +`gents-cloud-run-32-tenants.html` in this directory. + +13 of 13 scenarios passed in 376.5 s. 223 chain transactions committed, none +failed, over 352 blocks at a 1.06 s interval. + +## Consensus timings + +The devnet sets these in `crates/sourcehub-harness/src/genesis.rs`. Before, the +harness left CometBFT's shipped values in place, and a block cost 5.03 s. + +| Setting | Shipped | Devnet | +|---|---|---| +| `timeout_propose` | 3s | 500ms | +| `timeout_prevote` | 1s | 500ms | +| `timeout_precommit` | 1s | 500ms | +| `timeout_commit` | 5s | 1s | + +| Measure | Before | After | Change | +|---|---|---|---| +| Block interval (s) | 5.03 | 1.06 | 4.7x | +| Provision one tenant (ms) | 25167 | 6058 | 4.2x | +| Suite wall time (s) | 1212.5 | 376.5 | 3.2x | +| Signed write p50 (ms) | 4909 | 1483 | 3.3x | +| Stack bring-up (s) | 83.5 | 22.4 | 3.7x | + +## Scenarios + +| # | Scenario | Discharges | Wall time (s) | +|---|---|---|---| +| 1 | `h1_node_identity_no_privileged_read` | gents-cloud §1.2 row 2 [V], H1 §10.2, I-2, spike S6, readiness C4 | 8.3 | +| 2 | `i26_absence_denial_indistinguishable` | gents-cloud §11.6, I-26, Phase 2 gate | 0.0 | +| 3 | `grant_asymmetry` | gents-cloud §1.6 (asymmetric grant granularity), §11.4 revocation cost | 8.7 | +| 4 | `h5_two_clocks` | gents-cloud §10.5 (H5, two clocks), §1.6 rows 2 and 3, spike S5, Phase 2 gate I-16 | 15.6 | +| 5 | `h12_pre_on_vera` | gents-cloud §10.6 (H12), §1.6 PRE row, open [?] on the shipping backend | 3.3 | +| 6 | `h3_ring_gate_mechanism` | gents-cloud §12.2 (H3), §1.6 row 2, spike S7 [?] on what the ring checks | 0.9 | +| 7 | `l8_ring_below_threshold` | gents-cloud §24 rung L8, §22.4, decision 43 | 34.8 | +| 8 | `s7_signed_write_cost` | gents-cloud §12.3 cost, spike S7, §19.1 'ring round trip: to be measured' | 18.3 | +| 9 | `dry_account` | gents-cloud §1.6 (a cell that cannot write because its account is dry), §1.2 (unregistered means public), §24 | 3.3 | +| 10 | `s8_topic_collision_c1` | gents-cloud §11.7 (A-1, A-2), spike S8, readiness C1, I-30 | 46.9 | +| 11 | `i7_kill9_identity` | gents-cloud I-7, §5.3 'golden kill -9', §17.6 VolumeRestore, §1.2 peerstore row | 3.1 | +| 12 | `afterburner_sealed_packages` | gents-cloud §1.1 (the manifold is the whole permission surface), H4, H11, §26 ban list, I-3 | 0.2 | +| 13 | `scale_per_tenant_cost` | gents-cloud §19.1 density, §20.1 (the workspace is the shard key), §20.6 growth stages | 210.7 | + +## Scaling curve + +| Tenants | Cell RSS median (MiB) | All cells (MiB) | Read p50 (ms) | Provision p50 (ms) | Cell ignition p50 (ms) | +|---|---|---|---|---|---| +| 8 | 100 | 750 | 6 | 6282 | 311 | +| 16 | 100 | 1505 | 7 | 6101 | 303 | +| 24 | 70 | 1855 | 6 | 6261 | 303 | +| 32 | 71 | 2572 | 6 | 6058 | 300 | + +## Provisioning breakdown + +| Step | p50 (ms) | Waits on | +|---|---|---| +| Fund the tenant account | 974 | Vera block | +| Create the policy | 1083 | Vera block | +| Register the collection object | 846 | Vera block | +| Grant writer | 1361 | Vera block | +| Ignite the cell | 300 | the process starting | +| Add the schema | 19 | local | +| First ring-signed write | 1384 | ring round trip and a Vera registration | +| **Total** | **6058** | 70% is the four transactions alone | + +Each step is the median across 32 tenants, so the parts sum to 5967 ms rather +than exactly to the median total. + +## What the chain did + +| Message | Count | +|---|---| +| `/vera.acp.MsgDirectPolicyCmd` | 73 | +| `/vera.acp.MsgBearerPolicyCmd` | 69 | +| `/cosmos.bank.v1beta1.MsgSend` | 41 | +| `/vera.acp.MsgCreatePolicy` | 34 | +| `/vera.bulletin.MsgAddCollaborator` | 3 | +| `/vera.bulletin.MsgCreatePost` | 2 | +| `/vera.bulletin.MsgRegisterNamespace` | 1 | +| **Total** | **223** | + +| Chain measure | Value | +|---|---| +| Blocks produced | 352 | +| Block interval | 1.06 s | +| Transactions committed | 223 | +| Transactions failed | 0 | +| Gas used | 25,471,721 | +| Gas wanted | 368,516,753 | +| Policies registered | 35 | +| Chain data on disk | 8.8 MiB | + +## Component footprint + +| Component | Count | Peak resident (MiB) | Each (MiB) | +|---|---|---|---| +| DefraDB cells | 35 | 2846 | 81 | +| Vera (`verad`) | 1 | 218 | 218 | +| Orbis ring | 3 | 139 | 46 | + +Maxima over 33 samples taken every 15 seconds. + +## Projection to 100,000 tenants + +Arithmetic over the measurements above. No 100,000-tenant run happened. + +| Projection | Value | Assumption | +|---|---|---| +| Cells per 64 GiB node | 923 | 71 MiB per cell measured, no headroom for a supervisor or a guest | +| Nodes for 100,000 tenants | 109 | one cell per tenant, no replicas | +| Chain transactions to provision | 400,000 | four per tenant | +| Serial provisioning time | 168 h | at the measured p50; a real fleet provisions in parallel | +| Chain storage | 15 GiB | 8.8 MiB per 223 transactions, scaled | + +## Every recorded measurement + +| Measurement | Value | +|---|---| +| `stack_bring_up_secs` | 22.4 | +| `ring_signed_batch_create_3_docs_ms` | 3949 | +| `i26_forbidden_read_ms` | 15 | +| `i26_absent_read_ms` | 10 | +| `per_document_grant_3_docs_to_vera_visible_ms` | 3650 | +| `grant_read_gate_eager_cell_lag_after_vera_ms` | 0 | +| `revocation_vera_visible_after_submit_ms` | 124 | +| `revocation_read_gate_eager_cell_ms` | 9 | +| `revocation_read_gate_ttl_cell_ms` | 2131 (cache ttl 15s) | +| `revoked_update_response` | `{"data":{"update_Transcript":[]}}` | +| `create_after_collection_writer_revoked` | accepted (no create gate under Vera: DefraDB sends the ring no ACP tuple; G-4 and H3 remain open) | +| `pre_authorised_reader` | refused by the ring despite the Vera relation | +| `ring_sign_with_vera_acp_check_ms` | 54 | +| `ring_signs_without_acp_tuple` | yes (DefraDB's Vera write path sends none) | +| `l8_refusal_latency_ms` | 30022 | +| `l8_ring_recovery_to_first_signed_write_ms` | 4731 | +| `s7_create_p50_ring_signed_ms` | 1483 | +| `s7_create_p50_unsigned_ms` | 769 | +| `s7_ring_round_trip_p50_ms` | 714 (signed minus unsigned over 8 samples; both are dominated by the Vera registration, so a value at or below zero would mean the round trip is not separable at this sample size, not that it is free) | +| `dry_account_leaves_public_document` | yes: 1 document readable by an unrelated DID after the failed registration | +| `s8_shared_collection_topic` | `bafyreibk625p5tcsamse3hqa7nzxuuxyf4277akdwhqdvhzzzqt4x2e4gy` | +| `s8_block_crossed_on_shared_topic` | no: within 45 s, with a peer connection and a subscription but no replicator | +| `c1_registration_is_chain_side` | yes (a replicated document stays registered on Vera, so the receiving cell gates it) | +| `i7_kill9_to_ready_ms` | 1405 | +| `afterburner_sealed_packages` | 4 packages, all sealed; engine 1,476,654 bytes | +| `scale_tenants_provisioned` | 32 | +| `scale_policies_on_vera` | 35 | +| `scale_cell_rss_median_mib` | 71 | +| `scale_read_p50_ms_one_tenant` | 8 | +| `scale_read_p50_ms_all_tenants` | 6 | +| `scale_cross_tenant_reads_denied` | 32 ordered pairs | +| `scale_provision_p50_total_ms` | 6058 | +| `scale_provision_p50_fund_ms` | 974 | +| `scale_provision_p50_policy_ms` | 1083 | +| `scale_provision_p50_register_object_ms` | 846 | +| `scale_provision_p50_grant_writer_ms` | 1361 | +| `scale_provision_p50_cell_ignition_ms` | 300 | +| `scale_provision_p50_schema_ms` | 19 | +| `scale_provision_p50_first_write_ms` | 1384 | +| `scale_provision_p50_chain_share` | 70% of the total is the four Vera transactions waiting for a block | +| `scale_projected_cells_per_64gib_node` | 923 (projected) | +| `scale_projected_nodes_for_100k_tenants` | 109 (projected) | + +## Reproduce + +| Component | Source | +|---|---| +| Vera | `sourcenetwork/vera` main @ ddcb612, binary `verad` | +| DefraDB | defradb.rs `vclq/vera-compat` (PR 1681) | +| Orbis | orbis-rs `vclq/vera-compat` (PR 264) | +| Suite | backbone `vclq/gents-cloud-vera` (PR 30) | + +``` +GENTS_CLOUD_TENANTS=32 cargo test --test gents_cloud -- --ignored --nocapture +``` + +Binaries resolve from `VERA_BINARY`, `DEFRA_BINARY`, `ORBIS_BINARY` and +`ORBIS_CLI_BINARY`, or from the pins in `backbone.toml`. From e18f1f836968ec6077850f2c3283529b13c1fd26 Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Thu, 3 Sep 2026 23:01:27 +0200 Subject: [PATCH 08/11] docs: give the run report a doctype and charset so it opens standalone --- docs/gents-cloud-run-32-tenants.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/gents-cloud-run-32-tenants.html b/docs/gents-cloud-run-32-tenants.html index 8f1b56d..32fe6cd 100644 --- a/docs/gents-cloud-run-32-tenants.html +++ b/docs/gents-cloud-run-32-tenants.html @@ -1,3 +1,7 @@ + + + + 32-Tenant Vera Run @@ -677,3 +681,4 @@

How to reproduce

+ From d26b77846db5bb2694fd43739ec19dfe0c8f6efd Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Fri, 4 Sep 2026 01:24:04 +0200 Subject: [PATCH 09/11] fix(gents-cloud): read cell RSS through ps on macOS, /proc is Linux-only --- tests/gents_cloud/scale.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/gents_cloud/scale.rs b/tests/gents_cloud/scale.rs index 0710c27..2da1c8d 100644 --- a/tests/gents_cloud/scale.rs +++ b/tests/gents_cloud/scale.rs @@ -435,14 +435,29 @@ fn checkpoint_every(tenant_count: usize) -> usize { (tenant_count / 4).max(1) } -/// Resident memory of a cell's process, from `/proc//status`. +/// Resident memory of a cell's process, in KiB. +/// +/// `/proc` is Linux-only, so macOS reads the same figure through `ps`, which +/// reports RSS in KiB there as well. A cell whose memory cannot be read is +/// left out of the curve rather than counted as zero: a zero would render as a +/// real measurement of an idle cell. fn cell_rss_kib(cell: &Cell) -> Option { let pid = cell.node.process.id()?; - let status = std::fs::read_to_string(format!("/proc/{}/status", pid)).ok()?; - status - .lines() - .find_map(|line| line.strip_prefix("VmRSS:")) - .and_then(|value| value.split_whitespace().next()?.parse().ok()) + if cfg!(target_os = "linux") { + let status = std::fs::read_to_string(format!("/proc/{}/status", pid)).ok()?; + return status + .lines() + .find_map(|line| line.strip_prefix("VmRSS:")) + .and_then(|value| value.split_whitespace().next()?.parse().ok()); + } + let output = std::process::Command::new("ps") + .args(["-o", "rss=", "-p", &pid.to_string()]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8_lossy(&output.stdout).trim().parse().ok() } /// Median wall time of three reads of `collection` as `identity`. From 6d8cd80dc174ecac0828f0f16d6655cfdba6df24 Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Fri, 4 Sep 2026 01:38:38 +0200 Subject: [PATCH 10/11] fix(full-stack): give the Orbis-signing nodes an ed25519 service key --- tests/full_stack.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/full_stack.rs b/tests/full_stack.rs index 81bff1e..1c25ba0 100644 --- a/tests/full_stack.rs +++ b/tests/full_stack.rs @@ -14,6 +14,7 @@ use std::time::{Duration, Instant}; use defra_harness::node::RustNode; use defra_harness::sse::{open_acp_events_sse, wait_for_acp_invalidation}; use defra_harness::{DefraClient, NodeKind}; +use orbis_harness::cli::ed25519_identity_hex; use orbis_harness::cli::signer_did_for_pk; use orbis_harness::cli::types::RingPayload; use orbis_harness::defradb::identity::{did_key_from_secp256k1, DefraHttpClient}; @@ -546,7 +547,7 @@ async fn secure_training_data_compartments() { endpoint: ring.node(0).grpc_addr(), ring_id: ring_id.clone(), derivation: "acme-corp".to_string(), - service_identity: None, + service_identity: Some(ed25519_identity_hex(&acme_defra_svc.private_key_hex)), }); let acme_defra = start_node(&acme_defra_node, acme_defra_config, Duration::from_secs(30)) @@ -587,7 +588,7 @@ async fn secure_training_data_compartments() { endpoint: ring.node(0).grpc_addr(), ring_id: ring_id.clone(), derivation: "platform".to_string(), - service_identity: None, + service_identity: Some(ed25519_identity_hex(&platform_defra_svc.private_key_hex)), }); let platform_defra = start_node( @@ -879,7 +880,7 @@ async fn secure_training_data_compartments() { endpoint: ring.node(0).grpc_addr(), ring_id: ring_id.clone(), derivation: "globex-inc".to_string(), - service_identity: None, + service_identity: Some(ed25519_identity_hex(&globex_defra_svc.private_key_hex)), }); let globex_defra = start_node( From d814f46c594e26266c4a3956ed97c50cb96d0a91 Mon Sep 17 00:00:00 2001 From: Theo Bulut Date: Fri, 4 Sep 2026 02:56:26 +0200 Subject: [PATCH 11/11] fix(ci): build defra and hubd from source when the pinned commit has no artifact --- .github/scripts/ensure-binaries.sh | 69 +++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/.github/scripts/ensure-binaries.sh b/.github/scripts/ensure-binaries.sh index 8ddf56e..1a45329 100755 --- a/.github/scripts/ensure-binaries.sh +++ b/.github/scripts/ensure-binaries.sh @@ -8,12 +8,15 @@ set -euo pipefail # (branch, tag, or commit SHA). # # defra and hub.rs binaries are downloaded as GitHub Actions artifacts -# from their CI workflows. The exact commit for each ref must have a -# successful CI run with uploaded artifacts, or this script fails. +# from their CI workflows when the pinned commit has them. Those artifacts +# are only produced on a push to main, so a ref pinned to a branch still +# under review has none; that case falls back to a source build rather than +# failing, because a pinned branch is the normal state while a cross-repo +# change is in flight. # -# orbis-rs has no CI artifact pipeline, so this script resolves the -# commit, clones/fetches into a persistent local checkout, and does -# an incremental cargo build --release. +# orbis-rs has no CI artifact pipeline at all, so it is always built from +# source: resolve the commit, clone or fetch into a persistent local +# checkout, and do an incremental cargo build --release. # # Required: # backbone.toml — in the repo root (or any ancestor directory) @@ -90,11 +93,11 @@ resolve_commit() { } # Download a binary artifact from a GitHub Actions workflow run. +# Try to place $binary_name in the cache from a CI artifact. Returns non-zero +# without exiting when the commit has no successful run or that run published +# no matching artifact, so the caller can build from source instead. download_artifact() { - local repo=$1 ref=$2 artifact_name=$3 binary_name=$4 - local commit - commit=$(resolve_commit "$repo" "$ref") - echo "$repo: $ref → ${commit:0:12}" + local repo=$1 commit=$2 artifact_name=$3 binary_name=$4 local run_id run_id=$(gh run list -R "sourcenetwork/$repo" \ @@ -102,29 +105,33 @@ download_artifact() { --limit 1 --json databaseId -q '.[0].databaseId') if [[ -z "$run_id" ]]; then - echo " ERROR: no successful CI run found for $repo@${commit:0:12}" >&2 - echo " The CI for $repo must complete successfully before backbone CI can run." >&2 - exit 1 + echo " no successful CI run for $repo@${commit:0:12}" + return 1 fi echo " Downloading $artifact_name from run $run_id..." local tmp_dir tmp_dir=$(mktemp -d) - gh run download "$run_id" -R "sourcenetwork/$repo" \ - --name "$artifact_name" --dir "$tmp_dir" + if ! gh run download "$run_id" -R "sourcenetwork/$repo" \ + --name "$artifact_name" --dir "$tmp_dir" 2>/dev/null; then + echo " run $run_id published no $artifact_name" + rm -rf "$tmp_dir" + return 1 + fi local found found=$(find "$tmp_dir" -type f | head -1) if [[ -z "$found" ]]; then - echo " ERROR: artifact $artifact_name was empty" >&2 + echo " artifact $artifact_name was empty" rm -rf "$tmp_dir" - exit 1 + return 1 fi cp "$found" "$CACHE_DIR/$binary_name" chmod +x "$CACHE_DIR/$binary_name" rm -rf "$tmp_dir" - echo " $binary_name: ready" + echo " $binary_name: ready (artifact)" + return 0 } # --- orbis-rs helper functions (source build, no CI artifacts) --- @@ -246,11 +253,31 @@ for var in DEFRA_REPO DEFRA_REF HUBD_REPO HUBD_REF ORBIS_REPO ORBIS_REF; do echo " $var=${!var}" done -# defra and hub.rs: download release artifacts from their CI +# defra and hub.rs: prefer a CI artifact, build from source when the pinned +# commit has none. Their artifact jobs are gated on a push to main, so a ref +# pinned to a branch under review always takes the source path. echo "" -echo "--- defra/hub.rs (GitHub Actions artifacts) ---" -download_artifact "$DEFRA_REPO" "$DEFRA_REF" "defra-iroh-aarch64-apple-darwin" "defra-iroh" -download_artifact "$HUBD_REPO" "$HUBD_REF" "hubd-aarch64-apple-darwin" "hubd" +echo "--- defra (artifact, else source) ---" +DEFRA_COMMIT=$(resolve_commit "$DEFRA_REPO" "$DEFRA_REF") +echo "$DEFRA_REPO: $DEFRA_REF → ${DEFRA_COMMIT:0:12}" +if ! download_artifact "$DEFRA_REPO" "$DEFRA_COMMIT" \ + "defra-iroh-aarch64-apple-darwin" "defra-iroh"; then + echo " Falling back to a source build." + build_if_missing "$DEFRA_REPO" "$DEFRA_REF" "$DEFRA_COMMIT" \ + "cli:defra:defra-iroh:sourcehub,orbis,iroh" + prune_old_versions "$DEFRA_REPO" +fi + +echo "" +echo "--- hub.rs (artifact, else source) ---" +HUBD_COMMIT=$(resolve_commit "$HUBD_REPO" "$HUBD_REF") +echo "$HUBD_REPO: $HUBD_REF → ${HUBD_COMMIT:0:12}" +if ! download_artifact "$HUBD_REPO" "$HUBD_COMMIT" \ + "hubd-aarch64-apple-darwin" "hubd"; then + echo " Falling back to a source build." + build_if_missing "$HUBD_REPO" "$HUBD_REF" "$HUBD_COMMIT" "hubd:hubd" + prune_old_versions "$HUBD_REPO" +fi # orbis-rs: no CI artifact pipeline, build from source echo ""